├── requirements.txt
├── pheno_info.py
├── download_abide.py
├── .gitignore
├── README.md
├── model.py
├── utils.py
├── nn_evaluate.py
├── prepare_data.py
├── Experiments_Colab_From_Xuehan.ipynb
├── LICENSE
├── nn.py
└── data
    └── list-of-subjects.csv


/requirements.txt:
--------------------------------------------------------------------------------
1 | tensorflow[and-cuda]
2 | pandas
3 | numpy
4 | docopt
5 | scikit-learn
6 | scipy
7 | 


--------------------------------------------------------------------------------
/pheno_info.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | """
 5 | 
 6 | Describe phenotypical info
 7 | 
 8 | Usage:
 9 |   pheno_info.py
10 |   pheno_info.py (-h | --help)
11 | 
12 | Options:
13 |   -h --help           Show this screen
14 | 
15 | """
16 | 
17 | from docopt import docopt
18 | from utils import load_phenotypes_2
19 | 
20 | 
21 | if __name__ == "__main__":
22 | 
23 |     arguments = docopt(__doc__)
24 | 
25 |     pheno_path = "./data/phenotypes/Phenotypic_V1_0b_preprocessed1.csv"
26 |     #pheno_path = "./data/phenotypes/Phenotypic_V1_0b.csv"
27 |     pheno = load_phenotypes_2(pheno_path)
28 | 
29 |     for site, df_site in pheno.groupby("SITE_ID"):
30 |         asd = df_site[df_site["DX_GROUP"] == 0]
31 |         tc = df_site[df_site["DX_GROUP"] == 1]
32 |         fmt = "% 8s &    % 8.1f (% 8.1f) & % 8.1f (% 8.1f) & % 8.1f (% 8.1f)   &  M % 3d, F % 3d &    % 8.1f (% 8.1f) &  M % 3d, F % 3d \\\\"
33 | 
34 |         print (fmt % (
35 |             site,
36 |             asd["AGE"].mean(),
37 |             asd["AGE"].std(),
38 |             asd["HANDEDNESS_SCORES"].mean(),
39 |             asd["HANDEDNESS_SCORES"].std(),
40 |             asd["FIQ"].mean(),
41 |             asd["FIQ"].std(),
42 |             int(asd[asd["SEX"] == "M"].shape[0]),
43 |             int(asd[asd["SEX"] == "F"].shape[0]),
44 |             tc["AGE"].mean(),
45 |             tc["AGE"].std(),
46 |             int(tc[tc["SEX"] == "M"].shape[0]),
47 |             int(tc[tc["SEX"] == "F"].shape[0]),
48 |         ))
49 | 
50 |     # for site, df_site in pheno.groupby("SITE_ID"):
51 |     #     asd = df_site[df_site["DX_GROUP"] == 0]
52 |     #     tc = df_site[df_site["DX_GROUP"] == 1]
53 |     #     fmt = "% 8s &    % 8.1f (% 8.1f) & % 8.1f (% 8.1f)    &  M % 3d, F % 3d &    % 8.1f (% 8.1f) &  M % 3d, F % 3d \\\\"
54 | 
55 |     #     print (fmt % (
56 |     #         site,
57 |     #         asd["AGE"].mean(),
58 |     #         asd["AGE"].std(),
59 |     #         #int(0),
60 |     #         #int(0),
61 |     #         #asd["ADOS"].mean(),
62 |     #         #asd["ADOS"].std(),
63 |     #         asd["MEAN_FD"].mean(),
64 |     #         asd["MEAN_FD"].std(),
65 |     #         int(asd[asd["SEX"] == "M"].shape[0]),
66 |     #         int(asd[asd["SEX"] == "F"].shape[0]),
67 |     #         tc["AGE"].mean(),
68 |     #         tc["AGE"].std(),
69 |     #         int(tc[tc["SEX"] == "M"].shape[0]),
70 |     #         int(tc[tc["SEX"] == "F"].shape[0]),
71 |     #     ))
72 | 


--------------------------------------------------------------------------------
/download_abide.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | """
 5 | 
 6 | Data download
 7 | 
 8 | Usage:
 9 |   download_abide.py [--pipeline=cpac] [--strategy=filt_global] [<derivative> ...]
10 |   download_abide.py (-h | --help)
11 | 
12 | Options:
13 |   -h --help              Show this screen
14 |   --pipeline=cpac        Pipeline [default: cpac]
15 |   --strategy=filt_global Strategy [default: filt_global]
16 |   derivative             Derivatives to download
17 | 
18 | """
19 | 
20 | 
21 | import os
22 | import urllib
23 | import urllib.request
24 | from docopt import docopt
25 | 
26 | 
27 | 
28 | def collect_and_download(derivative, pipeline, strategy, out_dir):
29 | 
30 |     s3_prefix = "https://s3.amazonaws.com/fcp-indi/data/Projects/ABIDE_Initiative"
31 | 
32 |     derivative = derivative.lower()
33 |     pipeline = pipeline.lower()
34 |     strategy = strategy.lower()
35 | 
36 |     if "roi" in derivative:
37 |         extension = ".1D"
38 |     else:
39 |         extension = ".nii.gz"
40 | 
41 |     if not os.path.exists(out_dir):
42 |         os.makedirs(out_dir)
43 | 
44 |     s3_pheno_file = open("./data/phenotypes/Phenotypic_V1_0b_preprocessed1.csv", "r")
45 |     pheno_list = s3_pheno_file.readlines()
46 | 
47 |     header = pheno_list[0].split(",")
48 |     file_idx = header.index("FILE_ID")
49 | 
50 |     s3_paths = []
51 |     for pheno_row in pheno_list[1:]:
52 |         cs_row = pheno_row.split(",")
53 |         row_file_id = cs_row[file_idx]
54 |         if row_file_id == "no_filename":
55 |             continue
56 |         filename = row_file_id + "_" + derivative + extension
57 |         s3_path = "/".join([s3_prefix, "Outputs", pipeline, strategy, derivative, filename])
58 |         s3_paths.append(s3_path)
59 | 
60 |     total_num_files = len(s3_paths)
61 |     for path_idx, s3_path in enumerate(s3_paths):
62 |         rel_path = s3_path.lstrip(s3_prefix).split("/")[-1]
63 |         download_file = os.path.join(out_dir, rel_path)
64 |         download_dir = os.path.dirname(download_file)
65 |         if not os.path.exists(download_dir):
66 |             os.makedirs(download_dir)
67 |         if not os.path.exists(download_file):
68 |             print ("Retrieving: %s" % download_file)
69 |             urllib.request.urlretrieve(s3_path, download_file)
70 |             print ("%.3f%% percent complete" % (100*(float(path_idx+1)/total_num_files)))
71 |         else:
72 |             print ("File %s already exists, skipping..." % download_file)
73 | 
74 | 
75 | if __name__ == "__main__":
76 | 
77 |     arguments = docopt(__doc__)
78 | 
79 |     if not arguments['<derivative>']:
80 |         arguments['<derivative>'] = ['rois_aal', 'rois_cc200', 'rois_dosenbach160', 'rois_ez', 'rois_ho', 'rois_tt']
81 | 
82 |     pipeline = arguments.get('pipeline', 'cpac')
83 |     strategy = arguments.get('strategy', 'filt_global')
84 | 
85 |     out_dir = os.path.abspath("data/functionals/cpac/filt_global/")
86 | 
87 |     for derivative in arguments['<derivative>']:
88 |         collect_and_download(derivative, pipeline, strategy, os.path.join(out_dir, derivative))
89 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | .make_env_ready.sh
  2 | 
  3 | *.npy
  4 | 
  5 | # Byte-compiled / optimized / DLL files
  6 | __pycache__/
  7 | *.py[cod]
  8 | *$py.class
  9 | 
 10 | # C extensions
 11 | *.so
 12 | 
 13 | # Distribution / packaging
 14 | .Python
 15 | build/
 16 | develop-eggs/
 17 | dist/
 18 | downloads/
 19 | eggs/
 20 | .eggs/
 21 | lib/
 22 | lib64/
 23 | parts/
 24 | sdist/
 25 | var/
 26 | wheels/
 27 | share/python-wheels/
 28 | *.egg-info/
 29 | .installed.cfg
 30 | *.egg
 31 | MANIFEST
 32 | 
 33 | # PyInstaller
 34 | #  Usually these files are written by a python script from a template
 35 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 36 | *.manifest
 37 | *.spec
 38 | 
 39 | # Installer logs
 40 | pip-log.txt
 41 | pip-delete-this-directory.txt
 42 | 
 43 | # Unit test / coverage reports
 44 | htmlcov/
 45 | .tox/
 46 | .nox/
 47 | .coverage
 48 | .coverage.*
 49 | .cache
 50 | nosetests.xml
 51 | coverage.xml
 52 | *.cover
 53 | *.py,cover
 54 | .hypothesis/
 55 | .pytest_cache/
 56 | cover/
 57 | 
 58 | # Translations
 59 | *.mo
 60 | *.pot
 61 | 
 62 | # Django stuff:
 63 | *.log
 64 | local_settings.py
 65 | db.sqlite3
 66 | db.sqlite3-journal
 67 | 
 68 | # Flask stuff:
 69 | instance/
 70 | .webassets-cache
 71 | 
 72 | # Scrapy stuff:
 73 | .scrapy
 74 | 
 75 | # Sphinx documentation
 76 | docs/_build/
 77 | 
 78 | # PyBuilder
 79 | .pybuilder/
 80 | target/
 81 | 
 82 | # Jupyter Notebook
 83 | .ipynb_checkpoints
 84 | 
 85 | # IPython
 86 | profile_default/
 87 | ipython_config.py
 88 | 
 89 | # pyenv
 90 | #   For a library or package, you might want to ignore these files since the code is
 91 | #   intended to run in multiple environments; otherwise, check them in:
 92 | # .python-version
 93 | 
 94 | # pipenv
 95 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 96 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 97 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 98 | #   install all needed dependencies.
 99 | #Pipfile.lock
100 | 
101 | # poetry
102 | #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
103 | #   This is especially recommended for binary packages to ensure reproducibility, and is more
104 | #   commonly ignored for libraries.
105 | #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
106 | #poetry.lock
107 | 
108 | # pdm
109 | #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
110 | #pdm.lock
111 | #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
112 | #   in version control.
113 | #   https://pdm.fming.dev/#use-with-ide
114 | .pdm.toml
115 | 
116 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
117 | __pypackages__/
118 | 
119 | # Celery stuff
120 | celerybeat-schedule
121 | celerybeat.pid
122 | 
123 | # SageMath parsed files
124 | *.sage.py
125 | 
126 | # Environments
127 | .env
128 | .venv
129 | env/
130 | venv/
131 | ENV/
132 | env.bak/
133 | venv.bak/
134 | 
135 | # Spyder project settings
136 | .spyderproject
137 | .spyproject
138 | 
139 | # Rope project settings
140 | .ropeproject
141 | 
142 | # mkdocs documentation
143 | /site
144 | 
145 | # mypy
146 | .mypy_cache/
147 | .dmypy.json
148 | dmypy.json
149 | 
150 | # Pyre type checker
151 | .pyre/
152 | 
153 | # pytype static type analyzer
154 | .pytype/
155 | 
156 | # Cython debug symbols
157 | cython_debug/
158 | 
159 | # PyCharm
160 | #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
161 | #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
162 | #  and can be added to the global gitignore or merged into this file.  For a more nuclear
163 | #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
164 | #.idea/
165 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # MADE-for-ASD: A Multi-Atlas Deep Ensemble Network for Detecting Autism Spectrum Disorder
 2 | 
 3 | This repository contains the code of our above-titled paper, published in the "Computers in Biology and Medicine" journal. **Codes were primarily developed by [Xuehan Liu](https://github.com/lxhwww)**.
 4 | 
 5 | Paper link: [[ScienceDirect](https://doi.org/10.1016/j.compbiomed.2024.109083)] [[arXiv](https://arxiv.org/abs/2407.07076)] [[PDF](https://hasan-rakibul.github.io/pdfs/liu2024made.pdf)]
 6 | 
 7 | **Author**: Xuehan Liu*, Md Rakibul Hasan*, Tom Gedeon and Md Zakir Hossain <br>
 8 | (*Equal contribution)
 9 | 
10 | **Abstract:** In response to the global need for efficient early diagnosis of Autism Spectrum Disorder (ASD), we aim to bridge the gap between traditional, time-consuming diagnostic methods and potential automated solutions. To this end, we propose a multi-atlas deep ensemble network, **MADE-for-ASD**, that integrates multiple atlases of the brain's functional magnetic resonance imaging (fMRI) data through a weighted deep ensemble network. We further integrate demographic information into the prediction workflow, which enhances ASD detection accuracy and offers a more holistic perspective on patient profiling. We experiment with the well-known publicly available Autism Brain Imaging Data Exchange (ABIDE), consisting of resting state fMRI data from 17 different laboratories around the globe. Our proposed system achieves an accuracy of 75.2% on the whole dataset and 96.40% on a subset – both surpassing the reported ASD detection performances in the literature. The proposed system can potentially pave the way for more cost-effective, efficient and scalable strategies in ASD diagnosis.
11 | 
12 | **Disclaimer:** While we have made every effort to ensure the completeness of this codebase, some components (implementation of k-fold cross-validation and weighted voting mechanism) are missing. We, the co-authors, have actively tried to contact the first author, [Xuehan Liu](https://github.com/lxhwww), to obtain the most up-to-date version of the code once we detected the issue. Unfortunately, our efforts were unsuccessful, as her university email is no longer active following her graduation. We will promptly update this repository should we receive any further information. To get the missing bits, we recommend reaching out to her directly if possible in any way.
13 | 
14 | ## Environment Setup
15 | 
16 | Please be aware that this code is meant to be run with Python 3 under Linux (MacOS may work, Windows probably not). Download the packages from `requirements.txt`:
17 | 
18 | ```bash
19 | python -m pip install -r requirements.txt
20 | ```
21 | We use the [ABIDE I dataset](http://fcon_1000.projects.nitrc.org/indi/abide/). We use the pre-processing method from the [preprocessed-connectomes-project](https://github.com/preprocessed-connectomes-project/abide).
22 | 
23 | 
24 | ## Run Experiments
25 | 
26 | We put all the experiment command and the result in the `Experiments.ipynb` file. As an example, we prepare the hdf5 files in advance of the CC200, AAL, EZ atlas data from the NYU site, and show the entire experiment on this data as followings:
27 | 
28 | 1. Run `download_abide.py` to download the raw data.
29 | 
30 | We use the command below:
31 | 
32 | ```bash
33 | #download_abide.py [--pipeline=cpac] [--strategy=filt_global] [<derivative> ...]
34 | python download_abide.py
35 | ```
36 | 
37 | 2. To show the demographic Information of ABIDE I.
38 | 
39 | ```bash
40 | python pheno_info.py
41 | ```
42 | 
43 | 3. Run `prepare_data.py` to compute the correlation. Then we can get the hdf5 files. The dataset (hdf5) can be downloaded from [link](https://drive.google.com/file/d/1-WyQ7IOqSxaGcoA6MR4ydJdazlqzKYMY/view?usp=drive_link); you need to put it in the "data" folder. Or you can simply run the code as:
44 | 
45 | ```bash
46 | #download_abide.py [--folds=N] [--whole] [--male] [--threshold] [--leave-site-out] [--NYU-site-out] [<derivative> ...]
47 | 
48 | python prepare_data.py --folds=10 --whole cc200 aal ez
49 | ```
50 | 
51 | 4. Using Stacked Sparse Denoising Autoencoder (SSDAE) to perform Multi-atlas Deep Feature Representation, and using multilayer perceptron (MLP) and ensemble learning to classify the ASD and TC.
52 | 
53 | ```bash
54 | #nn.py [--whole] [--male] [--threshold] [--leave-site-out] [<derivative> ...]
55 | 
56 | rm ./data/models/*mlp*
57 | python nn.py --whole cc200 aal ez
58 | ```  
59 | 
60 | 5. Evaluating the MLP model on test dataset. You can use the saved models from provious or download the models from the [link](https://drive.google.com/drive/folders/1rIZpXdafzI-nb0YonkL0XQs6pOSSxRUf?usp=drive_link), and **can run this command directly without running previous steps**.
61 | 
62 | ```bash
63 | #nn_evaluate.py [--whole] [--male] [--threshold] [--leave-site-out] [--NYU-site-out] [<derivative> ...]
64 | 
65 | python nn_evaluate.py --whole cc200 aal ez
66 | ```
67 |         
68 | ## Quality Analysis and Visualisation
69 | 
70 | All the analysis for the subset and the visualisation of top ROIs, which mentioned in our work, are included in the `analysis_and_visulaisation.ipynb` file.
71 | 
72 | ## Citation
73 | If you find this repository useful in your research, please cite our paper:
74 | ```bibtex
75 | @article{liu2024made,
76 |     title = {{MADE}-for-{ASD}: A Multi-Atlas Deep Ensemble Network for Diagnosing Autism Spectrum Disorder},
77 |     author = {Xuehan Liu and Md Rakibul Hasan and Tom Gedeon and Md Zakir Hossain},
78 |     journal = {Computers in Biology and Medicine},
79 |     volume = {182},
80 |     pages = {109083},
81 |     year = {2024},
82 |     issn = {0010-4825},
83 |     doi = {10.1016/j.compbiomed.2024.109083}
84 | }
85 | ```
86 | 


--------------------------------------------------------------------------------
/model.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | import math
  5 | #import tensorflow as tf
  6 | import tensorflow.compat.v1 as tf
  7 | 
  8 | def ae(input_size, code_size,
  9 |        corruption=0.0, tight=False,
 10 |        enc=tf.nn.tanh, dec=tf.nn.tanh):
 11 |     """
 12 | 
 13 |     Autoencoder model: input_size -> code_size -> input_size
 14 |     Supports tight weights and corruption.
 15 | 
 16 |     """
 17 | 
 18 |     # Define data input placeholder
 19 |     x = tf.placeholder(tf.float32, [None, input_size])
 20 | 
 21 |     if corruption > 0.0:
 22 | 
 23 |         # Corrupt data based on random sampling
 24 |         _x = tf.multiply(x, tf.cast(tf.random_uniform(shape=tf.shape(x),
 25 |                                                       minval=0,
 26 |                                                       maxval=1 - corruption,
 27 |                                                       dtype=tf.float32), tf.float32))
 28 | 
 29 |     else:
 30 |         _x = x
 31 | 
 32 |     # Initialize encoder bias
 33 |     b_enc = tf.Variable(tf.zeros([code_size]))
 34 | 
 35 |     # Initialize encoder weights using Glorot method
 36 |     W_enc = tf.Variable(tf.random_uniform(
 37 |                 [input_size, code_size],
 38 |                 -6.0 / math.sqrt(input_size + code_size),
 39 |                 6.0 / math.sqrt(input_size + code_size))
 40 |             )
 41 | 
 42 |     # Compute activation for encoding
 43 |     encode = tf.matmul(_x, W_enc) + b_enc
 44 |     if enc is not None:
 45 |         encode = enc(encode)
 46 | 
 47 |     # Initialize decoder bias
 48 |     b_dec = tf.Variable(tf.zeros([input_size]))
 49 |     if tight:
 50 | 
 51 |         # Tightening using encoder weights
 52 |         W_dec = tf.transpose(W_enc)
 53 | 
 54 |     else:
 55 | 
 56 |         # Initialize decoder weights using Glorot method
 57 |         W_dec = tf.Variable(tf.random_uniform(
 58 |                     [code_size, input_size],
 59 |                     -6.0 / math.sqrt(code_size + input_size),
 60 |                     6.0 / math.sqrt(code_size + input_size))
 61 |                 )
 62 | 
 63 |     # Compute activation for decoding
 64 |     decode = tf.matmul(encode, W_dec) + b_dec
 65 |     if dec is not None:
 66 |         decode = enc(decode)
 67 | 
 68 |     model = {
 69 | 
 70 |         # Input placeholder
 71 |         "input": x,
 72 | 
 73 |         # Encode function
 74 |         "encode": encode,
 75 | 
 76 |         # Decode function
 77 |         "decode": decode,
 78 | 
 79 |         # Cost function: mean squared error
 80 |         "cost": tf.sqrt(tf.reduce_mean(tf.square(x - decode))),
 81 |         # Model parameters
 82 | 
 83 |         "params": {
 84 |             "W_enc": W_enc,
 85 |             "b_enc": b_enc,
 86 |             "b_dec": b_dec,
 87 |         }
 88 | 
 89 |     }
 90 | 
 91 |     # Add weight decoder parameters
 92 |     if not tight:
 93 |         model["params"]["W_dec"] = W_dec
 94 | 
 95 |     return model
 96 | 
 97 | 
 98 | def nn(input_size, n_classes, layers, init=None):
 99 |     """
100 | 
101 |     Multi-layer model
102 |     Supports tight weights and corruption.
103 | 
104 |     """
105 | 
106 |     # Define data input placeholder
107 |     tf.compat.v1.disable_eager_execution()
108 |     input = x = tf.placeholder(tf.float32, [None, input_size])
109 | 
110 |     # Define expected output placeholder
111 |     y = tf.placeholder("float", [None, n_classes])
112 | 
113 |     actvs = []
114 |     dropouts = []
115 |     params = {}
116 |     for i, layer in enumerate(layers):
117 |         #print("layer",i)
118 | 
119 |         # Define dropout placeholder
120 |         
121 |         dropout = tf.placeholder(tf.float32)
122 | 
123 |         if init is None:
124 |             # Initialize empty weights
125 |             W = tf.Variable(tf.zeros([input_size, layer["size"]]))
126 |             b = tf.Variable(tf.zeros([layer["size"]]))
127 |             
128 |         else:
129 |             # Initialize weights with pre-training
130 |             W = tf.Variable(init[i]["W"])
131 |             b = tf.Variable(init[i]["b"])
132 | 
133 |         # Compute layer activation
134 |         x = tf.matmul(x, W) + b
135 | 
136 |         if "actv" in layer and layer["actv"] is not None:
137 |             x = layer["actv"](x)
138 | 
139 |         #print(x.tolist())
140 | 
141 |         # Compute layer dropout
142 |         x = tf.nn.dropout(x, dropout)
143 | 
144 |         # Store parameters
145 |         params.update({
146 |             "W_" + str(i+1): W,
147 |             "b_" + str(i+1): b,
148 |         })
149 |         actvs.append(x)
150 |         dropouts.append(dropout)
151 | 
152 |         input_size = layer["size"]
153 |       
154 |     
155 |     
156 | 
157 |     # Initialize output weights
158 |     W = tf.Variable(tf.random_uniform(
159 |             [input_size, n_classes],
160 |             -3.0 / math.sqrt(input_size + n_classes),
161 |             3.0 / math.sqrt(input_size + n_classes)))
162 |     b = tf.Variable(tf.zeros([n_classes]))
163 | 
164 |     # Compute logits output
165 |     y_hat = tf.matmul(x, W) + b
166 | 
167 |     # Add layer parameters
168 |     params.update({"W_out": W, "b_out": b})
169 |     actvs.append(y_hat)
170 | 
171 |     return {
172 | 
173 |         # Input placeholder
174 |         "input": input,
175 | 
176 |         # Expected output placeholder
177 |         "expected": y,
178 | 
179 |         # NN output function
180 |         "output": tf.nn.softmax(y_hat),
181 | 
182 |         # Cost function: cross-entropy
183 |         "cost": tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_hat, labels=y)),
184 | 
185 |         # Droupout placeholders
186 |         "dropouts": dropouts,#dropouts,
187 | 
188 |         # Layer activations
189 |         "actvs": actvs,
190 | 
191 |         # Model parameters
192 |         "params": params,
193 |     }
194 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | import os
  3 | import re
  4 | import sys
  5 | import h5py
  6 | import time
  7 | import string
  8 | import contextlib
  9 | import multiprocessing
 10 | import pandas as pd
 11 | import numpy as np
 12 | import tensorflow as tf
 13 | from model import ae
 14 | from tensorflow.python.framework import ops
 15 | import tensorflow.compat.v1 as tf
 16 | 
 17 | identifier = '(([a-zA-Z]_)?([a-zA-Z0-9_]*))'
 18 | replacement_field = '{' + identifier + '}'
 19 | 
 20 | 
 21 | def reset():
 22 |     #tf.reset_default_graph()
 23 |     ops.reset_default_graph()
 24 |     #tf.compat.v1.reset_default_graph()
 25 |     #random.seed(19)
 26 |     #np.random.seed(19)
 27 |     tf.set_random_seed(19)
 28 |     #tf.random.set_seed(19)
 29 | 
 30 | 
 31 | def load_phenotypes(pheno_path):
 32 |     pheno = pd.read_csv(pheno_path)
 33 |     
 34 |     pheno = pheno[pheno['FILE_ID'] != 'no_filename']
 35 |     #pheno = pheno[pheno['FILE_ID'].str.contains("NYU")]
 36 |     pheno['DX_GROUP'] = pheno['DX_GROUP'].apply(lambda v: int(v)-1)
 37 |     pheno['SITE_ID'] = pheno['SITE_ID'].apply(lambda v: re.sub('_[0-9]', '', v))
 38 |     #pheno['SEX'] = pheno['SEX'].apply(lambda v: {1: "M", 2: "F"}[v])
 39 |     pheno['SEX'] = pheno['SEX']
 40 |     pheno['MEAN_FD'] = pheno['func_mean_fd']
 41 |     pheno['SUB_IN_SMP'] = pheno['SUB_IN_SMP'].apply(lambda v: v == 1)
 42 |     pheno["STRAT"] = pheno[["SITE_ID", "DX_GROUP"]].apply(lambda x: "_".join([str(s) for s in x]), axis=1)
 43 |     pheno["AGE"] = pheno['AGE_AT_SCAN']
 44 | 
 45 |     pheno.index = pheno['FILE_ID']
 46 | 
 47 |     return pheno[['FILE_ID', 'DX_GROUP', 'SEX', 'SITE_ID', 'MEAN_FD', 'SUB_IN_SMP', 'STRAT','AGE']]
 48 | 
 49 | 
 50 | def load_phenotypes_2(pheno_path):
 51 | 
 52 |     pheno = pd.read_csv(pheno_path)
 53 |     
 54 |     pheno = pheno[pheno['FILE_ID'] != 'no_filename']
 55 |     #pheno = pheno[pheno['FILE_ID'].str.contains("NYU")]
 56 |     pheno['DX_GROUP'] = pheno['DX_GROUP'].apply(lambda v: int(v)-1)
 57 |     pheno['SITE_ID'] = pheno['SITE_ID'].apply(lambda v: re.sub('_[0-9]', '', v))
 58 |     pheno['SEX'] = pheno['SEX'].apply(lambda v: {1: "M", 2: "F"}[v])
 59 |     pheno['SEX'] = pheno['SEX']
 60 |     pheno['MEAN_FD'] = pheno['func_mean_fd']
 61 |     pheno['SUB_IN_SMP'] = pheno['SUB_IN_SMP'].apply(lambda v: v == 1)
 62 |     pheno["STRAT"] = pheno[["SITE_ID", "DX_GROUP"]].apply(lambda x: "_".join([str(s) for s in x]), axis=1)
 63 |     pheno["AGE"] = pheno['AGE_AT_SCAN']
 64 | 
 65 |     #pheno["FIQ"] = pheno['FIQ'].apply(lambda v: {-9999: '100'}[v])
 66 |     pheno["FIQ"] = pheno['FIQ'].fillna(pheno['FIQ'].mean())
 67 | 
 68 |     pheno['HANDEDNESS_SCORES'] = pheno['HANDEDNESS_SCORES'].fillna(method='bfill')
 69 |     pheno.index = pheno['FILE_ID']
 70 | 
 71 |     return pheno[['FILE_ID', 'DX_GROUP', 'SEX', 'SITE_ID', 'MEAN_FD', 'SUB_IN_SMP', 'STRAT','AGE','HANDEDNESS_SCORES','FIQ']]
 72 | 
 73 | 
 74 | def hdf5_handler(filename, mode="r"):
 75 |     h5py.File(filename, "a").close()
 76 |     propfaid = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
 77 |     settings = list(propfaid.get_cache())
 78 |     settings[1] = 0
 79 |     settings[2] = 0
 80 |     propfaid.set_cache(*settings)
 81 |     with contextlib.closing(h5py.h5f.open(filename, fapl=propfaid)) as fid:
 82 |         f = h5py.File(fid, mode)
 83 |         # f.attrs.create(dtype=h5py.special_dtype(vlen=str)) 
 84 |         return f
 85 | 
 86 | def load_fold(patients, experiment, fold):
 87 | 
 88 |     derivative = experiment.attrs["derivative"]
 89 | 
 90 |     pheno_path = "./data/phenotypes/Phenotypic_V1_0b_preprocessed1.csv"
 91 |     pheno = load_phenotypes(pheno_path)
 92 |     #print(pheno['FILE_ID'])
 93 | 
 94 |     X_train = []
 95 |     y_train = []
 96 |    
 97 |     for pid in experiment[fold]["train"]:
 98 |         
 99 |         p=pheno[pheno['FILE_ID']==pid.decode(encoding = 'UTF-8')]
100 |         x=np.array(patients[pid][derivative])
101 |         x=np.append(x,int(p['SEX'].values))
102 |         x=np.append(x,float(p['AGE'].values))
103 |         # print(p['FILE_ID'])
104 |         # print(p['SEX'].values)
105 |         # print(p['AGE'].values)
106 |         # print(x.shape)
107 |         # X_train.append(np.array(patients[pid][derivative]))
108 |         #print(len(patients[pid][derivative]))
109 |         X_train.append(x)
110 |         y_train.append(patients[pid].attrs["y"])
111 | 
112 |     X_valid = []
113 |     y_valid = []
114 |     for pid in experiment[fold]["valid"]:
115 | 
116 |         p=pheno[pheno['FILE_ID']==pid.decode(encoding = 'UTF-8')]
117 |         x=np.array(patients[pid][derivative])
118 |         x=np.append(x,int(p['SEX'].values))
119 |         x=np.append(x,float(p['AGE'].values))
120 | 
121 |         X_valid.append(x)
122 |         y_valid.append(patients[pid].attrs["y"])
123 | 
124 |     X_test = []
125 |     y_test = []
126 |     for pid in experiment[fold]["test"]:
127 | 
128 |         p=pheno[pheno['FILE_ID']==pid.decode(encoding = 'UTF-8')]
129 |         x=np.array(patients[pid][derivative])
130 |         x=np.append(x,int(p['SEX'].values))
131 |         x=np.append(x,float(p['AGE'].values))
132 | 
133 |         X_test.append(x)
134 |         y_test.append(patients[pid].attrs["y"])
135 | 
136 | 
137 | 
138 |     return np.array(X_train), y_train, \
139 |            np.array(X_valid), y_valid, \
140 |            np.array(X_test), y_test
141 | 
142 | 
143 | class SafeFormat(dict):
144 | 
145 |     def __missing__(self, key):
146 |         return "{" + key + "}"
147 | 
148 |     def __getitem__(self, key):
149 |         if key not in self:
150 |             return self.__missing__(key)
151 |         return dict.__getitem__(self, key)
152 | 
153 | 
154 | def merge_dicts(*dict_args):
155 |     result = {}
156 |     for dictionary in dict_args:
157 |         result.update(dictionary)
158 |     return result
159 | 
160 | 
161 | def format_config(s, *d):
162 |     dd = merge_dicts(*d)
163 |     return string.Formatter().vformat(s, [], SafeFormat(dd))
164 | 
165 | 
166 | def elapsed_time(tstart):
167 |     tnow = time.time()
168 |     total = tnow - tstart
169 |     m, s = divmod(total, 60)
170 |     h, m = divmod(m, 60)
171 |     return "%d:%02d:%02d" % (h, m, s)
172 | 
173 | 
174 | def run_progress(callable_func, items, message=None, jobs=1):
175 | 
176 |     results = []
177 | 
178 |     print ('Starting pool of %d jobs' % jobs)
179 | 
180 |     current = 0
181 |     total = len(items)
182 | 
183 |     if jobs == 1:
184 |         results = []
185 |         for item in items:
186 |             results.append(callable_func(item))
187 |             current = len(results)
188 |             if message is not None:
189 |                 args = {'current': current, 'total': total}
190 |                 sys.stdout.write("\r" + message.format(**args))
191 |                 sys.stdout.flush()
192 | 
193 |     # Or allocate a pool for multithreading
194 |     else:
195 |         pool = multiprocessing.Pool(processes=jobs)
196 |         for item in items:
197 |             pool.apply_async(callable_func, args=(item,), callback=results.append)
198 | 
199 |         while current < total:
200 |             current = len(results)
201 |             if message is not None:
202 |                 args = {'current': current, 'total': total}
203 |                 sys.stdout.write("\r" + message.format(**args))
204 |                 sys.stdout.flush()
205 |             time.sleep(0.5)
206 | 
207 |         pool.close()
208 |         pool.join()
209 | 
210 |     return results
211 | 
212 | 
213 | def root():
214 |     return os.path.dirname(os.path.realpath(__file__))
215 | 
216 | 
217 | def to_softmax(n_classes, classe):
218 |     sm = [0.0] * n_classes
219 |     sm[int(classe)] = 1.0
220 |     return sm
221 | 
222 | 
223 | def load_ae_encoder(input_size, code_size, model_path):
224 |     model = ae(input_size, code_size)
225 |     init = tf.global_variables_initializer()
226 |     try:
227 |         with tf.Session() as sess:
228 |             sess.run(init)
229 |             saver = tf.train.Saver(model["params"], write_version= tf.train.SaverDef.V2)
230 |             if os.path.isfile(model_path):
231 |                 print ("Restoring", model_path)
232 |                 saver.restore(sess, model_path)
233 |             params = sess.run(model["params"])
234 |             return {"W_enc": params["W_enc"], "b_enc": params["b_enc"]}
235 |     finally:
236 |         reset()
237 | 
238 | 
239 | def sparsity_penalty(x, p, coeff):
240 |     p_hat = tf.reduce_mean(tf.abs(x), 0)
241 |     kl = p * tf.log(p / p_hat) + \
242 |         (1 - p) * tf.log((1 - p) / (1 - p_hat))
243 |     return coeff * tf.reduce_sum(kl)
244 | 


--------------------------------------------------------------------------------
/nn_evaluate.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | """
  5 | 
  6 | Autoencoders evaluation.
  7 | 
  8 | Usage:
  9 |   nn_evaluate.py [--whole] [--male] [--threshold] [--leave-site-out] [--NYU-site-out] [<derivative> ...]
 10 |   nn_evaluate.py (-h | --help)
 11 | 
 12 | Options:
 13 |   -h --help           Show this screen
 14 |   --whole             Run model for the whole dataset
 15 |   --male              Run model for male subjects
 16 |   --threshold         Run model for thresholded subjects
 17 |   --leave-site-out    Prepare data using leave-site-out method
 18 |   derivative          Derivatives to process
 19 | 
 20 | """
 21 | import numpy as np
 22 | import pandas as pd
 23 | #import tensorflow as tf
 24 | from docopt import docopt
 25 | from nn import nn
 26 | from utils import (load_phenotypes, format_config, hdf5_handler,
 27 |                    reset, to_softmax, load_ae_encoder, load_fold)
 28 | from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_auc_score
 29 | 
 30 | import tensorflow.compat.v1 as tf
 31 | from sklearn.feature_selection import SelectKBest,f_classif
 32 | 
 33 | 
 34 | def nn_results(hdf5, experiment, code_size_1, code_size_2):
 35 | 
 36 |     import pdb; pdb.set_trace()
 37 |     exp_storage = hdf5["experiments"][experiment]
 38 | 
 39 |     n_classes = 2
 40 | 
 41 |     results = []
 42 | 
 43 |     count=0
 44 |     for fold in exp_storage:
 45 |         if count==1:
 46 |           break
 47 |         count+=1
 48 |         experiment_cv = format_config("{experiment}_{fold}", {
 49 |             "experiment": experiment,
 50 |             "fold": fold,
 51 |         })
 52 | 
 53 |         
 54 |         X_train, y_train, \
 55 |         X_valid, y_valid, \
 56 |         X_test, y_test = load_fold(hdf5["patients"], exp_storage, fold)
 57 | 
 58 | 
 59 |         X_all=np.vstack((X_train,X_valid,X_test))
 60 |         y_all=np.concatenate((np.array(y_train),np.array(y_valid),np.array(y_test)),axis=0)
 61 | 
 62 |         # print(X_all.shape)
 63 |         # print(y_all.shape)
 64 | 
 65 |         ks=0
 66 |         if X_train.shape[1]<10000:
 67 |           ks=1000
 68 |         else:
 69 |           ks=3000    
 70 |         X_new=SelectKBest(f_classif,k=ks).fit_transform(X_all[:,:-2], y_all)
 71 |         # print(X_new.shape)
 72 | 
 73 |         train=X_train.shape[0]
 74 |         valid=X_valid.shape[0]
 75 |         test=X_test.shape[0]
 76 | 
 77 |         X_new=np.concatenate((X_new,X_all[:,-2:]),axis=1)
 78 | 
 79 |         X_train=X_new[:train]
 80 |         X_valid=X_new[train:train+valid]
 81 |         X_test=X_new[train+valid:train+valid+test]
 82 | 
 83 |         # print(X_new.shape)
 84 | 
 85 |         #y_test = np.array([to_softmax(n_classes, y) for y in y_test])
 86 | 
 87 | 
 88 |         #X_test=X_new[:]
 89 |         y_test = np.array([to_softmax(n_classes, y) for y in y_all])
 90 | 
 91 |         '''
 92 |         X_NYU=np.load('X_NYU.npy')
 93 |         y_NYU=np.load('y_NYU.npy')
 94 | 
 95 |         X_sub=np.load('/content/drive/MyDrive/acerta-abide/X_sub_without_NYU.npy')
 96 |         y_sub=np.load('/content/drive/MyDrive/acerta-abide/y_sub_without_NYU.npy')
 97 |         
 98 |         X=np.vstack((X_NYU,X_sub))
 99 |         y=np.concatenate((y_NYU,y_sub))
100 | 
101 |         ks=0
102 |         if X.shape[1]<10000:
103 |           ks=1000
104 |         else:
105 |           ks=3000
106 |         X_new=SelectKBest(f_classif,k=ks).fit_transform(X, y)
107 |         print(X_new.shape)
108 | 
109 |         
110 |         X_train,X_test,y_train,y_test = train_test_split(X_new,y,test_size=0.1,random_state=0)
111 |         X_train,X_valid,y_train,y_valid = train_test_split(X_train,y_train,test_size=0.33,random_state=0)
112 |         '''
113 | 
114 | 
115 |         ae1_model_path = format_config("./data/models/{experiment}_autoencoder-1.ckpt", {
116 |             "experiment": experiment_cv,
117 |         })
118 |         ae2_model_path = format_config("./data/models/{experiment}_autoencoder-2.ckpt", {
119 |             "experiment": experiment_cv,
120 |         })
121 |         nn_model_path = format_config("./data/models/{experiment}_mlp.ckpt", {
122 |             "experiment": experiment_cv,
123 |         })
124 | 
125 |         try:
126 | 
127 |             model = nn(X_test.shape[1], n_classes, [
128 |                 {"size": 1000, "actv": tf.nn.tanh},
129 |                 {"size": 600, "actv": tf.nn.tanh},
130 |                 {"size": 100, "actv": tf.nn.tanh},
131 | 
132 |             ])
133 | 
134 |             init = tf.compat.v1.global_variables_initializer()
135 |             #init = tf.global_variables_initializer()
136 |             with tf.Session() as sess:
137 | 
138 |                 sess.run(init)
139 | 
140 |                 saver = tf.train.Saver(model["params"])
141 |                 saver.restore(sess, nn_model_path)
142 | 
143 |                 output = sess.run(
144 |                     model["output"],
145 |                     feed_dict={
146 |                         model["input"]: X_new,
147 |                         model["dropouts"][0]: 1.0,
148 |                         model["dropouts"][1]: 1.0,
149 |                         model["dropouts"][2]: 1.0,
150 |                     }
151 |                 )
152 | 
153 |                 y_pred = np.argmax(output, axis=1)
154 |                 y_true = np.argmax(y_test, axis=1)
155 | 
156 |                 # print(y_true.shape)
157 |                 # print(y_pred.shape)
158 |                 # print(y_pred)
159 |                 # print(y_true)
160 | 
161 |                 # print(sum(np.equal(y_pred,y_true)))
162 | 
163 |                 X_sub=[]
164 |                 y_sub=[]
165 | 
166 |                 for i in range(X_new.shape[0]):
167 |                   if y_pred[i]==y_true[i]:
168 |                     X_sub.append(X_all[i])
169 |                     y_sub.append(y_pred[i])
170 | 
171 |                 X_sub=np.array(X_sub)
172 |                 # print(X_sub.shape)
173 |                 y_sub=np.array(y_sub)
174 | 
175 |                 # print(X_sub.shape)
176 | 
177 |                 np.save('X_sub_without_NYU.npy',X_sub)
178 |                 np.save('y_sub_without_NYU.npy',y_sub)
179 | 
180 |                 [[TN, FP], [FN, TP]] = confusion_matrix(y_true, y_pred, labels=[0, 1]).astype(float)
181 |                 specificity = TN/(FP+TN)
182 |                 precision = TP/(TP+FP)
183 |                 sensivity = TP/(TP+FN)
184 | 
185 |                 accuracy = accuracy_score(y_true, y_pred)
186 |                 fscore = f1_score(y_true, y_pred)
187 |                 roc_auc = roc_auc_score(y_true, y_pred)
188 | 
189 |                 results.append([accuracy, precision, fscore, sensivity, specificity, roc_auc])
190 |         finally:
191 |             reset()
192 | 
193 |     return [experiment] + np.mean(results, axis=0).tolist()
194 | 
195 | if __name__ == "__main__":
196 | 
197 |     reset()
198 | 
199 |     arguments = docopt(__doc__)
200 | 
201 |     pd.set_option("display.expand_frame_repr", False)
202 | 
203 |     pheno_path = "./data/phenotypes/Phenotypic_V1_0b_preprocessed1.csv"
204 |     pheno = load_phenotypes(pheno_path)
205 | 
206 |     hdf5 = hdf5_handler(bytes("./data/abide.hdf5",encoding="utf8"), "a")
207 | 
208 |     valid_derivatives = ["cc200", "aal", "ez", "ho", "tt", "dosenbach160"]
209 |     derivatives = [derivative for derivative
210 |                    in arguments["<derivative>"]
211 |                    if derivative in valid_derivatives]
212 | 
213 |     experiments = []
214 | 
215 |     for derivative in derivatives:
216 | 
217 |         config = {"derivative": derivative}
218 | 
219 |         if arguments["--whole"]:
220 |             experiments += format_config("{derivative}_whole", config), # removed []
221 | 
222 |         if arguments["--male"]:
223 |             experiments += [format_config("{derivative}_male", config)]
224 | 
225 |         if arguments["--threshold"]:
226 |             experiments += [format_config("{derivative}_threshold", config)]
227 | 
228 |         if arguments["--leave-site-out"]:
229 |             for site in pheno["SITE_ID"].unique():
230 |                 if site=='NYU':
231 |                   site_config = {"site": site}
232 |                   experiments += [
233 |                       format_config("{derivative}_leavesiteout-{site}",
234 |                                     config, site_config)
235 |                   ]
236 | 
237 |         if arguments["--NYU-site-out"]:
238 |             experiments += [format_config("{derivative}_leavesiteout-NYU", config)]
239 | 
240 | 
241 |     # First autoencoder bottleneck
242 |     code_size_1 = 1000 # not used
243 | 
244 |     # Second autoencoder bottleneck
245 |     code_size_2 = 600 # not used
246 | 
247 |     results = []
248 | 
249 |     experiments = sorted(experiments)
250 |     print(experiments)
251 |     for experiment in experiments:
252 |         print(experiment)
253 |         results.append(nn_results(hdf5, experiment, code_size_1, code_size_2))
254 | 
255 |     cols = ["Exp", "Accuracy", "Precision", "F1-score", "Sensivity", "Specificity", "ROC-AUC"]
256 |     df = pd.DataFrame(results, columns=cols)
257 | 
258 |     print('aaa',df[cols] \
259 |         .sort_values(["Exp"]) \
260 |         .reset_index())
261 | 


--------------------------------------------------------------------------------
/prepare_data.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | """
  5 | 
  6 | Data preparation
  7 | Usage:
  8 |   prepare_data.py [--folds=N] [--whole] [--male] [--threshold] [--leave-site-out] [--NYU-site-out] [<derivative> ...]
  9 |   prepare_data.py (-h | --help)
 10 | 
 11 | Options:
 12 |   -h --help           Show this screen
 13 |   --folds=N           Number of folds [default: 10]
 14 |   --whole             Prepare data of the whole dataset
 15 |   --male              Prepare data of male subjects
 16 |   --threshold         Prepare data of thresholded subjects
 17 |   --leave-site-out    Prepare data using leave-site-out method
 18 |   derivative          Derivatives to process
 19 | 
 20 | """
 21 | import numpy as np
 22 | import pandas as pd
 23 | 
 24 | import os
 25 | import random
 26 | import pandas as pd
 27 | import numpy as np
 28 | import numpy.ma as ma
 29 | from docopt import docopt
 30 | from functools import partial
 31 | from sklearn import preprocessing
 32 | from sklearn.model_selection import StratifiedKFold, train_test_split
 33 | from utils import (load_phenotypes, format_config, run_progress, hdf5_handler)
 34 | 
 35 | 
 36 | def compute_connectivity(functional):
 37 |     with np.errstate(invalid="ignore"):
 38 |       # granger_matrix = np.zeros((200, 200))
 39 |       # data=functional
 40 |       # for i in range(200):
 41 |       #   for j in range(200):
 42 |       #     if i != j:
 43 |       #         # 将第i和第j列的时间序列作为输入数据
 44 |       #         x = pd.DataFrame({'x': data[:,i], 'y': data[:,j]})
 45 |       #         # 计算格兰杰因果检验，maxlag=1表示只考虑滞后一步
 46 |       #         granger_test = sm.tsa.stattools.grangercausalitytests(x, maxlag=3, verbose=False)
 47 |       #         # 提取格兰杰因果检验结果的p值
 48 |       #         p_value = granger_test[1][0]['ssr_ftest'][1]
 49 |       #         # 将p值存储到格兰杰因果检验矩阵中
 50 |       #         granger_matrix[i, j] = p_value
 51 |       #   print(granger_matrix.shape)
 52 |        
 53 |       #   granger_matrix=np.nan_to_num(granger_matrix)
 54 |         corr = np.nan_to_num(np.corrcoef(functional))
 55 |         mask = np.invert(np.tri(corr.shape[0], k=-1, dtype=bool))
 56 |         m = ma.masked_where(mask == 1, mask)
 57 |         return ma.masked_where(m, corr).compressed()
 58 |         
 59 |         #return granger_matrix.flatten()
 60 | 
 61 | 
 62 | def load_patient(subj, tmpl):
 63 |     df = pd.read_csv(format_config(tmpl, {
 64 |         "subject": subj,
 65 |     }), sep="\t", header=0)
 66 |     
 67 |     df = df.apply(lambda x: pd.to_numeric(x, errors='coerce'))
 68 | 
 69 |     print(format_config(tmpl, {
 70 |          "subject": subj,
 71 |      }))
 72 |     # print(df.shape)
 73 | 
 74 |     ROIs = ["#" + str(y) for y in sorted([int(x[1:]) for x in df.keys().tolist()])]
 75 |     print(df.keys())
 76 | 
 77 |     functional = np.nan_to_num(df[ROIs].to_numpy().T).tolist()
 78 |     functional = preprocessing.scale(functional, axis=1)
 79 |     #print(len(functional),len(functional[0]))
 80 |     functional = compute_connectivity(functional)
 81 |     #print(np.count_nonzero(np.array(functional)))
 82 |     functional = functional.astype(np.float32)
 83 | 
 84 |     # T=90
 85 |     # sampleList=random.sample(range(0,len(functional)-1-T),10)
 86 |     # print(sampleList)
 87 |     # samples=[]
 88 |     # for i in sampleList:
 89 |     #   sample=functional[i:i+T]
 90 |     #   samples.append(sample)
 91 |     # print(np.array(samples).shape)
 92 | 
 93 |     return subj,functional
 94 | 
 95 | 
 96 | def load_patients(subjs, tmpl, jobs=1):
 97 |     partial_load_patient = partial(load_patient, tmpl=tmpl)
 98 |     msg = "Processing {current} of {total}"
 99 |     return dict(run_progress(partial_load_patient, subjs, message=msg, jobs=jobs))
100 | 
101 | 
102 | def prepare_folds(hdf5, folds, pheno, derivatives, experiment):
103 | 
104 |     exps = hdf5.require_group("experiments")
105 |     ids = pheno["FILE_ID"]
106 |     #print(hdf5["experiments"]["cc200_whole"]["0"])
107 |     for derivative in derivatives:
108 |         exp = exps.require_group(format_config(
109 |             experiment,
110 |             {
111 |                 "derivative": derivative,
112 |             }
113 |         ))
114 |         exp.attrs["derivative"] = derivative
115 | 
116 |         skf = StratifiedKFold(n_splits=folds, shuffle=False)
117 |         for i, (train_index, test_index) in enumerate(skf.split(ids, pheno["STRAT"])):
118 |         
119 |             train_index, valid_index = train_test_split(train_index, test_size=0.33)
120 | 
121 |             fold = exp.require_group(str(i))
122 |             
123 |             fold['train'] = [ind.encode('utf8') for ind in ids[train_index]] 
124 |             fold['valid'] = [indv.encode('utf8') for indv in ids[valid_index]]
125 |             fold["test"] = [indt.encode('utf8') for indt in ids[test_index]]
126 | 
127 | 
128 |         # print(fold['train'].shape)
129 |         # for fa in fold.attrs:
130 |         #   print(fa)
131 | 
132 |             # fold["train"] = ids[train_index].tolist()
133 |             # fold["valid"] = ids[valid_index].tolist()
134 |             # fold["test"] = ids[test_index].tolist()
135 | 
136 | 
137 | def load_patients_to_file(hdf5, pheno, derivatives):
138 | 
139 |     download_root = "./data/functionals"
140 |     derivatives_path = {
141 |         "aal": "cpac/filt_global/rois_aal/{subject}_rois_aal.1D",
142 |         "cc200": "cpac/filt_global/rois_cc200/{subject}_rois_cc200.1D",
143 |         "dosenbach160": "cpac/filt_global/rois_dosenbach160/{subject}_rois_dosenbach160.1D",
144 |         "ez": "cpac/filt_global/rois_ez/{subject}_rois_ez.1D",
145 |         "ho": "cpac/filt_global/rois_ho/{subject}_rois_ho.1D",
146 |         "tt": "cpac/filt_global/rois_tt/{subject}_rois_tt.1D",
147 |     }
148 |     #print('storing_patients')
149 |     storage = hdf5.require_group("patients")
150 | 
151 |     #temp=[x for x in pheno["FILE_ID"].values if x.contains('NYU')==True]
152 |     #print(pheno[pheno["FILE_ID"].str.contains('NYU')]["FILE_ID"])
153 | 
154 |     #file_ids = pheno[pheno["FILE_ID"].str.contains('NYU')]["FILE_ID"].tolist()
155 |     #print('storing_finished')
156 |     file_ids = pheno["FILE_ID"].tolist()
157 | 
158 |     for derivative in derivatives:
159 | 
160 |         #print('derivative_loop')
161 |         file_template = os.path.join(download_root, derivatives_path[derivative])
162 |         print(file_template)
163 |         # print('one_over')
164 |         func_data = load_patients(file_ids, tmpl=file_template)
165 |         #print('two_over')
166 | 
167 | 
168 |         for pid in func_data:
169 |             print('func_data_filling')
170 |             record = pheno[pheno["FILE_ID"] == pid].iloc[0]
171 |             patient_storage = storage.require_group(pid)
172 |             patient_storage.attrs["id"] = record["FILE_ID"]
173 |             patient_storage.attrs["y"] = record["DX_GROUP"]
174 |             patient_storage.attrs["site"] = record["SITE_ID"]
175 |             patient_storage.attrs["sex"] = record["SEX"]
176 |             # path=os.path.join("/content/drive/MyDrive/dataset_nx200",str(record["DX_GROUP"]))
177 |             # # if not os.path.exists(path):
178 |             # #  os.mkdir(path)
179 |             # print(func_data[pid])
180 |             # np.save(path+"/"+record["FILE_ID"]+".npy",func_data[pid])
181 |             # #cv2.imwrite(path+"/"+record["FILE_ID"]+".jpeg",func_data[pid])
182 |             patient_storage.create_dataset(derivative, data=func_data[pid])
183 |   
184 | 
185 | if __name__ == "__main__":
186 | 
187 |     random.seed(19)
188 |     np.random.seed(19)
189 | 
190 |     arguments = docopt(__doc__)
191 | 
192 |     folds = int(arguments["--folds"])
193 |     pheno_path = "./data/phenotypes/Phenotypic_V1_0b_preprocessed1.csv"
194 |     pheno = load_phenotypes(pheno_path)
195 | 
196 |     hdf5 = hdf5_handler(bytes("./data/abide.hdf5", encoding="utf8"), 'a')
197 | 
198 |     valid_derivatives = ["cc200", "aal", "ez", "ho", "tt", "dosenbach160"]
199 |     derivatives = [derivative for derivative in arguments["<derivative>"] if derivative in valid_derivatives]
200 | 
201 |     load_patients_to_file(hdf5, pheno, derivatives)
202 |     #os._exit()
203 | 
204 |     if "patients" not in hdf5:
205 |         load_patients_to_file(hdf5, pheno, derivatives)
206 | 
207 |     if arguments["--whole"]:
208 |         
209 |         print ("Preparing whole dataset")
210 |         prepare_folds(hdf5, folds, pheno, derivatives, experiment="{derivative}_whole")
211 | 
212 |     if arguments["--male"]:
213 |         
214 |         print ("Preparing male dataset")
215 |         pheno_male = pheno[pheno["SEX"] == "M"]
216 |         prepare_folds(hdf5, folds, pheno_male, derivatives, experiment="{derivative}_male")
217 | 
218 |     if arguments["--threshold"]:
219 |         
220 |         print ("Preparing thresholded dataset")
221 |         pheno_thresh = pheno[pheno["MEAN_FD"] <= 0.2]
222 |         prepare_folds(hdf5, folds, pheno_thresh, derivatives, experiment="{derivative}_threshold")
223 | 
224 |     if arguments["--leave-site-out"]:
225 |         
226 |         print ("Preparing leave-site-out dataset")
227 |         for site in pheno["SITE_ID"].unique():
228 |             if site=='NYU':
229 |               pheno_without_site = pheno[pheno["SITE_ID"] == site]
230 |               prepare_folds(hdf5, folds, pheno_without_site, derivatives, experiment=format_config(
231 |                 "{derivative}_leavesiteout-{site}",
232 |                 {
233 |                     "site": site,
234 |                 })
235 |               )
236 | 
237 |     if arguments["--NYU-site-out"]:
238 |         
239 |         print ("Preparing leave-NYU-out dataset")
240 | 
241 |         pheno_without_site = pheno[pheno["SITE_ID"] != 'NYU']
242 |         prepare_folds(hdf5, folds, pheno_without_site, derivatives,experiment="{derivative}_leavesiteout-NYU")
243 |         


--------------------------------------------------------------------------------
/Experiments_Colab_From_Xuehan.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |   "cells": [
  3 |     {
  4 |       "cell_type": "markdown",
  5 |       "metadata": {
  6 |         "id": "view-in-github",
  7 |         "colab_type": "text"
  8 |       },
  9 |       "source": [
 10 |         "<a href=\"https://colab.research.google.com/github/lxhwww/Autism-Spectrum-Disorder-Diagnosis-with-Ensemble-DNN/blob/main/Experiments.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
 11 |       ]
 12 |     },
 13 |     {
 14 |       "cell_type": "code",
 15 |       "execution_count": null,
 16 |       "metadata": {
 17 |         "id": "_QnM9Qqa2cM9"
 18 |       },
 19 |       "outputs": [],
 20 |       "source": [
 21 |         "from google.colab import drive\n",
 22 |         "drive.mount('/content/drive')"
 23 |       ]
 24 |     },
 25 |     {
 26 |       "cell_type": "code",
 27 |       "source": [
 28 |         "import os\n",
 29 |         "os.chdir(\"/content/drive/MyDrive/acerta-abide\")"
 30 |       ],
 31 |       "metadata": {
 32 |         "id": "O5hX0Nzgf7fq"
 33 |       },
 34 |       "execution_count": 2,
 35 |       "outputs": []
 36 |     },
 37 |     {
 38 |       "cell_type": "code",
 39 |       "execution_count": null,
 40 |       "metadata": {
 41 |         "id": "3VSco6ae6y1E"
 42 |       },
 43 |       "outputs": [],
 44 |       "source": [
 45 |         "!sudo python3 -m pip install aiohttp\n",
 46 |         "!pip3 install --upgrade setuptools\n",
 47 |         "!pip install -r /content/drive/MyDrive/acerta-abide/requirements.txt\n",
 48 |         "!pip install docopt "
 49 |       ]
 50 |     },
 51 |     {
 52 |       "cell_type": "code",
 53 |       "execution_count": null,
 54 |       "metadata": {
 55 |         "id": "Kz6mBsIgnX3Y"
 56 |       },
 57 |       "outputs": [],
 58 |       "source": [
 59 |         "#use for data download\n",
 60 |         "#download_abide.py [--pipeline=cpac] [--strategy=filt_global] [<derivative> ...]\n",
 61 |         "!python /content/drive/MyDrive/acerta-abide/download_abide.py"
 62 |       ]
 63 |     },
 64 |     {
 65 |       "cell_type": "code",
 66 |       "execution_count": null,
 67 |       "metadata": {
 68 |         "id": "KeNN_hhVhCwd"
 69 |       },
 70 |       "outputs": [],
 71 |       "source": [
 72 |         "os.chdir(\"/content/drive/MyDrive/acerta-abide\")"
 73 |       ]
 74 |     },
 75 |     {
 76 |       "cell_type": "code",
 77 |       "source": [
 78 |         "#show the demographic Information of ABIDE I\n",
 79 |         "!python pheno_info.py"
 80 |       ],
 81 |       "metadata": {
 82 |         "id": "cWn7u4DDr-pe",
 83 |         "colab": {
 84 |           "base_uri": "https://localhost:8080/"
 85 |         },
 86 |         "outputId": "b2679738-2cb1-4922-e140-3e59d648cc3d"
 87 |       },
 88 |       "execution_count": 7,
 89 |       "outputs": [
 90 |         {
 91 |           "output_type": "stream",
 92 |           "name": "stdout",
 93 |           "text": [
 94 |             "2023-06-06 18:06:36.568086: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
 95 |             "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
 96 |             "2023-06-06 18:06:37.816772: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
 97 |             " CALTECH &        27.4 (    10.3) &   -100.0 (     0.0) &   -423.8 (  2318.8)   &  M  15, F   4 &        28.0 (    10.9) &  M  14, F   4 \\\\\n",
 98 |             "     CMU &        26.4 (     5.8) &    100.0 (     0.0) &    114.5 (    11.6)   &  M  11, F   3 &        26.8 (     5.7) &  M  10, F   3 \\\\\n",
 99 |             "     KKI &        10.0 (     1.4) &     65.3 (    52.5) &     97.9 (    17.5)   &  M  16, F   4 &        10.0 (     1.2) &  M  20, F   8 \\\\\n",
100 |             "  LEUVEN &        17.8 (     5.0) &    100.0 (     0.0) &    -74.0 (   180.5)   &  M  26, F   3 &        18.2 (     5.1) &  M  29, F   5 \\\\\n",
101 |             " MAX_MUN &        26.1 (    14.9) &   -100.0 (     0.0) &   -732.5 (  2854.1)   &  M  21, F   3 &        24.6 (     8.8) &  M  27, F   1 \\\\\n",
102 |             "     NYU &        14.7 (     7.1) &    -93.7 (  1160.3) &    107.1 (    16.4)   &  M  65, F  10 &        15.7 (     6.2) &  M  74, F  26 \\\\\n",
103 |             "    OHSU &        11.4 (     2.2) &    -60.0 (     0.0) &   -736.1 (  2917.1)   &  M  12, F   0 &        10.1 (     1.1) &  M  14, F   0 \\\\\n",
104 |             "    OLIN &        16.5 (     3.4) &    -60.0 (     0.0) &   -951.8 (  3188.2)   &  M  16, F   3 &        16.7 (     3.6) &  M  13, F   2 \\\\\n",
105 |             "    PITT &        19.0 (     7.3) &    -60.0 (     0.0) &    110.2 (    14.6)   &  M  25, F   4 &        18.9 (     6.6) &  M  23, F   4 \\\\\n",
106 |             "     SBL &        35.0 (    10.4) &   -617.3 (  2596.3) &  -6629.6 (  4932.3)   &  M  15, F   0 &        33.7 (     6.6) &  M  15, F   0 \\\\\n",
107 |             "    SDSU &        14.7 (     1.8) &    -60.0 (     0.0) &    111.4 (    18.0)   &  M  13, F   1 &        14.2 (     1.9) &  M  16, F   6 \\\\\n",
108 |             "STANFORD &        10.0 (     1.6) &   -100.0 (     0.0) &    110.7 (    16.1)   &  M  15, F   4 &        10.0 (     1.6) &  M  16, F   4 \\\\\n",
109 |             " TRINITY &        16.8 (     3.2) &    -60.0 (     0.0) &    108.9 (    15.5)   &  M  22, F   0 &        17.1 (     3.8) &  M  25, F   0 \\\\\n",
110 |             "    UCLA &        13.0 (     2.5) &   -100.0 (     0.0) &    100.4 (    13.5)   &  M  48, F   6 &        13.0 (     1.9) &  M  38, F   6 \\\\\n",
111 |             "      UM &        13.2 (     2.4) &    -60.0 (     0.0) &    -47.6 (  1243.9)   &  M  57, F   9 &        14.8 (     3.6) &  M  56, F  18 \\\\\n",
112 |             "     USM &        23.5 (     8.3) &     67.5 (    46.1) &     99.7 (    16.6)   &  M  46, F   0 &        21.3 (     8.4) &  M  25, F   0 \\\\\n",
113 |             "    YALE &        12.7 (     3.0) &    100.0 (     0.0) &     94.6 (    21.6)   &  M  20, F   8 &        12.7 (     2.8) &  M  20, F   8 \\\\\n"
114 |           ]
115 |         }
116 |       ]
117 |     },
118 |     {
119 |       "cell_type": "code",
120 |       "source": [
121 |         "#Data preparation\n",
122 |         "#prepare_data.py [--folds=N] [--whole] [--male] [--threshold] [--leave-site-out] [--NYU-site-out] [<derivative> ...]\n",
123 |         "!python prepare_data.py --leave-site-out cc200 aal ez"
124 |       ],
125 |       "metadata": {
126 |         "id": "l1IXag5c_e8q"
127 |       },
128 |       "execution_count": null,
129 |       "outputs": []
130 |     },
131 |     {
132 |       "cell_type": "code",
133 |       "execution_count": null,
134 |       "metadata": {
135 |         "id": "MtvhmBUb_WIS"
136 |       },
137 |       "outputs": [],
138 |       "source": [
139 |         "#use for train the model\n",
140 |         "#nn.py [--whole] [--male] [--threshold] [--leave-site-out] [<derivative> ...]\n",
141 |         "!rm /content/drive/MyDrive/acerta-abide/data/models/*mlp*\n",
142 |         "!python nn.py --leave-site-out cc200 aal ez"
143 |       ]
144 |     },
145 |     {
146 |       "cell_type": "code",
147 |       "execution_count": 9,
148 |       "metadata": {
149 |         "id": "WUUYOTjWP7VJ",
150 |         "colab": {
151 |           "base_uri": "https://localhost:8080/"
152 |         },
153 |         "outputId": "4e14070f-c239-44ec-faeb-dcc644a79301"
154 |       },
155 |       "outputs": [
156 |         {
157 |           "output_type": "stream",
158 |           "name": "stdout",
159 |           "text": [
160 |             "2023-06-06 18:12:38.120168: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
161 |             "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
162 |             "2023-06-06 18:12:39.889241: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
163 |             "['aal_leavesiteout-NYU', 'cc200_leavesiteout-NYU', 'ez_leavesiteout-NYU']\n",
164 |             "aal_leavesiteout-NYU\n",
165 |             "WARNING:tensorflow:From /usr/local/lib/python3.10/dist-packages/tensorflow/python/util/dispatch.py:1176: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
166 |             "Instructions for updating:\n",
167 |             "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
168 |             "WARNING:tensorflow:From /usr/local/lib/python3.10/dist-packages/tensorflow/python/util/dispatch.py:1176: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
169 |             "Instructions for updating:\n",
170 |             "\n",
171 |             "Future major versions of TensorFlow will allow gradients to flow\n",
172 |             "into the labels input on backprop by default.\n",
173 |             "\n",
174 |             "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
175 |             "\n",
176 |             "2023-06-06 18:12:43.366832: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:353] MLIR V1 optimization pass is not enabled\n",
177 |             "cc200_leavesiteout-NYU\n",
178 |             "ez_leavesiteout-NYU\n",
179 |             "aaa    index                     Exp  Accuracy  Precision  Recall   F-score  Sensivity  Specificity\n",
180 |             "0      0    aal_leavesiteout-NYU  0.754286   0.702128    0.99  0.821577       0.99     0.440000\n",
181 |             "1      1  cc200_leavesiteout-NYU  0.965714   0.979592    0.96  0.969697       0.96     0.973333\n",
182 |             "2      2     ez_leavesiteout-NYU  0.834286   0.793388    0.96  0.868778       0.96     0.666667\n"
183 |           ]
184 |         }
185 |       ],
186 |       "source": [
187 |         "#nn_evaluate.py [--whole] [--male] [--threshold] [--leave-site-out] [--NYU-site-out] [<derivative> ...]\n",
188 |         "!python nn_evaluate.py --leave-site-out cc200 aal ez"
189 |       ]
190 |     }
191 |   ],
192 |   "metadata": {
193 |     "colab": {
194 |       "provenance": [],
195 |       "gpuType": "T4",
196 |       "include_colab_link": true
197 |     },
198 |     "gpuClass": "standard",
199 |     "kernelspec": {
200 |       "display_name": "Python 3",
201 |       "name": "python3"
202 |     },
203 |     "language_info": {
204 |       "name": "python"
205 |     }
206 |   },
207 |   "nbformat": 4,
208 |   "nbformat_minor": 0
209 | }


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/nn.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | """
  5 | 
  6 | Autoencoders training and fine-tuning.
  7 | 
  8 | Usage:
  9 |   nn.py [--whole] [--male] [--threshold] [--leave-site-out] [<derivative> ...]
 10 |   nn.py (-h | --help)
 11 | 
 12 | Options:
 13 |   -h --help           Show this screen
 14 |   --whole             Run model for the whole dataset
 15 |   --male              Run model for male subjects
 16 |   --threshold         Run model for thresholded subjects
 17 |   --leave-site-out    Prepare data using leave-site-out method
 18 |   derivative          Derivatives to process
 19 | 
 20 | """
 21 | 
 22 | import os
 23 | import numpy as np
 24 | import tensorflow.compat.v1 as tf
 25 | from sklearn.feature_selection import SelectKBest, f_classif
 26 | 
 27 | 
 28 | from docopt import docopt
 29 | from utils import (load_phenotypes, format_config, hdf5_handler, load_fold,
 30 |                    sparsity_penalty, reset, to_softmax, load_ae_encoder)
 31 | 
 32 | from model import ae, nn
 33 | 
 34 | 
 35 | def run_autoencoder1(experiment,
 36 |                      X_train, y_train, X_valid, y_valid, X_test, y_test,
 37 |                      model_path, code_size=1000):
 38 |     """
 39 | 
 40 |     Run the first autoencoder.
 41 |     It takes the original data dimensionality and compresses it into `code_size`
 42 | 
 43 |     """
 44 | 
 45 |     # Hyperparameters
 46 |     learning_rate = 0.0001
 47 |     sparse = True  # Add sparsity penalty
 48 |     sparse_p = 0.2
 49 |     sparse_coeff = 0.5
 50 |     corruption = 0.7  # Data corruption ratio for denoising
 51 |     ae_enc = tf.nn.tanh  # Tangent hyperbolic
 52 |     ae_dec = None  # Linear activation
 53 | 
 54 |     training_iters = 700
 55 |     batch_size = 100
 56 |     n_classes = 2
 57 | 
 58 |     if os.path.isfile(model_path) or \
 59 |        os.path.isfile(model_path + ".meta"):
 60 |         return
 61 |     
 62 |     tf.disable_v2_behavior()
 63 |     print(len(X_train[100]))
 64 |     # Create model and add sparsity penalty (if requested)
 65 |     model = ae(X_train.shape[1], code_size, corruption=corruption, enc=ae_enc, dec=ae_dec)
 66 |     print(X_train.shape[1])
 67 |     if sparse:
 68 |         model["cost"] += sparsity_penalty(model["encode"], sparse_p, sparse_coeff)
 69 | 
 70 |     # Use GD for optimization of model cost
 71 |     optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(model["cost"])
 72 | 
 73 |     # Initialize Tensorflow session
 74 |     init = tf.global_variables_initializer()
 75 |     with tf.Session() as sess:
 76 |         sess.run(init)
 77 | 
 78 |         # Define model saver
 79 |         saver = tf.train.Saver(model["params"], write_version=tf.train.SaverDef.V2)
 80 | 
 81 |         # Initialize with an absurd cost for model selection
 82 |         prev_costs = np.array([9999999999] * 3)
 83 | 
 84 |         for epoch in range(training_iters):
 85 | 
 86 |             # Break training set into batches
 87 |             batches = range(int(len(X_train) / batch_size))
 88 |             costs = np.zeros((len(batches), 3))
 89 | 
 90 |             for ib in batches:
 91 | 
 92 |                 # Compute start and end of batch from training set data array
 93 |                 from_i = ib * batch_size
 94 |                 to_i = (ib + 1) * batch_size
 95 | 
 96 |                 # Select current batch
 97 |                 batch_xs, batch_ys = X_train[from_i:to_i], y_train[from_i:to_i]
 98 | 
 99 |                 # Run optimization and retrieve training cost
100 |                 _, cost_train = sess.run(
101 |                     [optimizer, model["cost"]],
102 |                     feed_dict={
103 |                         model["input"]: batch_xs
104 |                     }
105 |                 )
106 | 
107 |                 # Compute validation cost
108 |                 cost_valid = sess.run(
109 |                     model["cost"],
110 |                     feed_dict={
111 |                         model["input"]: X_valid
112 |                     }
113 |                 )
114 | 
115 |                 # Compute test cost
116 |                 cost_test = sess.run(
117 |                     model["cost"],
118 |                     feed_dict={
119 |                         model["input"]: X_test
120 |                     }
121 |                 )
122 | 
123 |                 costs[ib] = [cost_train, cost_valid, cost_test]
124 | 
125 |             # Compute the average costs from all batches
126 |             costs = costs.mean(axis=0)
127 |             cost_train, cost_valid, cost_test = costs
128 | 
129 |             # Pretty print training info
130 | 
131 |             print (
132 |                 "Exp={experiment}, Model= ae1, Iter={epoch:5d}, Cost={cost_train:.6f} {cost_valid:.6f} {cost_test:.6f}",
133 |                 {
134 |                     "experiment": experiment,
135 |                     "epoch": epoch,
136 |                     "cost_train": cost_train,
137 |                     "cost_valid": cost_valid,
138 |                     "cost_test": cost_test,
139 |                 } 
140 |                 )
141 | 
142 |         
143 | 
144 |             # Save better model if optimization achieves a lower cost
145 |             if cost_valid < prev_costs[1]:
146 |                 print ("Saving better model")
147 |                 saver.save(sess, model_path)
148 |                 prev_costs = costs
149 |             else:
150 |                 print
151 | 
152 | 
153 | def run_autoencoder2(experiment,
154 |                      X_train, y_train, X_valid, y_valid, X_test, y_test,
155 |                      model_path, prev_model_path,
156 |                      code_size=600, prev_code_size=1000):
157 |     """
158 | 
159 |     Run the second autoencoder.
160 |     It takes the dimensionality from first autoencoder and compresses it into the new `code_size`
161 |     Firstly, we need to convert original data to the new projection from autoencoder 1.
162 | 
163 |     """
164 | 
165 |     if os.path.isfile(model_path) or \
166 |        os.path.isfile(model_path + ".meta"):
167 |         return
168 | 
169 |     tf.disable_v2_behavior()
170 | 
171 |     # Convert training, validation and test set to the new representation
172 |     prev_model = ae(X_train.shape[1], prev_code_size,
173 |                     corruption=0.0,  # Disable corruption for conversion
174 |                     enc=tf.nn.tanh, dec=None)
175 | 
176 |     init = tf.global_variables_initializer()
177 |     with tf.Session() as sess:
178 |         sess.run(init)
179 |         saver = tf.train.Saver(prev_model["params"], write_version=tf.train.SaverDef.V2)
180 |         if os.path.isfile(prev_model_path):
181 |             saver.restore(sess, prev_model_path)
182 |         X_train = sess.run(prev_model["encode"], feed_dict={prev_model["input"]: X_train})
183 |         X_valid = sess.run(prev_model["encode"], feed_dict={prev_model["input"]: X_valid})
184 |         X_test = sess.run(prev_model["encode"], feed_dict={prev_model["input"]: X_test})
185 |     del prev_model
186 | 
187 |     reset()
188 | 
189 |     # Hyperparameters
190 |     learning_rate = 0.0001
191 |     corruption = 0.9
192 |     ae_enc = tf.nn.tanh
193 |     ae_dec = None
194 | 
195 |     training_iters = 1000
196 |     batch_size = 10
197 |     n_classes = 2
198 | 
199 |     # Load model
200 |     model = ae(prev_code_size, code_size, corruption=corruption, enc=ae_enc, dec=ae_dec)
201 | 
202 |     # Use GD for optimization of model cost
203 |     optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(model["cost"])
204 | 
205 |     # Initialize Tensorflow session
206 |     init = tf.global_variables_initializer()
207 |     with tf.Session() as sess:
208 |         sess.run(init)
209 | 
210 |         # Define model saver
211 |         saver = tf.train.Saver(model["params"], write_version=tf.train.SaverDef.V2)
212 | 
213 |         # Initialize with an absurd cost for model selection
214 |         prev_costs = np.array([9999999999] * 3)
215 | 
216 |         # Iterate Epochs
217 |         for epoch in range(training_iters):
218 | 
219 |             # Break training set into batches
220 |             batches = range(int(len(X_train) / batch_size))
221 |             costs = np.zeros((len(batches), 3))
222 | 
223 |             for ib in batches:
224 | 
225 |                 # Compute start and end of batch from training set data array
226 |                 from_i = ib * batch_size
227 |                 to_i = (ib + 1) * batch_size
228 | 
229 |                 # Select current batch
230 |                 batch_xs, batch_ys = X_train[from_i:to_i], y_train[from_i:to_i]
231 | 
232 |                 # Run optimization and retrieve training cost
233 |                 _, cost_train = sess.run(
234 |                     [optimizer, model["cost"]],
235 |                     feed_dict={
236 |                         model["input"]: batch_xs
237 |                     }
238 |                 )
239 | 
240 |                 # Compute validation cost
241 |                 cost_valid = sess.run(
242 |                     model["cost"],
243 |                     feed_dict={
244 |                         model["input"]: X_valid
245 |                     }
246 |                 )
247 | 
248 |                 # Compute test cost
249 |                 cost_test = sess.run(
250 |                     model["cost"],
251 |                     feed_dict={
252 |                         model["input"]: X_test
253 |                     }
254 |                 )
255 | 
256 |                 costs[ib] = [cost_train, cost_valid, cost_test]
257 | 
258 |             # Compute the average costs from all batches
259 |             costs = costs.mean(axis=0)
260 |             cost_train, cost_valid, cost_test = costs
261 | 
262 |             # Pretty print training info
263 |             print (
264 |                 "Exp={experiment}, Model=ae2, Iter={epoch:5d}, Cost={cost_train:.6f} {cost_valid:.6f} {cost_test:.6f}",
265 |                 {
266 |                     "experiment": experiment,
267 |                     "epoch": epoch,
268 |                     "cost_train": cost_train,
269 |                     "cost_valid": cost_valid,
270 |                     "cost_test": cost_test,
271 |                 }
272 |             )
273 | 
274 |             # Save better model if optimization achieves a lower cost
275 |             if cost_valid < prev_costs[1]:
276 |                 print ("Saving better model")
277 |                 saver.save(sess, model_path)
278 |                 prev_costs = costs
279 |             else:
280 |                 print
281 | 
282 | 
283 | def run_finetuning(experiment,
284 |                    X_train, y_train, X_valid, y_valid, X_test, y_test,
285 |                    model_path, prev_model_1_path, prev_model_2_path,
286 |                    code_size_1=1000, code_size_2=600):
287 |     """
288 | 
289 |     Run the pre-trained NN for fine-tuning, using first and second autoencoders' weights
290 | 
291 |     """
292 | 
293 |     # Hyperparameters
294 |     learning_rate = 0.0005
295 |     dropout_1 = 0.5
296 |     dropout_2 = 0.8
297 |     dropout_3 = 0.3
298 |     initial_momentum = 0.1
299 |     final_momentum = 0.9  # Increase momentum along epochs to avoid fluctiations
300 |     saturate_momentum = 100
301 | 
302 |     training_iters = 200
303 |     start_saving_at = 20
304 |     batch_size = 10
305 |     n_classes = 2
306 | 
307 | 
308 |     if os.path.isfile(model_path) or \
309 |        os.path.isfile(model_path + ".meta"):
310 |         return
311 | 
312 |     # Convert output to one-hot encoding
313 |     y_train = np.array([to_softmax(n_classes, y) for y in y_train])
314 |     y_valid = np.array([to_softmax(n_classes, y) for y in y_valid])
315 |     y_test = np.array([to_softmax(n_classes, y) for y in y_test])
316 | 
317 |     # Load pretrained encoder weights
318 |     ae1 = load_ae_encoder(X_train.shape[1], code_size_1, prev_model_1_path)
319 |     ae2 = load_ae_encoder(code_size_1, code_size_2, prev_model_2_path)
320 | 
321 |     # Initialize NN model with the encoder weights
322 |     model = nn(X_train.shape[1], n_classes, [
323 |         {"size": code_size_1, "actv": tf.nn.tanh},
324 |         {"size": code_size_2, "actv": tf.nn.tanh},
325 |         {"size": 100, "actv": tf.nn.tanh},
326 |     ], [
327 |         {"W": ae1["W_enc"], "b": ae1["b_enc"]},
328 |         {"W": ae2["W_enc"], "b": ae2["b_enc"]},
329 |         {"W": (np.random.randn(600,100)/10000).astype(np.float32), "b": ae2["b_enc"][:100]},
330 |     ])
331 | 
332 |     # Place GD + momentum optimizer
333 |     model["momentum"] = tf.placeholder("float32")
334 |     optimizer = tf.train.MomentumOptimizer(learning_rate, model["momentum"]).minimize(model["cost"])
335 | 
336 |     # Compute accuracies
337 |     correct_prediction = tf.equal(
338 |         tf.argmax(model["output"], 1),
339 |         tf.argmax(model["expected"], 1)
340 |     )
341 |     accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
342 | 
343 |     # Initialize Tensorflow session
344 |     init = tf.global_variables_initializer()
345 |     with tf.Session() as sess:
346 |         sess.run(init)
347 | 
348 |         # Define model saver
349 |         saver = tf.train.Saver(model["params"], write_version=tf.train.SaverDef.V2)
350 | 
351 |         # Initialize with an absurd cost and accuracy for model selection
352 |         prev_costs = np.array([9999999999] * 3)
353 |         prev_accs = np.array([0.0] * 3)
354 | 
355 |         # Iterate Epochs
356 |         
357 |         for epoch in range(training_iters):
358 | 
359 |             # Break training set into batches
360 |             batches = range(int(len(X_train) / batch_size))
361 |             costs = np.zeros((len(batches), 3))
362 |             accs = np.zeros((len(batches), 3))
363 | 
364 |             # Compute momentum saturation
365 |             alpha = float(epoch) / float(saturate_momentum)
366 |             if alpha < 0.:
367 |                 alpha = 0.
368 |             if alpha > 1.:
369 |                 alpha = 1.
370 |             momentum = initial_momentum * (1 - alpha) + alpha * final_momentum
371 | 
372 |             total_train=0
373 |             for ib in batches:
374 | 
375 |                 # Compute start and end of batch from training set data array
376 |                 from_i = ib * batch_size
377 |                 to_i = (ib + 1) * batch_size
378 | 
379 |                 # Select current batch
380 |                 batch_xs, batch_ys = X_train[from_i:to_i], y_train[from_i:to_i]
381 | 
382 |                 # Run optimization and retrieve training cost and accuracy
383 |                 _, cost_train, acc_train,y_train_expected,y_train_output = sess.run(
384 |                     [optimizer, model["cost"], accuracy, model["expected"], model["output"]],
385 |                     feed_dict={
386 |                         model["input"]: batch_xs,
387 |                         model["expected"]: batch_ys,
388 |                         model["dropouts"][0]: dropout_1,
389 |                         model["dropouts"][1]: dropout_2,
390 |                         model["dropouts"][2]: dropout_3,
391 |                         model["momentum"]: momentum,
392 |                     }
393 |                 )
394 |                 
395 |                 # Compute validation cost and accuracy
396 |                 cost_valid, acc_valid = sess.run(
397 |                     [model["cost"], accuracy],
398 |                     feed_dict={
399 |                         model["input"]: X_valid,
400 |                         model["expected"]: y_valid,
401 |                         model["dropouts"][0]: 1.0,
402 |                         model["dropouts"][1]: 1.0,
403 |                         model["dropouts"][2]: 1.0,
404 |                     }
405 |                 )
406 | 
407 |                 # Compute test cost and accuracy
408 |                 cost_test,acc_test,y,y_pred= sess.run(
409 |                     [model["cost"], accuracy, model["expected"], model["output"]],
410 |                     feed_dict={
411 |                         model["input"]: X_test,
412 |                         model["expected"]: y_test,
413 |                         model["dropouts"][0]: 1.0,
414 |                         model["dropouts"][1]: 1.0,
415 |                         model["dropouts"][2]: 1.0,
416 |                     }
417 |                 )
418 | 
419 |                 # if epoch==20:
420 |                 #   print(batch_xs.shape)
421 |                 #   total_train+=batch_xs.shape[0]
422 |                 #   print(y_train_expected,y_train_output)
423 |                   
424 |                 costs[ib] = [cost_train, cost_valid, cost_test]
425 |                 accs[ib] = [acc_train, acc_valid, acc_test]
426 | 
427 |             # print(total_train+X_valid.shape[0]+X_test.shape[0])
428 | 
429 |             # Compute the average costs from all batches
430 |             costs = costs.mean(axis=0)
431 |             cost_train, cost_valid, cost_test = costs
432 | 
433 |             # Compute the average accuracy from all batches
434 |             accs = accs.mean(axis=0)
435 |             acc_train, acc_valid, acc_test = accs
436 | 
437 |             # Pretty print training info
438 |             print (
439 |                 "Exp={experiment}, Model=mlp, Iter={epoch:5d}, Acc={acc_train:.6f} {acc_valid:.6f} {acc_test:.6f}, Momentum={momentum:.6f}",
440 |                 {
441 |                     "experiment": experiment,
442 |                     "epoch": epoch,
443 |                     "acc_train": acc_train,
444 |                     "acc_valid": acc_valid,
445 |                     "acc_test": acc_test,
446 |                     "momentum": momentum,
447 |                 }
448 |             )
449 |         
450 |             # Save better model if optimization achieves a lower accuracy
451 |             # and avoid initial epochs because of the fluctuations
452 |             if acc_valid > prev_accs[1] and epoch > start_saving_at:
453 |                 print ("Saving better model")
454 |                 saver.save(sess, model_path)
455 |                 prev_accs = accs
456 |                 prev_costs = costs
457 |             else:
458 |                 print
459 |           
460 | 
461 |         #print(y,y_pred)
462 |         return y,y_pred
463 | 
464 | 
465 | def run_nn(hdf5, experiment, code_size_1, code_size_2):
466 |     tf.disable_v2_behavior()
467 | 
468 |     exp_storage = hdf5["experiments"][experiment]
469 | 
470 |     for fold in exp_storage:
471 |         
472 |         experiment_cv = format_config("{experiment}_{fold}", {
473 |             "experiment": experiment,
474 |             "fold": fold,
475 |         })
476 |        
477 |         #print(hdf5["patients"])
478 |         # print(hdf5["experiments"].keys())
479 |         # for i in hdf5["experiments"]["cc200_whole"]["0"]["train"]:
480 |         #   print(i)
481 |         
482 |         X_train, y_train, \
483 |         X_valid, y_valid, \
484 |         X_test, y_test = load_fold(hdf5["patients"], exp_storage, fold)
485 | 
486 |         X_all=np.vstack((X_train,X_valid,X_test))
487 |         #np.save('X_NYU.npy',X_all)
488 |         print(X_all.shape)
489 |         
490 | 
491 |         X_all=X_all[:,:-2]
492 |         y_all=np.concatenate((np.array(y_train),np.array(y_valid),np.array(y_test)),axis=0)
493 |         #np.save('y_NYU.npy',y_all)
494 |         
495 |         print(y_all.shape)
496 | 
497 |         ks=0
498 |         if X_all.shape[1]<10000:
499 |           ks=1000
500 |         else:
501 |           ks=3000
502 |         X_new=SelectKBest(f_classif, k=ks).fit_transform(X_all, y_all)
503 |         print(X_new.shape)
504 | 
505 |         train=X_train.shape[0]
506 |         valid=X_valid.shape[0]
507 |         test=X_test.shape[0]
508 | 
509 |       
510 |         X_train=X_new[:train]
511 |         X_valid=X_new[train:train+valid]
512 |         X_test=X_new[train+valid:train+valid+test]
513 | 
514 | 
515 |         X_pheno=np.concatenate((X_new,X_all[:,-2:]),axis=1)
516 |         print(X_pheno.shape)
517 | 
518 |         X_train_2=X_pheno[:train]
519 |         X_valid_2=X_pheno[train:train+valid]
520 |         X_test_2=X_pheno[train+valid:train+valid+test]
521 | 
522 |         print(X_test_2.shape)
523 | 
524 |         #os._exit()
525 | 
526 |         '''
527 |         X_NYU=np.load('X_NYU.npy')
528 |         y_NYU=np.load('y_NYU.npy')
529 | 
530 |         X_sub=np.load('/content/drive/MyDrive/acerta-abide/X_sub_without_NYU.npy')
531 |         y_sub=np.load('/content/drive/MyDrive/acerta-abide/y_sub_without_NYU.npy')
532 |         
533 |         X=np.vstack((X_NYU,X_sub))
534 |         y=np.concatenate((y_NYU,y_sub))
535 | 
536 |         ks=0
537 |         if X.shape[1]<10000:
538 |           ks=1000
539 |         else:
540 |           ks=3000
541 |         X_new=SelectKBest(f_classif,k=ks).fit_transform(X, y)
542 |         print(X_new.shape)
543 | 
544 |         
545 |         X_train,X_test,y_train,y_test = train_test_split(X_new,y,test_size=0.1)
546 |         X_train,X_valid,y_train,y_valid = train_test_split(X_train,y_train,test_size=0.33)
547 | 
548 |         print(X_train.shape,X_valid.shape,X_test.shape)'''
549 |         
550 | 
551 | 
552 |         
553 |         ae1_model_path = format_config("./data/models/{experiment}_autoencoder-1.ckpt", {
554 |             "experiment": experiment_cv,
555 |         })
556 |         ae2_model_path = format_config("./data/models/{experiment}_autoencoder-2.ckpt", {
557 |             "experiment": experiment_cv,
558 |         })
559 |         nn_model_path = format_config("./data/models/{experiment}_mlp.ckpt", {
560 |             "experiment": experiment_cv,
561 |         })
562 | 
563 |         reset()
564 | 
565 |         # Run first autoencoder
566 |         run_autoencoder1(experiment_cv,
567 |                          X_train, y_train, X_valid, y_valid, X_test, y_test,
568 |                          model_path=ae1_model_path,
569 |                          code_size=code_size_1)
570 | 
571 |         reset()
572 | 
573 |         # Run second autoencoder
574 |         run_autoencoder2(experiment_cv,
575 |                          X_train, y_train, X_valid, y_valid, X_test, y_test,
576 |                          model_path=ae2_model_path,
577 |                          prev_model_path=ae1_model_path,
578 |                          prev_code_size=code_size_1,
579 |                          code_size=code_size_2)
580 | 
581 |         reset()
582 | 
583 |         # Run multilayer NN with pre-trained autoencoders
584 |         y_test,y_pred=run_finetuning(experiment_cv,
585 |                        X_train_2, y_train, X_valid_2, y_valid, X_test_2, y_test,
586 |                        model_path=nn_model_path,
587 |                        prev_model_1_path=ae1_model_path,
588 |                        prev_model_2_path=ae2_model_path,
589 |                        code_size_1=code_size_1,
590 |                        code_size_2=code_size_2)
591 |         return y_test, y_pred
592 | 
593 | if __name__ == "__main__":
594 | 
595 |     reset()
596 | 
597 |     arguments = docopt(__doc__)
598 | 
599 |     pheno_path = "./data/phenotypes/Phenotypic_V1_0b_preprocessed1.csv"
600 |     pheno = load_phenotypes(pheno_path)
601 | 
602 |     # hdf5 = hdf5_handler("./data/abide.hdf5", "a")
603 |     hdf5 = hdf5_handler(bytes("./data/abide.hdf5",encoding="utf8"), 'a')
604 | 
605 |     valid_derivatives = ["cc200", "aal", "ez", "ho", "tt", "dosenbach160"]
606 |     derivatives = [derivative for derivative
607 |                    in arguments["<derivative>"]
608 |                    if derivative in valid_derivatives]
609 | 
610 |     experiments = []
611 |     #print(derivatives)
612 | 
613 |     for derivative in derivatives:
614 | 
615 |         config = {"derivative": derivative}
616 | 
617 |         if arguments["--whole"]:
618 |             experiments += [format_config("{derivative}_whole", config)]
619 | 
620 |         if arguments["--male"]:
621 |             experiments += [format_config("{derivative}_male", config)]
622 | 
623 |         if arguments["--threshold"]:
624 |             experiments += [format_config("{derivative}_threshold", config)]
625 | 
626 |         if arguments["--leave-site-out"]:
627 |             for site in pheno["SITE_ID"].unique():
628 |                 if site=='NYU':
629 |                   site_config = {"site": site}
630 |                   experiments += [
631 |                     format_config("{derivative}_leavesiteout-{site}",
632 |                                   config, site_config)
633 |                   ]
634 | 
635 |     # First autoencoder bottleneck
636 |     code_size_1 = 1000
637 | 
638 |     # Second autoencoder bottleneck
639 |     code_size_2 = 600
640 | 
641 |     experiments = sorted(experiments)
642 | 
643 |     y_pred=[]
644 |     turn=0
645 |     for experiment in experiments:
646 |  
647 |         print(experiment)
648 |         #y_pred = run_nn(hdf5, experiment, code_size_1, code_size_2)
649 |         if len(y_pred)==0:
650 |           y_test,y_pred=run_nn(hdf5, experiment, code_size_1, code_size_2)
651 |         else:
652 |           _,pred=run_nn(hdf5, experiment, code_size_1, code_size_2)
653 |           if turn==1:
654 |             y_pred+=0.5*pred
655 |           if turn==2:
656 |             y_pred+=0.7*pred
657 |         turn+=1
658 | 
659 |     
660 | 
661 |     # print(np.argmax(y_pred, 1))
662 |     # print(np.argmax(y_test, 1))
663 | 
664 |     correct_pred=sum(np.equal(np.argmax(y_pred, 1),np.argmax(y_test, 1)))
665 | 
666 |     print(correct_pred/len(np.argmax(y_test, 1)))
667 |     
668 |     #accuracy = tf.cast(correct_prediction, "float")
669 |     #print('accuracy:',accuracy)
670 |     #print(y_pred)
671 |     #print(y_test)
672 | 
673 | 


--------------------------------------------------------------------------------
/data/list-of-subjects.csv:
--------------------------------------------------------------------------------
   1 | SUB_ID,SITE_ID,FILE_ID,DX_GROUP
   2 | 50003,PITT,Pitt_0050003,1
   3 | 50004,PITT,Pitt_0050004,1
   4 | 50005,PITT,Pitt_0050005,1
   5 | 50006,PITT,Pitt_0050006,1
   6 | 50007,PITT,Pitt_0050007,1
   7 | 50008,PITT,Pitt_0050008,1
   8 | 50009,PITT,Pitt_0050009,1
   9 | 50010,PITT,Pitt_0050010,1
  10 | 50011,PITT,Pitt_0050011,1
  11 | 50012,PITT,Pitt_0050012,1
  12 | 50013,PITT,Pitt_0050013,1
  13 | 50014,PITT,Pitt_0050014,1
  14 | 50015,PITT,Pitt_0050015,1
  15 | 50016,PITT,Pitt_0050016,1
  16 | 50017,PITT,Pitt_0050017,1
  17 | 50019,PITT,Pitt_0050019,1
  18 | 50020,PITT,Pitt_0050020,1
  19 | 50022,PITT,Pitt_0050022,1
  20 | 50023,PITT,Pitt_0050023,1
  21 | 50024,PITT,Pitt_0050024,1
  22 | 50025,PITT,Pitt_0050025,1
  23 | 50026,PITT,Pitt_0050026,1
  24 | 50027,PITT,Pitt_0050027,1
  25 | 50028,PITT,Pitt_0050028,1
  26 | 50029,PITT,Pitt_0050029,1
  27 | 50030,PITT,Pitt_0050030,2
  28 | 50031,PITT,Pitt_0050031,2
  29 | 50032,PITT,Pitt_0050032,2
  30 | 50033,PITT,Pitt_0050033,2
  31 | 50034,PITT,Pitt_0050034,2
  32 | 50035,PITT,Pitt_0050035,2
  33 | 50036,PITT,Pitt_0050036,2
  34 | 50037,PITT,Pitt_0050037,2
  35 | 50038,PITT,Pitt_0050038,2
  36 | 50039,PITT,Pitt_0050039,2
  37 | 50040,PITT,Pitt_0050040,2
  38 | 50041,PITT,Pitt_0050041,2
  39 | 50042,PITT,Pitt_0050042,2
  40 | 50043,PITT,Pitt_0050043,2
  41 | 50044,PITT,Pitt_0050044,2
  42 | 50045,PITT,Pitt_0050045,2
  43 | 50046,PITT,Pitt_0050046,2
  44 | 50047,PITT,Pitt_0050047,2
  45 | 50048,PITT,Pitt_0050048,2
  46 | 50049,PITT,Pitt_0050049,2
  47 | 50050,PITT,Pitt_0050050,2
  48 | 50051,PITT,Pitt_0050051,2
  49 | 50052,PITT,Pitt_0050052,2
  50 | 50053,PITT,Pitt_0050053,1
  51 | 50054,PITT,Pitt_0050054,2
  52 | 50055,PITT,Pitt_0050055,1
  53 | 50056,PITT,Pitt_0050056,1
  54 | 50057,PITT,Pitt_0050057,1
  55 | 50058,PITT,Pitt_0050058,2
  56 | 50059,PITT,Pitt_0050059,2
  57 | 50060,PITT,Pitt_0050060,2
  58 | 50102,OLIN,Olin_0050102,2
  59 | 50103,OLIN,Olin_0050103,2
  60 | 50104,OLIN,Olin_0050104,2
  61 | 50105,OLIN,Olin_0050105,2
  62 | 50106,OLIN,Olin_0050106,2
  63 | 50107,OLIN,Olin_0050107,2
  64 | 50109,OLIN,Olin_0050109,2
  65 | 50110,OLIN,Olin_0050110,2
  66 | 50111,OLIN,Olin_0050111,2
  67 | 50112,OLIN,Olin_0050112,2
  68 | 50113,OLIN,Olin_0050113,2
  69 | 50114,OLIN,Olin_0050114,2
  70 | 50115,OLIN,Olin_0050115,2
  71 | 50116,OLIN,Olin_0050116,2
  72 | 50117,OLIN,Olin_0050117,2
  73 | 50118,OLIN,Olin_0050118,1
  74 | 50119,OLIN,Olin_0050119,1
  75 | 50120,OLIN,Olin_0050120,1
  76 | 50121,OLIN,Olin_0050121,1
  77 | 50122,OLIN,Olin_0050122,1
  78 | 50123,OLIN,Olin_0050123,1
  79 | 50124,OLIN,Olin_0050124,1
  80 | 50125,OLIN,Olin_0050125,1
  81 | 50126,OLIN,Olin_0050126,1
  82 | 50127,OLIN,Olin_0050127,1
  83 | 50128,OLIN,Olin_0050128,1
  84 | 50129,OLIN,Olin_0050129,1
  85 | 50130,OLIN,Olin_0050130,1
  86 | 50131,OLIN,Olin_0050131,1
  87 | 50132,OLIN,Olin_0050132,1
  88 | 50133,OLIN,Olin_0050133,1
  89 | 50134,OLIN,Olin_0050134,1
  90 | 50135,OLIN,Olin_0050135,1
  91 | 50136,OLIN,Olin_0050136,1
  92 | 50142,OHSU,OHSU_0050142,1
  93 | 50143,OHSU,OHSU_0050143,1
  94 | 50144,OHSU,OHSU_0050144,1
  95 | 50145,OHSU,OHSU_0050145,1
  96 | 50146,OHSU,OHSU_0050146,1
  97 | 50147,OHSU,OHSU_0050147,1
  98 | 50148,OHSU,OHSU_0050148,1
  99 | 50149,OHSU,OHSU_0050149,1
 100 | 50150,OHSU,OHSU_0050150,1
 101 | 50152,OHSU,OHSU_0050152,1
 102 | 50153,OHSU,OHSU_0050153,1
 103 | 50156,OHSU,OHSU_0050156,1
 104 | 50157,OHSU,OHSU_0050157,2
 105 | 50158,OHSU,OHSU_0050158,2
 106 | 50159,OHSU,OHSU_0050159,2
 107 | 50160,OHSU,OHSU_0050160,2
 108 | 50161,OHSU,OHSU_0050161,2
 109 | 50162,OHSU,OHSU_0050162,2
 110 | 50163,OHSU,OHSU_0050163,2
 111 | 50164,OHSU,OHSU_0050164,2
 112 | 50166,OHSU,OHSU_0050166,2
 113 | 50167,OHSU,OHSU_0050167,2
 114 | 50168,OHSU,OHSU_0050168,2
 115 | 50169,OHSU,OHSU_0050169,2
 116 | 50170,OHSU,OHSU_0050170,2
 117 | 50171,OHSU,OHSU_0050171,2
 118 | 50182,SDSU,SDSU_0050182,1
 119 | 50183,SDSU,SDSU_0050183,1
 120 | 50184,SDSU,SDSU_0050184,1
 121 | 50185,SDSU,SDSU_0050185,1
 122 | 50186,SDSU,SDSU_0050186,1
 123 | 50187,SDSU,SDSU_0050187,1
 124 | 50188,SDSU,SDSU_0050188,1
 125 | 50189,SDSU,SDSU_0050189,1
 126 | 50190,SDSU,SDSU_0050190,1
 127 | 50191,SDSU,SDSU_0050191,1
 128 | 50192,SDSU,SDSU_0050192,1
 129 | 50193,SDSU,SDSU_0050193,2
 130 | 50194,SDSU,SDSU_0050194,2
 131 | 50195,SDSU,SDSU_0050195,2
 132 | 50196,SDSU,SDSU_0050196,2
 133 | 50197,SDSU,SDSU_0050197,2
 134 | 50198,SDSU,SDSU_0050198,2
 135 | 50199,SDSU,SDSU_0050199,2
 136 | 50200,SDSU,SDSU_0050200,2
 137 | 50201,SDSU,SDSU_0050201,2
 138 | 50202,SDSU,SDSU_0050202,2
 139 | 50203,SDSU,SDSU_0050203,2
 140 | 50204,SDSU,SDSU_0050204,2
 141 | 50205,SDSU,SDSU_0050205,2
 142 | 50206,SDSU,SDSU_0050206,2
 143 | 50207,SDSU,SDSU_0050207,1
 144 | 50208,SDSU,SDSU_0050208,2
 145 | 50209,SDSU,SDSU_0050209,2
 146 | 50210,SDSU,SDSU_0050210,2
 147 | 50211,SDSU,SDSU_0050211,2
 148 | 50212,SDSU,SDSU_0050212,1
 149 | 50213,SDSU,SDSU_0050213,2
 150 | 50214,SDSU,SDSU_0050214,2
 151 | 50215,SDSU,SDSU_0050215,2
 152 | 50216,SDSU,SDSU_0050216,1
 153 | 50217,SDSU,SDSU_0050217,2
 154 | 50232,TRINITY,Trinity_0050232,1
 155 | 50233,TRINITY,Trinity_0050233,1
 156 | 50234,TRINITY,Trinity_0050234,1
 157 | 50235,TRINITY,Trinity_0050235,1
 158 | 50236,TRINITY,Trinity_0050236,1
 159 | 50237,TRINITY,Trinity_0050237,1
 160 | 50239,TRINITY,Trinity_0050239,1
 161 | 50240,TRINITY,Trinity_0050240,1
 162 | 50241,TRINITY,Trinity_0050241,1
 163 | 50242,TRINITY,Trinity_0050242,1
 164 | 50243,TRINITY,Trinity_0050243,1
 165 | 50245,TRINITY,Trinity_0050245,1
 166 | 50246,TRINITY,Trinity_0050246,1
 167 | 50247,TRINITY,Trinity_0050247,1
 168 | 50248,TRINITY,Trinity_0050248,1
 169 | 50249,TRINITY,Trinity_0050249,1
 170 | 50250,TRINITY,Trinity_0050250,1
 171 | 50251,TRINITY,Trinity_0050251,1
 172 | 50252,TRINITY,Trinity_0050252,1
 173 | 50253,TRINITY,Trinity_0050253,1
 174 | 50254,TRINITY,Trinity_0050254,1
 175 | 50255,TRINITY,Trinity_0050255,1
 176 | 50257,TRINITY,Trinity_0050257,2
 177 | 50259,TRINITY,Trinity_0050259,2
 178 | 50260,TRINITY,Trinity_0050260,2
 179 | 50261,TRINITY,Trinity_0050261,2
 180 | 50262,TRINITY,Trinity_0050262,2
 181 | 50263,TRINITY,Trinity_0050263,2
 182 | 50264,TRINITY,Trinity_0050264,2
 183 | 50265,TRINITY,Trinity_0050265,2
 184 | 50266,TRINITY,Trinity_0050266,2
 185 | 50267,TRINITY,Trinity_0050267,2
 186 | 50268,TRINITY,Trinity_0050268,2
 187 | 50269,TRINITY,Trinity_0050269,2
 188 | 50270,TRINITY,Trinity_0050270,2
 189 | 50271,TRINITY,Trinity_0050271,2
 190 | 50272,UM_1,UM_1_0050272,1
 191 | 50273,UM_1,UM_1_0050273,1
 192 | 50274,UM_1,UM_1_0050274,1
 193 | 50275,UM_1,UM_1_0050275,1
 194 | 50276,UM_1,UM_1_0050276,1
 195 | 50277,UM_1,UM_1_0050277,1
 196 | 50278,UM_1,UM_1_0050278,1
 197 | 50279,UM_1,UM_1_0050279,1
 198 | 50280,UM_1,UM_1_0050280,1
 199 | 50281,UM_1,UM_1_0050281,1
 200 | 50282,UM_1,UM_1_0050282,1
 201 | 50283,UM_1,UM_1_0050283,1
 202 | 50284,UM_1,UM_1_0050284,1
 203 | 50285,UM_1,UM_1_0050285,1
 204 | 50286,UM_1,UM_1_0050286,1
 205 | 50287,UM_1,UM_1_0050287,1
 206 | 50288,UM_1,UM_1_0050288,1
 207 | 50289,UM_1,UM_1_0050289,1
 208 | 50290,UM_1,UM_1_0050290,1
 209 | 50291,UM_1,UM_1_0050291,1
 210 | 50292,UM_1,UM_1_0050292,1
 211 | 50293,UM_1,UM_1_0050293,1
 212 | 50294,UM_1,UM_1_0050294,1
 213 | 50295,UM_1,UM_1_0050295,1
 214 | 50296,UM_1,UM_1_0050296,1
 215 | 50297,UM_1,UM_1_0050297,1
 216 | 50298,UM_1,UM_1_0050298,1
 217 | 50299,UM_1,UM_1_0050299,1
 218 | 50300,UM_1,UM_1_0050300,1
 219 | 50301,UM_1,UM_1_0050301,1
 220 | 50302,UM_1,UM_1_0050302,1
 221 | 50303,UM_1,UM_1_0050303,1
 222 | 50304,UM_1,UM_1_0050304,1
 223 | 50305,UM_1,UM_1_0050305,1
 224 | 50306,UM_1,UM_1_0050306,1
 225 | 50307,UM_1,UM_1_0050307,1
 226 | 50308,UM_1,UM_1_0050308,1
 227 | 50310,UM_1,UM_1_0050310,1
 228 | 50311,UM_1,UM_1_0050311,1
 229 | 50312,UM_1,UM_1_0050312,1
 230 | 50313,UM_1,UM_1_0050313,1
 231 | 50314,UM_1,UM_1_0050314,1
 232 | 50315,UM_1,UM_1_0050315,1
 233 | 50316,UM_1,UM_1_0050316,1
 234 | 50317,UM_1,UM_1_0050317,1
 235 | 50318,UM_1,UM_1_0050318,1
 236 | 50319,UM_1,UM_1_0050319,1
 237 | 50320,UM_1,UM_1_0050320,1
 238 | 50321,UM_1,UM_1_0050321,1
 239 | 50322,UM_1,UM_1_0050322,1
 240 | 50324,UM_1,UM_1_0050324,1
 241 | 50325,UM_1,UM_1_0050325,1
 242 | 50326,UM_1,UM_1_0050326,1
 243 | 50327,UM_1,UM_1_0050327,2
 244 | 50329,UM_1,UM_1_0050329,2
 245 | 50330,UM_1,UM_1_0050330,2
 246 | 50331,UM_1,UM_1_0050331,2
 247 | 50332,UM_1,UM_1_0050332,2
 248 | 50333,UM_1,UM_1_0050333,2
 249 | 50334,UM_1,UM_1_0050334,2
 250 | 50335,UM_1,UM_1_0050335,2
 251 | 50336,UM_1,UM_1_0050336,2
 252 | 50337,UM_1,UM_1_0050337,2
 253 | 50338,UM_1,UM_1_0050338,2
 254 | 50339,UM_1,UM_1_0050339,2
 255 | 50340,UM_1,UM_1_0050340,2
 256 | 50341,UM_1,UM_1_0050341,2
 257 | 50342,UM_1,UM_1_0050342,2
 258 | 50343,UM_1,UM_1_0050343,2
 259 | 50344,UM_1,UM_1_0050344,2
 260 | 50345,UM_1,UM_1_0050345,2
 261 | 50346,UM_1,UM_1_0050346,2
 262 | 50347,UM_1,UM_1_0050347,2
 263 | 50348,UM_1,UM_1_0050348,2
 264 | 50349,UM_1,UM_1_0050349,2
 265 | 50350,UM_1,UM_1_0050350,2
 266 | 50351,UM_1,UM_1_0050351,2
 267 | 50352,UM_1,UM_1_0050352,2
 268 | 50353,UM_1,UM_1_0050353,2
 269 | 50354,UM_1,UM_1_0050354,2
 270 | 50355,UM_1,UM_1_0050355,2
 271 | 50356,UM_1,UM_1_0050356,2
 272 | 50357,UM_1,UM_1_0050357,2
 273 | 50358,UM_1,UM_1_0050358,2
 274 | 50359,UM_1,UM_1_0050359,2
 275 | 50360,UM_1,UM_1_0050360,2
 276 | 50361,UM_1,UM_1_0050361,2
 277 | 50362,UM_1,UM_1_0050362,2
 278 | 50363,UM_1,UM_1_0050363,2
 279 | 50364,UM_1,UM_1_0050364,2
 280 | 50365,UM_1,UM_1_0050365,2
 281 | 50366,UM_1,UM_1_0050366,2
 282 | 50367,UM_1,UM_1_0050367,2
 283 | 50368,UM_1,UM_1_0050368,2
 284 | 50369,UM_1,UM_1_0050369,2
 285 | 50370,UM_1,UM_1_0050370,2
 286 | 50371,UM_1,UM_1_0050371,2
 287 | 50372,UM_1,UM_1_0050372,2
 288 | 50373,UM_1,UM_1_0050373,2
 289 | 50374,UM_1,UM_1_0050374,2
 290 | 50375,UM_1,UM_1_0050375,2
 291 | 50376,UM_1,UM_1_0050376,2
 292 | 50377,UM_1,UM_1_0050377,2
 293 | 50379,UM_1,UM_1_0050379,2
 294 | 50380,UM_1,UM_1_0050380,2
 295 | 50381,UM_1,UM_1_0050381,2
 296 | 50382,UM_2,UM_2_0050382,2
 297 | 50383,UM_2,UM_2_0050383,2
 298 | 50385,UM_2,UM_2_0050385,2
 299 | 50386,UM_2,UM_2_0050386,2
 300 | 50387,UM_2,UM_2_0050387,2
 301 | 50388,UM_2,UM_2_0050388,2
 302 | 50390,UM_2,UM_2_0050390,2
 303 | 50391,UM_2,UM_2_0050391,2
 304 | 50397,UM_2,UM_2_0050397,1
 305 | 50399,UM_2,UM_2_0050399,1
 306 | 50402,UM_2,UM_2_0050402,1
 307 | 50403,UM_2,UM_2_0050403,1
 308 | 50404,UM_2,UM_2_0050404,1
 309 | 50405,UM_2,UM_2_0050405,1
 310 | 50406,UM_2,UM_2_0050406,1
 311 | 50407,UM_2,UM_2_0050407,1
 312 | 50408,UM_2,UM_2_0050408,1
 313 | 50410,UM_2,UM_2_0050410,1
 314 | 50411,UM_2,UM_2_0050411,1
 315 | 50412,UM_2,UM_2_0050412,1
 316 | 50413,UM_2,UM_2_0050413,1
 317 | 50414,UM_2,UM_2_0050414,2
 318 | 50415,UM_2,UM_2_0050415,2
 319 | 50416,UM_2,UM_2_0050416,2
 320 | 50417,UM_2,UM_2_0050417,2
 321 | 50418,UM_2,UM_2_0050418,2
 322 | 50419,UM_2,UM_2_0050419,2
 323 | 50421,UM_2,UM_2_0050421,2
 324 | 50422,UM_2,UM_2_0050422,2
 325 | 50424,UM_2,UM_2_0050424,2
 326 | 50425,UM_2,UM_2_0050425,2
 327 | 50426,UM_2,UM_2_0050426,2
 328 | 50427,UM_2,UM_2_0050427,2
 329 | 50428,UM_2,UM_2_0050428,2
 330 | 50433,USM,USM_0050433,2
 331 | 50434,USM,USM_0050434,2
 332 | 50435,USM,USM_0050435,2
 333 | 50436,USM,USM_0050436,2
 334 | 50437,USM,USM_0050437,2
 335 | 50438,USM,USM_0050438,2
 336 | 50439,USM,USM_0050439,2
 337 | 50440,USM,USM_0050440,2
 338 | 50441,USM,USM_0050441,2
 339 | 50442,USM,USM_0050442,2
 340 | 50443,USM,USM_0050443,2
 341 | 50444,USM,USM_0050444,2
 342 | 50445,USM,USM_0050445,2
 343 | 50446,USM,USM_0050446,2
 344 | 50447,USM,USM_0050447,2
 345 | 50448,USM,USM_0050448,2
 346 | 50449,USM,USM_0050449,2
 347 | 50453,USM,USM_0050453,2
 348 | 50455,USM,USM_0050455,2
 349 | 50463,USM,USM_0050463,2
 350 | 50466,USM,USM_0050466,2
 351 | 50467,USM,USM_0050467,2
 352 | 50468,USM,USM_0050468,2
 353 | 50469,USM,USM_0050469,2
 354 | 50470,USM,USM_0050470,2
 355 | 50477,USM,USM_0050477,1
 356 | 50480,USM,USM_0050480,1
 357 | 50481,USM,USM_0050481,1
 358 | 50482,USM,USM_0050482,1
 359 | 50483,USM,USM_0050483,1
 360 | 50485,USM,USM_0050485,1
 361 | 50486,USM,USM_0050486,1
 362 | 50487,USM,USM_0050487,1
 363 | 50488,USM,USM_0050488,1
 364 | 50489,USM,USM_0050489,1
 365 | 50490,USM,USM_0050490,1
 366 | 50491,USM,USM_0050491,1
 367 | 50492,USM,USM_0050492,1
 368 | 50493,USM,USM_0050493,1
 369 | 50494,USM,USM_0050494,1
 370 | 50496,USM,USM_0050496,1
 371 | 50497,USM,USM_0050497,1
 372 | 50498,USM,USM_0050498,1
 373 | 50499,USM,USM_0050499,1
 374 | 50500,USM,USM_0050500,1
 375 | 50501,USM,USM_0050501,1
 376 | 50502,USM,USM_0050502,1
 377 | 50503,USM,USM_0050503,1
 378 | 50504,USM,USM_0050504,1
 379 | 50505,USM,USM_0050505,1
 380 | 50507,USM,USM_0050507,1
 381 | 50509,USM,USM_0050509,1
 382 | 50510,USM,USM_0050510,1
 383 | 50511,USM,USM_0050511,1
 384 | 50514,USM,USM_0050514,1
 385 | 50515,USM,USM_0050515,1
 386 | 50516,USM,USM_0050516,1
 387 | 50518,USM,USM_0050518,1
 388 | 50519,USM,USM_0050519,1
 389 | 50520,USM,USM_0050520,1
 390 | 50521,USM,USM_0050521,1
 391 | 50523,USM,USM_0050523,1
 392 | 50524,USM,USM_0050524,1
 393 | 50525,USM,USM_0050525,1
 394 | 50526,USM,USM_0050526,1
 395 | 50527,USM,USM_0050527,1
 396 | 50528,USM,USM_0050528,1
 397 | 50529,USM,USM_0050529,1
 398 | 50530,USM,USM_0050530,1
 399 | 50531,USM,USM_0050531,1
 400 | 50532,USM,USM_0050532,1
 401 | 50551,YALE,Yale_0050551,2
 402 | 50552,YALE,Yale_0050552,2
 403 | 50553,YALE,Yale_0050553,2
 404 | 50554,YALE,Yale_0050554,2
 405 | 50555,YALE,Yale_0050555,2
 406 | 50556,YALE,Yale_0050556,2
 407 | 50557,YALE,Yale_0050557,2
 408 | 50558,YALE,Yale_0050558,2
 409 | 50559,YALE,Yale_0050559,2
 410 | 50560,YALE,Yale_0050560,2
 411 | 50561,YALE,Yale_0050561,2
 412 | 50562,YALE,Yale_0050562,2
 413 | 50563,YALE,Yale_0050563,2
 414 | 50564,YALE,Yale_0050564,2
 415 | 50565,YALE,Yale_0050565,2
 416 | 50566,YALE,Yale_0050566,2
 417 | 50567,YALE,Yale_0050567,2
 418 | 50568,YALE,Yale_0050568,2
 419 | 50569,YALE,Yale_0050569,2
 420 | 50570,YALE,Yale_0050570,2
 421 | 50571,YALE,Yale_0050571,2
 422 | 50572,YALE,Yale_0050572,2
 423 | 50573,YALE,Yale_0050573,2
 424 | 50574,YALE,Yale_0050574,2
 425 | 50575,YALE,Yale_0050575,2
 426 | 50576,YALE,Yale_0050576,2
 427 | 50577,YALE,Yale_0050577,2
 428 | 50578,YALE,Yale_0050578,2
 429 | 50601,YALE,Yale_0050601,1
 430 | 50602,YALE,Yale_0050602,1
 431 | 50603,YALE,Yale_0050603,1
 432 | 50604,YALE,Yale_0050604,1
 433 | 50605,YALE,Yale_0050605,1
 434 | 50606,YALE,Yale_0050606,1
 435 | 50607,YALE,Yale_0050607,1
 436 | 50608,YALE,Yale_0050608,1
 437 | 50609,YALE,Yale_0050609,1
 438 | 50610,YALE,Yale_0050610,1
 439 | 50611,YALE,Yale_0050611,1
 440 | 50612,YALE,Yale_0050612,1
 441 | 50613,YALE,Yale_0050613,1
 442 | 50614,YALE,Yale_0050614,1
 443 | 50615,YALE,Yale_0050615,1
 444 | 50616,YALE,Yale_0050616,1
 445 | 50617,YALE,Yale_0050617,1
 446 | 50618,YALE,Yale_0050618,1
 447 | 50619,YALE,Yale_0050619,1
 448 | 50620,YALE,Yale_0050620,1
 449 | 50621,YALE,Yale_0050621,1
 450 | 50622,YALE,Yale_0050622,1
 451 | 50623,YALE,Yale_0050623,1
 452 | 50624,YALE,Yale_0050624,1
 453 | 50625,YALE,Yale_0050625,1
 454 | 50626,YALE,Yale_0050626,1
 455 | 50627,YALE,Yale_0050627,1
 456 | 50628,YALE,Yale_0050628,1
 457 | 50642,CMU,CMU_a_0050642,1
 458 | 50643,CMU,CMU_b_0050643,1
 459 | 50644,CMU,CMU_b_0050644,1
 460 | 50645,CMU,CMU_b_0050645,1
 461 | 50646,CMU,CMU_a_0050646,1
 462 | 50647,CMU,CMU_a_0050647,1
 463 | 50648,CMU,CMU_b_0050648,1
 464 | 50649,CMU,CMU_a_0050649,1
 465 | 50650,CMU,CMU_b_0050650,1
 466 | 50651,CMU,CMU_b_0050651,1
 467 | 50652,CMU,CMU_b_0050652,1
 468 | 50653,CMU,CMU_a_0050653,1
 469 | 50654,CMU,CMU_a_0050654,1
 470 | 50655,CMU,CMU_b_0050655,1
 471 | 50656,CMU,CMU_a_0050656,2
 472 | 50657,CMU,CMU_b_0050657,2
 473 | 50658,CMU,CMU_b_0050658,2
 474 | 50659,CMU,CMU_a_0050659,2
 475 | 50660,CMU,CMU_a_0050660,2
 476 | 50661,CMU,CMU_b_0050661,2
 477 | 50663,CMU,CMU_a_0050663,2
 478 | 50664,CMU,CMU_a_0050664,2
 479 | 50665,CMU,CMU_a_0050665,2
 480 | 50666,CMU,CMU_a_0050666,2
 481 | 50667,CMU,CMU_b_0050667,2
 482 | 50668,CMU,CMU_a_0050668,2
 483 | 50669,CMU,CMU_b_0050669,2
 484 | 50682,LEUVEN_1,Leuven_1_0050682,2
 485 | 50683,LEUVEN_1,Leuven_1_0050683,2
 486 | 50685,LEUVEN_1,Leuven_1_0050685,2
 487 | 50686,LEUVEN_1,Leuven_1_0050686,1
 488 | 50687,LEUVEN_1,Leuven_1_0050687,2
 489 | 50688,LEUVEN_1,Leuven_1_0050688,2
 490 | 50689,LEUVEN_1,Leuven_1_0050689,1
 491 | 50690,LEUVEN_1,Leuven_1_0050690,1
 492 | 50691,LEUVEN_1,Leuven_1_0050691,2
 493 | 50692,LEUVEN_1,Leuven_1_0050692,2
 494 | 50693,LEUVEN_1,Leuven_1_0050693,1
 495 | 50694,LEUVEN_1,Leuven_1_0050694,1
 496 | 50695,LEUVEN_1,Leuven_1_0050695,1
 497 | 50696,LEUVEN_1,Leuven_1_0050696,1
 498 | 50697,LEUVEN_1,Leuven_1_0050697,1
 499 | 50698,LEUVEN_1,Leuven_1_0050698,2
 500 | 50699,LEUVEN_1,Leuven_1_0050699,2
 501 | 50700,LEUVEN_1,Leuven_1_0050700,1
 502 | 50701,LEUVEN_1,Leuven_1_0050701,2
 503 | 50702,LEUVEN_1,Leuven_1_0050702,1
 504 | 50703,LEUVEN_1,Leuven_1_0050703,2
 505 | 50704,LEUVEN_1,Leuven_1_0050704,1
 506 | 50705,LEUVEN_1,Leuven_1_0050705,1
 507 | 50706,LEUVEN_1,Leuven_1_0050706,2
 508 | 50707,LEUVEN_1,Leuven_1_0050707,2
 509 | 50708,LEUVEN_1,Leuven_1_0050708,1
 510 | 50709,LEUVEN_1,Leuven_1_0050709,2
 511 | 50710,LEUVEN_1,Leuven_1_0050710,2
 512 | 50711,LEUVEN_1,Leuven_1_0050711,1
 513 | 50722,LEUVEN_2,Leuven_2_0050722,2
 514 | 50723,LEUVEN_2,Leuven_2_0050723,2
 515 | 50724,LEUVEN_2,Leuven_2_0050724,2
 516 | 50725,LEUVEN_2,Leuven_2_0050725,2
 517 | 50726,LEUVEN_2,Leuven_2_0050726,2
 518 | 50727,LEUVEN_2,Leuven_2_0050727,2
 519 | 50728,LEUVEN_2,Leuven_2_0050728,2
 520 | 50730,LEUVEN_2,Leuven_2_0050730,2
 521 | 50731,LEUVEN_2,Leuven_2_0050731,2
 522 | 50732,LEUVEN_2,Leuven_2_0050732,2
 523 | 50733,LEUVEN_2,Leuven_2_0050733,2
 524 | 50735,LEUVEN_2,Leuven_2_0050735,2
 525 | 50736,LEUVEN_2,Leuven_2_0050736,2
 526 | 50737,LEUVEN_2,Leuven_2_0050737,2
 527 | 50738,LEUVEN_2,Leuven_2_0050738,2
 528 | 50739,LEUVEN_2,Leuven_2_0050739,2
 529 | 50740,LEUVEN_2,Leuven_2_0050740,2
 530 | 50741,LEUVEN_2,Leuven_2_0050741,2
 531 | 50742,LEUVEN_2,Leuven_2_0050742,2
 532 | 50743,LEUVEN_2,Leuven_2_0050743,1
 533 | 50744,LEUVEN_2,Leuven_2_0050744,1
 534 | 50745,LEUVEN_2,Leuven_2_0050745,1
 535 | 50746,LEUVEN_2,Leuven_2_0050746,1
 536 | 50747,LEUVEN_2,Leuven_2_0050747,1
 537 | 50748,LEUVEN_2,Leuven_2_0050748,1
 538 | 50749,LEUVEN_2,Leuven_2_0050749,1
 539 | 50750,LEUVEN_2,Leuven_2_0050750,1
 540 | 50751,LEUVEN_2,Leuven_2_0050751,1
 541 | 50752,LEUVEN_2,Leuven_2_0050752,1
 542 | 50753,LEUVEN_2,Leuven_2_0050753,1
 543 | 50754,LEUVEN_2,Leuven_2_0050754,1
 544 | 50755,LEUVEN_2,Leuven_2_0050755,1
 545 | 50756,LEUVEN_2,Leuven_2_0050756,1
 546 | 50757,LEUVEN_2,Leuven_2_0050757,1
 547 | 50772,KKI,KKI_0050772,2
 548 | 50773,KKI,KKI_0050773,2
 549 | 50774,KKI,KKI_0050774,2
 550 | 50775,KKI,KKI_0050775,2
 551 | 50776,KKI,KKI_0050776,2
 552 | 50777,KKI,KKI_0050777,2
 553 | 50778,KKI,KKI_0050778,2
 554 | 50779,KKI,KKI_0050779,2
 555 | 50780,KKI,KKI_0050780,2
 556 | 50781,KKI,KKI_0050781,2
 557 | 50782,KKI,KKI_0050782,2
 558 | 50783,KKI,KKI_0050783,2
 559 | 50784,KKI,KKI_0050784,2
 560 | 50785,KKI,KKI_0050785,2
 561 | 50786,KKI,KKI_0050786,2
 562 | 50787,KKI,KKI_0050787,2
 563 | 50788,KKI,KKI_0050788,2
 564 | 50789,KKI,KKI_0050789,2
 565 | 50790,KKI,KKI_0050790,2
 566 | 50791,KKI,KKI_0050791,1
 567 | 50792,KKI,KKI_0050792,1
 568 | 50793,KKI,KKI_0050793,1
 569 | 50794,KKI,KKI_0050794,1
 570 | 50795,KKI,KKI_0050795,1
 571 | 50796,KKI,KKI_0050796,1
 572 | 50797,KKI,KKI_0050797,1
 573 | 50798,KKI,KKI_0050798,1
 574 | 50799,KKI,KKI_0050799,1
 575 | 50800,KKI,KKI_0050800,1
 576 | 50801,KKI,KKI_0050801,1
 577 | 50802,KKI,KKI_0050802,1
 578 | 50803,KKI,KKI_0050803,1
 579 | 50804,KKI,KKI_0050804,1
 580 | 50807,KKI,KKI_0050807,1
 581 | 50812,KKI,KKI_0050812,2
 582 | 50814,KKI,KKI_0050814,2
 583 | 50815,KKI,KKI_0050815,1
 584 | 50816,KKI,KKI_0050816,2
 585 | 50817,KKI,KKI_0050817,2
 586 | 50818,KKI,KKI_0050818,2
 587 | 50819,KKI,KKI_0050819,2
 588 | 50820,KKI,KKI_0050820,2
 589 | 50821,KKI,KKI_0050821,2
 590 | 50822,KKI,KKI_0050822,2
 591 | 50823,KKI,KKI_0050823,1
 592 | 50824,KKI,KKI_0050824,1
 593 | 50825,KKI,KKI_0050825,1
 594 | 50826,KKI,KKI_0050826,1
 595 | 50952,NYU,NYU_0050952,1
 596 | 50954,NYU,NYU_0050954,1
 597 | 50955,NYU,NYU_0050955,1
 598 | 50956,NYU,NYU_0050956,1
 599 | 50957,NYU,NYU_0050957,1
 600 | 50958,NYU,NYU_0050958,1
 601 | 50959,NYU,NYU_0050959,1
 602 | 50960,NYU,NYU_0050960,1
 603 | 50961,NYU,NYU_0050961,1
 604 | 50962,NYU,NYU_0050962,1
 605 | 50964,NYU,NYU_0050964,1
 606 | 50965,NYU,NYU_0050965,1
 607 | 50966,NYU,NYU_0050966,1
 608 | 50967,NYU,NYU_0050967,1
 609 | 50968,NYU,NYU_0050968,1
 610 | 50969,NYU,NYU_0050969,1
 611 | 50970,NYU,NYU_0050970,1
 612 | 50972,NYU,NYU_0050972,1
 613 | 50973,NYU,NYU_0050973,1
 614 | 50974,NYU,NYU_0050974,1
 615 | 50976,NYU,NYU_0050976,1
 616 | 50977,NYU,NYU_0050977,1
 617 | 50978,NYU,NYU_0050978,1
 618 | 50979,NYU,NYU_0050979,1
 619 | 50981,NYU,NYU_0050981,1
 620 | 50982,NYU,NYU_0050982,1
 621 | 50983,NYU,NYU_0050983,1
 622 | 50984,NYU,NYU_0050984,1
 623 | 50985,NYU,NYU_0050985,1
 624 | 50986,NYU,NYU_0050986,1
 625 | 50987,NYU,NYU_0050987,1
 626 | 50988,NYU,NYU_0050988,1
 627 | 50989,NYU,NYU_0050989,1
 628 | 50990,NYU,NYU_0050990,1
 629 | 50991,NYU,NYU_0050991,1
 630 | 50992,NYU,NYU_0050992,1
 631 | 50993,NYU,NYU_0050993,1
 632 | 50994,NYU,NYU_0050994,1
 633 | 50995,NYU,NYU_0050995,1
 634 | 50996,NYU,NYU_0050996,1
 635 | 50997,NYU,NYU_0050997,1
 636 | 50998,NYU,NYU_0050998,1
 637 | 50999,NYU,NYU_0050999,1
 638 | 51000,NYU,NYU_0051000,1
 639 | 51001,NYU,NYU_0051001,1
 640 | 51002,NYU,NYU_0051002,1
 641 | 51003,NYU,NYU_0051003,1
 642 | 51006,NYU,NYU_0051006,1
 643 | 51007,NYU,NYU_0051007,1
 644 | 51008,NYU,NYU_0051008,1
 645 | 51009,NYU,NYU_0051009,1
 646 | 51010,NYU,NYU_0051010,1
 647 | 51011,NYU,NYU_0051011,1
 648 | 51012,NYU,NYU_0051012,1
 649 | 51013,NYU,NYU_0051013,1
 650 | 51014,NYU,NYU_0051014,1
 651 | 51015,NYU,NYU_0051015,1
 652 | 51016,NYU,NYU_0051016,1
 653 | 51017,NYU,NYU_0051017,1
 654 | 51018,NYU,NYU_0051018,1
 655 | 51019,NYU,NYU_0051019,1
 656 | 51020,NYU,NYU_0051020,1
 657 | 51021,NYU,NYU_0051021,1
 658 | 51023,NYU,NYU_0051023,1
 659 | 51024,NYU,NYU_0051024,1
 660 | 51025,NYU,NYU_0051025,1
 661 | 51026,NYU,NYU_0051026,1
 662 | 51027,NYU,NYU_0051027,1
 663 | 51028,NYU,NYU_0051028,1
 664 | 51029,NYU,NYU_0051029,1
 665 | 51030,NYU,NYU_0051030,1
 666 | 51032,NYU,NYU_0051032,1
 667 | 51033,NYU,NYU_0051033,1
 668 | 51034,NYU,NYU_0051034,1
 669 | 51035,NYU,NYU_0051035,1
 670 | 51036,NYU,NYU_0051036,2
 671 | 51038,NYU,NYU_0051038,2
 672 | 51039,NYU,NYU_0051039,2
 673 | 51040,NYU,NYU_0051040,2
 674 | 51041,NYU,NYU_0051041,2
 675 | 51042,NYU,NYU_0051042,2
 676 | 51044,NYU,NYU_0051044,2
 677 | 51045,NYU,NYU_0051045,2
 678 | 51046,NYU,NYU_0051046,2
 679 | 51047,NYU,NYU_0051047,2
 680 | 51048,NYU,NYU_0051048,2
 681 | 51049,NYU,NYU_0051049,2
 682 | 51050,NYU,NYU_0051050,2
 683 | 51051,NYU,NYU_0051051,2
 684 | 51052,NYU,NYU_0051052,2
 685 | 51053,NYU,NYU_0051053,2
 686 | 51054,NYU,NYU_0051054,2
 687 | 51055,NYU,NYU_0051055,2
 688 | 51056,NYU,NYU_0051056,2
 689 | 51057,NYU,NYU_0051057,2
 690 | 51058,NYU,NYU_0051058,2
 691 | 51059,NYU,NYU_0051059,2
 692 | 51060,NYU,NYU_0051060,2
 693 | 51061,NYU,NYU_0051061,2
 694 | 51062,NYU,NYU_0051062,2
 695 | 51063,NYU,NYU_0051063,2
 696 | 51064,NYU,NYU_0051064,2
 697 | 51065,NYU,NYU_0051065,2
 698 | 51066,NYU,NYU_0051066,2
 699 | 51067,NYU,NYU_0051067,2
 700 | 51068,NYU,NYU_0051068,2
 701 | 51069,NYU,NYU_0051069,2
 702 | 51070,NYU,NYU_0051070,2
 703 | 51071,NYU,NYU_0051071,2
 704 | 51072,NYU,NYU_0051072,2
 705 | 51073,NYU,NYU_0051073,2
 706 | 51074,NYU,NYU_0051074,2
 707 | 51075,NYU,NYU_0051075,2
 708 | 51076,NYU,NYU_0051076,2
 709 | 51077,NYU,NYU_0051077,2
 710 | 51078,NYU,NYU_0051078,2
 711 | 51079,NYU,NYU_0051079,2
 712 | 51080,NYU,NYU_0051080,2
 713 | 51081,NYU,NYU_0051081,2
 714 | 51082,NYU,NYU_0051082,2
 715 | 51083,NYU,NYU_0051083,2
 716 | 51084,NYU,NYU_0051084,2
 717 | 51085,NYU,NYU_0051085,2
 718 | 51086,NYU,NYU_0051086,2
 719 | 51087,NYU,NYU_0051087,2
 720 | 51088,NYU,NYU_0051088,2
 721 | 51089,NYU,NYU_0051089,2
 722 | 51090,NYU,NYU_0051090,2
 723 | 51091,NYU,NYU_0051091,2
 724 | 51093,NYU,NYU_0051093,2
 725 | 51094,NYU,NYU_0051094,2
 726 | 51095,NYU,NYU_0051095,2
 727 | 51096,NYU,NYU_0051096,2
 728 | 51097,NYU,NYU_0051097,2
 729 | 51098,NYU,NYU_0051098,2
 730 | 51099,NYU,NYU_0051099,2
 731 | 51100,NYU,NYU_0051100,2
 732 | 51101,NYU,NYU_0051101,2
 733 | 51102,NYU,NYU_0051102,2
 734 | 51103,NYU,NYU_0051103,2
 735 | 51104,NYU,NYU_0051104,2
 736 | 51105,NYU,NYU_0051105,2
 737 | 51106,NYU,NYU_0051106,2
 738 | 51107,NYU,NYU_0051107,2
 739 | 51109,NYU,NYU_0051109,2
 740 | 51110,NYU,NYU_0051110,2
 741 | 51111,NYU,NYU_0051111,2
 742 | 51112,NYU,NYU_0051112,2
 743 | 51113,NYU,NYU_0051113,2
 744 | 51114,NYU,NYU_0051114,2
 745 | 51116,NYU,NYU_0051116,2
 746 | 51117,NYU,NYU_0051117,2
 747 | 51118,NYU,NYU_0051118,2
 748 | 51121,NYU,NYU_0051121,2
 749 | 51122,NYU,NYU_0051122,2
 750 | 51123,NYU,NYU_0051123,2
 751 | 51124,NYU,NYU_0051124,2
 752 | 51126,NYU,NYU_0051126,2
 753 | 51127,NYU,NYU_0051127,2
 754 | 51128,NYU,NYU_0051128,2
 755 | 51129,NYU,NYU_0051129,2
 756 | 51130,NYU,NYU_0051130,2
 757 | 51131,NYU,NYU_0051131,2
 758 | 51132,TRINITY,Trinity_0051132,2
 759 | 51133,TRINITY,Trinity_0051133,2
 760 | 51134,TRINITY,Trinity_0051134,2
 761 | 51135,TRINITY,Trinity_0051135,2
 762 | 51136,TRINITY,Trinity_0051136,2
 763 | 51137,TRINITY,Trinity_0051137,2
 764 | 51138,TRINITY,Trinity_0051138,2
 765 | 51139,TRINITY,Trinity_0051139,2
 766 | 51140,TRINITY,Trinity_0051140,2
 767 | 51141,TRINITY,Trinity_0051141,2
 768 | 51142,TRINITY,Trinity_0051142,2
 769 | 51146,NYU,NYU_0051146,2
 770 | 51147,NYU,NYU_0051147,2
 771 | 51148,NYU,NYU_0051148,2
 772 | 51149,NYU,NYU_0051149,2
 773 | 51150,NYU,NYU_0051150,2
 774 | 51151,NYU,NYU_0051151,2
 775 | 51152,NYU,NYU_0051152,2
 776 | 51153,NYU,NYU_0051153,2
 777 | 51154,NYU,NYU_0051154,2
 778 | 51155,NYU,NYU_0051155,2
 779 | 51156,NYU,NYU_0051156,2
 780 | 51159,NYU,NYU_0051159,2
 781 | 51160,STANFORD,Stanford_0051160,1
 782 | 51161,STANFORD,Stanford_0051161,1
 783 | 51162,STANFORD,Stanford_0051162,1
 784 | 51163,STANFORD,Stanford_0051163,1
 785 | 51164,STANFORD,Stanford_0051164,1
 786 | 51165,STANFORD,Stanford_0051165,1
 787 | 51166,STANFORD,Stanford_0051166,1
 788 | 51167,STANFORD,Stanford_0051167,1
 789 | 51168,STANFORD,Stanford_0051168,1
 790 | 51169,STANFORD,Stanford_0051169,1
 791 | 51170,STANFORD,Stanford_0051170,1
 792 | 51171,STANFORD,Stanford_0051171,1
 793 | 51172,STANFORD,Stanford_0051172,1
 794 | 51173,STANFORD,Stanford_0051173,1
 795 | 51174,STANFORD,Stanford_0051174,1
 796 | 51175,STANFORD,Stanford_0051175,1
 797 | 51177,STANFORD,Stanford_0051177,1
 798 | 51178,STANFORD,Stanford_0051178,1
 799 | 51179,STANFORD,Stanford_0051179,1
 800 | 51180,STANFORD,Stanford_0051180,2
 801 | 51181,STANFORD,Stanford_0051181,2
 802 | 51182,STANFORD,Stanford_0051182,2
 803 | 51183,STANFORD,Stanford_0051183,2
 804 | 51184,STANFORD,Stanford_0051184,2
 805 | 51185,STANFORD,Stanford_0051185,2
 806 | 51186,STANFORD,Stanford_0051186,2
 807 | 51187,STANFORD,Stanford_0051187,2
 808 | 51188,STANFORD,Stanford_0051188,2
 809 | 51189,STANFORD,Stanford_0051189,2
 810 | 51190,STANFORD,Stanford_0051190,2
 811 | 51191,STANFORD,Stanford_0051191,2
 812 | 51192,STANFORD,Stanford_0051192,2
 813 | 51193,STANFORD,Stanford_0051193,2
 814 | 51194,STANFORD,Stanford_0051194,2
 815 | 51195,STANFORD,Stanford_0051195,2
 816 | 51196,STANFORD,Stanford_0051196,2
 817 | 51197,STANFORD,Stanford_0051197,2
 818 | 51198,STANFORD,Stanford_0051198,2
 819 | 51199,STANFORD,Stanford_0051199,2
 820 | 51201,UCLA_1,UCLA_1_0051201,1
 821 | 51202,UCLA_1,UCLA_1_0051202,1
 822 | 51203,UCLA_1,UCLA_1_0051203,1
 823 | 51204,UCLA_1,UCLA_1_0051204,1
 824 | 51205,UCLA_1,UCLA_1_0051205,1
 825 | 51206,UCLA_1,UCLA_1_0051206,1
 826 | 51207,UCLA_1,UCLA_1_0051207,1
 827 | 51208,UCLA_1,UCLA_1_0051208,1
 828 | 51209,UCLA_1,UCLA_1_0051209,1
 829 | 51210,UCLA_1,UCLA_1_0051210,1
 830 | 51211,UCLA_1,UCLA_1_0051211,1
 831 | 51212,UCLA_1,UCLA_1_0051212,1
 832 | 51213,UCLA_1,UCLA_1_0051213,1
 833 | 51214,UCLA_1,UCLA_1_0051214,1
 834 | 51215,UCLA_1,UCLA_1_0051215,1
 835 | 51216,UCLA_1,UCLA_1_0051216,1
 836 | 51217,UCLA_1,UCLA_1_0051217,1
 837 | 51218,UCLA_1,UCLA_1_0051218,1
 838 | 51219,UCLA_1,UCLA_1_0051219,1
 839 | 51220,UCLA_1,UCLA_1_0051220,1
 840 | 51221,UCLA_1,UCLA_1_0051221,1
 841 | 51222,UCLA_1,UCLA_1_0051222,1
 842 | 51223,UCLA_1,UCLA_1_0051223,1
 843 | 51224,UCLA_1,UCLA_1_0051224,1
 844 | 51225,UCLA_1,UCLA_1_0051225,1
 845 | 51226,UCLA_1,UCLA_1_0051226,1
 846 | 51227,UCLA_1,UCLA_1_0051227,1
 847 | 51228,UCLA_1,UCLA_1_0051228,1
 848 | 51229,UCLA_1,UCLA_1_0051229,1
 849 | 51230,UCLA_1,UCLA_1_0051230,1
 850 | 51231,UCLA_1,UCLA_1_0051231,1
 851 | 51234,UCLA_1,UCLA_1_0051234,1
 852 | 51235,UCLA_1,UCLA_1_0051235,1
 853 | 51236,UCLA_1,UCLA_1_0051236,1
 854 | 51237,UCLA_1,UCLA_1_0051237,1
 855 | 51238,UCLA_1,UCLA_1_0051238,1
 856 | 51239,UCLA_1,UCLA_1_0051239,1
 857 | 51240,UCLA_1,UCLA_1_0051240,1
 858 | 51241,UCLA_1,UCLA_1_0051241,1
 859 | 51248,UCLA_1,UCLA_1_0051248,1
 860 | 51249,UCLA_1,UCLA_1_0051249,1
 861 | 51250,UCLA_1,UCLA_1_0051250,2
 862 | 51251,UCLA_1,UCLA_1_0051251,2
 863 | 51252,UCLA_1,UCLA_1_0051252,2
 864 | 51253,UCLA_1,UCLA_1_0051253,2
 865 | 51254,UCLA_1,UCLA_1_0051254,2
 866 | 51255,UCLA_1,UCLA_1_0051255,2
 867 | 51256,UCLA_1,UCLA_1_0051256,2
 868 | 51257,UCLA_1,UCLA_1_0051257,2
 869 | 51258,UCLA_1,UCLA_1_0051258,2
 870 | 51260,UCLA_1,UCLA_1_0051260,2
 871 | 51261,UCLA_1,UCLA_1_0051261,2
 872 | 51262,UCLA_1,UCLA_1_0051262,2
 873 | 51263,UCLA_1,UCLA_1_0051263,2
 874 | 51264,UCLA_1,UCLA_1_0051264,2
 875 | 51265,UCLA_1,UCLA_1_0051265,2
 876 | 51266,UCLA_1,UCLA_1_0051266,2
 877 | 51267,UCLA_1,UCLA_1_0051267,2
 878 | 51268,UCLA_1,UCLA_1_0051268,2
 879 | 51269,UCLA_1,UCLA_1_0051269,2
 880 | 51271,UCLA_1,UCLA_1_0051271,2
 881 | 51272,UCLA_1,UCLA_1_0051272,2
 882 | 51273,UCLA_1,UCLA_1_0051273,2
 883 | 51274,UCLA_1,UCLA_1_0051274,2
 884 | 51275,UCLA_1,UCLA_1_0051275,2
 885 | 51276,UCLA_1,UCLA_1_0051276,2
 886 | 51277,UCLA_1,UCLA_1_0051277,2
 887 | 51278,UCLA_1,UCLA_1_0051278,2
 888 | 51279,UCLA_1,UCLA_1_0051279,2
 889 | 51280,UCLA_1,UCLA_1_0051280,2
 890 | 51281,UCLA_1,UCLA_1_0051281,2
 891 | 51282,UCLA_1,UCLA_1_0051282,2
 892 | 51291,UCLA_2,UCLA_2_0051291,1
 893 | 51292,UCLA_2,UCLA_2_0051292,1
 894 | 51293,UCLA_2,UCLA_2_0051293,1
 895 | 51294,UCLA_2,UCLA_2_0051294,1
 896 | 51295,UCLA_2,UCLA_2_0051295,1
 897 | 51296,UCLA_2,UCLA_2_0051296,1
 898 | 51297,UCLA_2,UCLA_2_0051297,1
 899 | 51298,UCLA_2,UCLA_2_0051298,1
 900 | 51299,UCLA_2,UCLA_2_0051299,1
 901 | 51300,UCLA_2,UCLA_2_0051300,1
 902 | 51301,UCLA_2,UCLA_2_0051301,1
 903 | 51302,UCLA_2,UCLA_2_0051302,1
 904 | 51303,UCLA_2,UCLA_2_0051303,2
 905 | 51304,UCLA_2,UCLA_2_0051304,2
 906 | 51305,UCLA_2,UCLA_2_0051305,2
 907 | 51306,UCLA_2,UCLA_2_0051306,2
 908 | 51307,UCLA_2,UCLA_2_0051307,2
 909 | 51308,UCLA_2,UCLA_2_0051308,2
 910 | 51309,UCLA_2,UCLA_2_0051309,2
 911 | 51311,UCLA_2,UCLA_2_0051311,2
 912 | 51312,UCLA_2,UCLA_2_0051312,2
 913 | 51313,UCLA_2,UCLA_2_0051313,2
 914 | 51314,UCLA_2,UCLA_2_0051314,2
 915 | 51315,UCLA_2,UCLA_2_0051315,2
 916 | 51316,UCLA_2,UCLA_2_0051316,2
 917 | 51317,UCLA_2,UCLA_2_0051317,1
 918 | 51318,MAX_MUN,MaxMun_a_0051318,1
 919 | 51319,MAX_MUN,MaxMun_a_0051319,1
 920 | 51320,MAX_MUN,MaxMun_a_0051320,1
 921 | 51321,MAX_MUN,MaxMun_a_0051321,1
 922 | 51322,MAX_MUN,MaxMun_b_0051322,1
 923 | 51323,MAX_MUN,MaxMun_b_0051323,1
 924 | 51324,MAX_MUN,MaxMun_b_0051324,1
 925 | 51325,MAX_MUN,MaxMun_b_0051325,1
 926 | 51326,MAX_MUN,MaxMun_b_0051326,1
 927 | 51327,MAX_MUN,MaxMun_b_0051327,1
 928 | 51328,MAX_MUN,MaxMun_c_0051328,1
 929 | 51329,MAX_MUN,MaxMun_d_0051329,1
 930 | 51330,MAX_MUN,MaxMun_d_0051330,1
 931 | 51331,MAX_MUN,MaxMun_d_0051331,1
 932 | 51332,MAX_MUN,MaxMun_c_0051332,2
 933 | 51333,MAX_MUN,MaxMun_c_0051333,2
 934 | 51334,MAX_MUN,MaxMun_c_0051334,2
 935 | 51335,MAX_MUN,MaxMun_c_0051335,2
 936 | 51336,MAX_MUN,MaxMun_c_0051336,2
 937 | 51338,MAX_MUN,MaxMun_c_0051338,2
 938 | 51339,MAX_MUN,MaxMun_c_0051339,2
 939 | 51340,MAX_MUN,MaxMun_c_0051340,2
 940 | 51341,MAX_MUN,MaxMun_c_0051341,2
 941 | 51342,MAX_MUN,MaxMun_c_0051342,2
 942 | 51343,MAX_MUN,MaxMun_c_0051343,2
 943 | 51344,MAX_MUN,MaxMun_c_0051344,2
 944 | 51345,MAX_MUN,MaxMun_c_0051345,2
 945 | 51346,MAX_MUN,MaxMun_c_0051346,2
 946 | 51347,MAX_MUN,MaxMun_c_0051347,2
 947 | 51348,MAX_MUN,MaxMun_c_0051348,1
 948 | 51349,MAX_MUN,MaxMun_d_0051349,1
 949 | 51350,MAX_MUN,MaxMun_d_0051350,1
 950 | 51351,MAX_MUN,MaxMun_d_0051351,1
 951 | 51352,MAX_MUN,MaxMun_d_0051352,1
 952 | 51353,MAX_MUN,MaxMun_d_0051353,1
 953 | 51354,MAX_MUN,MaxMun_d_0051354,1
 954 | 51355,MAX_MUN,MaxMun_d_0051355,1
 955 | 51356,MAX_MUN,MaxMun_d_0051356,2
 956 | 51357,MAX_MUN,MaxMun_d_0051357,2
 957 | 51358,MAX_MUN,MaxMun_d_0051358,2
 958 | 51359,MAX_MUN,MaxMun_d_0051359,2
 959 | 51360,MAX_MUN,MaxMun_d_0051360,2
 960 | 51361,MAX_MUN,MaxMun_d_0051361,2
 961 | 51362,MAX_MUN,MaxMun_a_0051362,2
 962 | 51363,MAX_MUN,MaxMun_a_0051363,2
 963 | 51364,MAX_MUN,MaxMun_a_0051364,2
 964 | 51365,MAX_MUN,MaxMun_a_0051365,2
 965 | 51369,MAX_MUN,MaxMun_a_0051369,2
 966 | 51370,MAX_MUN,MaxMun_a_0051370,2
 967 | 51373,MAX_MUN,MaxMun_a_0051373,2
 968 | 51456,CALTECH,Caltech_0051456,1
 969 | 51457,CALTECH,Caltech_0051457,1
 970 | 51458,CALTECH,Caltech_0051458,1
 971 | 51459,CALTECH,Caltech_0051459,1
 972 | 51460,CALTECH,Caltech_0051460,1
 973 | 51461,CALTECH,Caltech_0051461,1
 974 | 51462,CALTECH,Caltech_0051462,1
 975 | 51463,CALTECH,Caltech_0051463,1
 976 | 51464,CALTECH,Caltech_0051464,1
 977 | 51465,CALTECH,Caltech_0051465,1
 978 | 51466,CALTECH,Caltech_0051466,1
 979 | 51467,CALTECH,Caltech_0051467,1
 980 | 51468,CALTECH,Caltech_0051468,1
 981 | 51469,CALTECH,Caltech_0051469,1
 982 | 51470,CALTECH,Caltech_0051470,1
 983 | 51471,CALTECH,Caltech_0051471,1
 984 | 51472,CALTECH,Caltech_0051472,1
 985 | 51473,CALTECH,Caltech_0051473,1
 986 | 51474,CALTECH,Caltech_0051474,1
 987 | 51476,CALTECH,Caltech_0051476,2
 988 | 51477,CALTECH,Caltech_0051477,2
 989 | 51478,CALTECH,Caltech_0051478,2
 990 | 51479,CALTECH,Caltech_0051479,2
 991 | 51480,CALTECH,Caltech_0051480,2
 992 | 51481,CALTECH,Caltech_0051481,2
 993 | 51482,CALTECH,Caltech_0051482,2
 994 | 51483,CALTECH,Caltech_0051483,2
 995 | 51484,CALTECH,Caltech_0051484,2
 996 | 51485,CALTECH,Caltech_0051485,2
 997 | 51486,CALTECH,Caltech_0051486,2
 998 | 51487,CALTECH,Caltech_0051487,2
 999 | 51488,CALTECH,Caltech_0051488,2
1000 | 51489,CALTECH,Caltech_0051489,2
1001 | 51490,CALTECH,Caltech_0051490,2
1002 | 51491,CALTECH,Caltech_0051491,2
1003 | 51492,CALTECH,Caltech_0051492,2
1004 | 51493,CALTECH,Caltech_0051493,2
1005 | 51556,SBL,SBL_0051556,2
1006 | 51557,SBL,SBL_0051557,2
1007 | 51558,SBL,SBL_0051558,2
1008 | 51559,SBL,SBL_0051559,2
1009 | 51560,SBL,SBL_0051560,2
1010 | 51561,SBL,SBL_0051561,2
1011 | 51562,SBL,SBL_0051562,2
1012 | 51563,SBL,SBL_0051563,2
1013 | 51564,SBL,SBL_0051564,2
1014 | 51565,SBL,SBL_0051565,2
1015 | 51566,SBL,SBL_0051566,2
1016 | 51567,SBL,SBL_0051567,2
1017 | 51568,SBL,SBL_0051568,2
1018 | 51569,SBL,SBL_0051569,2
1019 | 51570,SBL,SBL_0051570,2
1020 | 51571,SBL,SBL_0051571,1
1021 | 51572,SBL,SBL_0051572,1
1022 | 51573,SBL,SBL_0051573,1
1023 | 51574,SBL,SBL_0051574,1
1024 | 51575,SBL,SBL_0051575,1
1025 | 51576,SBL,SBL_0051576,1
1026 | 51577,SBL,SBL_0051577,1
1027 | 51578,SBL,SBL_0051578,1
1028 | 51579,SBL,SBL_0051579,1
1029 | 51580,SBL,SBL_0051580,1
1030 | 51581,SBL,SBL_0051581,1
1031 | 51582,SBL,SBL_0051582,1
1032 | 51583,SBL,SBL_0051583,1
1033 | 51584,SBL,SBL_0051584,1
1034 | 51585,SBL,SBL_0051585,1
1035 | 51606,MAX_MUN,MaxMun_a_0051606,1
1036 | 51607,MAX_MUN,MaxMun_a_0051607,1
1037 | 


--------------------------------------------------------------------------------