├── .bumpversion.cfg
├── .gitignore
├── .travis.yml
├── Dockerfile
├── LICENSE
├── README.md
├── deepimpute
    ├── __init__.py
    ├── deepImpute.py
    ├── maskedArrays.py
    ├── multinet.py
    ├── parser.py
    └── util.py
├── examples
    ├── notebook_example.ipynb
    └── test.csv
├── makefile
├── setup.py
├── tests
    ├── deepImpute_test.py
    ├── maskedArrays_test.py
    ├── multinet_test.py
    ├── test_data.py
    └── util_test.py
└── tox.ini


/.bumpversion.cfg:
--------------------------------------------------------------------------------
1 | [bumpversion]
2 | files = ./setup.py
3 | commit = True
4 | tag = True
5 | current_version = 1.2
6 | 
7 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | *#*
 2 | *.egg-info*
 3 | dist
 4 | *.pyc
 5 | .coverage
 6 | .vscode
 7 | __pycache__
 8 | htmlcov/
 9 | ignore/
10 | run
11 | .idea/
12 | .cache/
13 | .tox/
14 | .ipynb_checkpoints
15 | ignore/


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: python
2 | python:
3 |   - "3.6"
4 | install:
5 |   - pip install .
6 | script:
7 |   - pytest
8 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM tensorflow/tensorflow
2 | 
3 | MAINTAINER Breck Yunits <byunits@cc.hawaii.edu> 
4 | 
5 | RUN apt-get update && apt-get install -y git
6 | 
7 | RUN git clone https://github.com/lanagarmire/deepimpute && cd deepimpute && pip install --user .


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 Garmire group
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # DeepImpute: an accurate and efficient deep learning method for single-cell RNA-seq data imputation
  2 | 
  3 | [![Build Status](https://travis-ci.org/lanagarmire/deepimpute.svg?branch=master)](https://travis-ci.org/lanagarmire/deepimpute)
  4 | 
  5 | Arisdakessian, Cedric, Olivier Poirion, Breck Yunits, Xun Zhu, and Lana Garmire.  
  6 | "DeepImpute: an accurate, fast and scalable deep neural network method to impute single-cell RNA-Seq data.", *Genome biology* 20.1 (2019): 211"
  7 | https://genomebiology.biomedcentral.com/articles/10.1186/s13059-019-1837-6?fbclid=IwAR2wkwBbp_rQBv0muKEYlt-MDZGlJF6sej1sbKJOP58jvXX1XdD98aGuauo
  8 | 
  9 | DeepImpute has been implemented in Python2 and Python3. The recommended version is Python3.
 10 | 
 11 | ## Getting Started
 12 | 
 13 | These instructions will get you a copy of the project up and running on your local machine.
 14 | 
 15 | ### Installing
 16 | 
 17 | You can install DeepImpute's latest release using pip with the following command:
 18 | 
 19 | ```bash
 20 | pip install deepimpute
 21 | ```
 22 | 
 23 | To install the latest GitHub version, you can also clone this directory and
 24 | install it: 
 25 | 
 26 | ```bash
 27 | git clone https://github.com/lanagarmire/deepimpute
 28 | cd deepimpute
 29 | pip install --user .
 30 | ```
 31 | 
 32 | ### Usage
 33 | 
 34 | DeepImpute can be used either on the command line or as a Python package.
 35 | 
 36 | Command line:
 37 | 
 38 | ```
 39 | usage: deepImpute [-h] [-o OUTPUT] [--cores CORES]
 40 |                   [--cell-axis {rows,columns}] [--limit LIMIT]
 41 |                   [--minVMR MINVMR] [--subset SUBSET]
 42 |                   [--learning-rate LEARNING_RATE] [--batch-size BATCH_SIZE]
 43 |                   [--max-epochs MAX_EPOCHS] [--hidden-neurons HIDDEN_NEURONS]
 44 |                   [--dropout-rate DROPOUT_RATE]
 45 |                   [--output-neurons OUTPUT_NEURONS] [--n_pred N_PRED]
 46 |                   [--policy POLICY]
 47 |                   inputFile
 48 | 
 49 | scRNA-seq data imputation using DeepImpute.
 50 | 
 51 | positional arguments:
 52 |   inputFile             Path to input data.
 53 | 
 54 | optional arguments:
 55 |   -h, --help            show this help message and exit
 56 |   -o OUTPUT, --output OUTPUT
 57 |                         Path to output data counts. Default: ./imputed.csv
 58 |   --cores CORES         Number of cores. Default: all available cores
 59 |   --cell-axis {rows,columns}
 60 |                         Cell dimension in the matrix. Default: rows
 61 |   --limit LIMIT         Genes to impute (e.g. first 2000 genes). Default: auto
 62 |   --minVMR MINVMR       Min Variance over mean ratio for gene exclusion. Gene
 63 |                         with a VMR below ${minVMR} are discarded. Used if
 64 |                         --limit is set to 'auto'. Default: 0.5
 65 |   --subset SUBSET       Cell subset to speed up training. Either a ratio
 66 |                         (0<x<1) or a cell number (int). Default: 1 (all)
 67 |   --learning-rate LEARNING_RATE
 68 |                         Learning rate. Default: 0.0001
 69 |   --batch-size BATCH_SIZE
 70 |                         Batch size. Default: 64
 71 |   --max-epochs MAX_EPOCHS
 72 |                         Maximum number of epochs. Default: 500
 73 |   --hidden-neurons HIDDEN_NEURONS
 74 |                         Number of neurons in the hidden dense layer. Default:
 75 |                         256
 76 |   --dropout-rate DROPOUT_RATE
 77 |                         Dropout rate for the hidden dropout layer (0<rate<1).
 78 |                         Default: 0.2
 79 |   --output-neurons OUTPUT_NEURONS
 80 |                         Number of output neurons per sub-network. Default: 512
 81 |   --n_pred N_PRED       Number of predictors to consider. Consider using this
 82 |                         parameter if your RAM is limited or if you have a high
 83 |                         number of features. Default: All genes with nonzero
 84 |                         VMR
 85 |   --policy POLICY       Whether to restore positive values from the raw
 86 |                         dataset or keep the max between the imputed values and
 87 |                         the raw values. Choices are ['restore', 'max'].
 88 |                         Default: restore
 89 | ```
 90 | 
 91 | Python package:
 92 | 
 93 | ```python
 94 | from deepimpute.multinet import MultiNet
 95 | 
 96 | data = pd.read_csv('examples/test.csv', index_col=0) # dimension = (cells x genes)
 97 | model = MultiNet()
 98 | model.fit(data)
 99 | imputed = model.predict(data)
100 | ```
101 | 
102 | A more detailed usage of deepImpute's functionality is available in the iPython Notebook notebook_example.ipynb
103 | 
104 | ### Running the tests
105 | 
106 | Each file has been validated using a unittest script. They are all available in the test folder.
107 | To run all the tests at once, you can also use the makefile by running `make test`.
108 | 


--------------------------------------------------------------------------------
/deepimpute/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lanagarmire/deepimpute/713191cad69a9da6dedbe49a78282aef3f0930c2/deepimpute/__init__.py


--------------------------------------------------------------------------------
/deepimpute/deepImpute.py:
--------------------------------------------------------------------------------
 1 | import pandas as pd
 2 | 
 3 | from deepimpute.parser import parse_args
 4 | from deepimpute.multinet import MultiNet
 5 | 
 6 | def deepImpute(**kwargs):
 7 | 
 8 |     args = parse_args()
 9 | 
10 |     for key, value in kwargs.items():
11 |         setattr(args, key, value)
12 |     
13 |     data = pd.read_csv(args.inputFile, index_col=0)
14 | 
15 |     if args.cell_axis == "columns":
16 |         data = data.T
17 | 
18 |     NN_params = {
19 |         'learning_rate': args.learning_rate,
20 |         'batch_size': args.batch_size,
21 |         'max_epochs': args.max_epochs,
22 |         'ncores': args.cores,
23 |         'sub_outputdim': args.output_neurons,
24 |         'architecture': [
25 |             {"type": "dense", "activation": "relu", "neurons": args.hidden_neurons},
26 |             {"type": "dropout", "activation": "dropout", "rate": args.dropout_rate}]
27 |     }
28 | 
29 |     multi = MultiNet(**NN_params)
30 |     multi.fit(data, NN_lim=args.limit, cell_subset=args.subset, minVMR=args.minVMR, n_pred=args.n_pred)
31 | 
32 |     imputed = multi.predict(data, imputed_only=False, policy=args.policy)
33 | 
34 |     if args.output is not None:
35 |         imputed.to_csv(args.output)
36 |     else:
37 |         return imputed
38 | 
39 | if __name__ == "__main__":
40 |     deepImpute()
41 | 


--------------------------------------------------------------------------------
/deepimpute/maskedArrays.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | from scipy.stats import expon
  3 | 
  4 | 
  5 | class MaskedArray(object):
  6 | 
  7 |     def __init__(self, data=None, mask=None, distr="exp", dropout=0.01, seed=1):
  8 |         self.data = np.array(data)
  9 |         self._binMask = np.array(mask)
 10 |         self.shape = data.shape
 11 |         self.distr = distr
 12 |         self.dropout = dropout
 13 |         self.seed = seed
 14 | 
 15 |     @property
 16 |     def binMask(self):
 17 |         return self._binMask
 18 | 
 19 |     @binMask.setter
 20 |     def binMask(self, value):
 21 |         self._binMask = value.astype(bool)
 22 | 
 23 |     def getMaskedMatrix(self):
 24 |         maskedMatrix = self.data.copy()
 25 |         maskedMatrix[~self.binMask] = 0
 26 |         return maskedMatrix
 27 | 
 28 |     def getMasked(self, rows=True):
 29 |         """ Generator for row or column mask """
 30 |         compt = 0
 31 |         if rows:
 32 |             while compt < self.shape[0]:
 33 |                 yield [
 34 |                     self.data[compt, idx]
 35 |                     for idx in range(self.shape[1])
 36 |                     if not self.binMask[compt, idx]
 37 |                 ]
 38 |                 compt += 1
 39 |         else:
 40 |             while compt < self.shape[1]:
 41 |                 yield [
 42 |                     self.data[idx, compt]
 43 |                     for idx in range(self.shape[0])
 44 |                     if not self.binMask[idx, compt]
 45 |                 ]
 46 |                 compt += 1
 47 | 
 48 |     def getMasked_flat(self):
 49 |         return self.data[~self.binMask]
 50 | 
 51 |     def copy(self):
 52 |         args = {"data": self.data.copy(), "mask": self.binMask.copy()}
 53 |         return MaskedArray(**args)
 54 | 
 55 |     def get_probs(self, vec):
 56 |         return {
 57 |             "exp": expon.pdf(vec, 0, 20),
 58 |             "uniform": np.tile([1. / len(vec)], len(vec)),
 59 |         }.get(self.distr)
 60 | 
 61 |     def get_Nmasked(self, idx):
 62 |         cells_g = self.data[:, idx]
 63 |         dp_i = (1 + (cells_g == 0).sum() * 1.) / self.shape[0]
 64 |         dp_f = np.exp(-2 * np.log10(cells_g.mean()) ** 2)
 65 |         return 1 + int((cells_g == 0).sum() * dp_f / dp_i)
 66 | 
 67 |     # def generate_sim(self):
 68 |     #     np.random.seed(self.seed)
 69 |     #     self.binMask = np.ones(self.shape).astype(bool)
 70 | 
 71 |     #     for g in range(self.shape[1]):
 72 |     #         cells_g = self.data[:,g]
 73 |     #         # Retrieve indices of positive values
 74 |     #         ind_pos = np.arange(self.shape[0])[cells_g>0]
 75 |     #         cells_g_pos = cells_g[ind_pos]
 76 |     #         # Get masking probability of each value
 77 |     #         if cells_g_pos.size > 5:
 78 |     #             probs = self.get_probs(cells_g_pos)
 79 |     #             n_masked = self.get_Nmasked(g)
 80 |     #             if n_masked >= cells_g_pos.size:
 81 |     #                 print('Warning: too many cells masked for gene {} ({}/{})'.
 82 |     #                       format(g,n_masked,cells_g_pos.size))
 83 |     #                 n_masked = 1+int(0.5*cells_g_pos.size)
 84 |     #             masked_idx = np.random.choice(cells_g_pos.size,n_masked,
 85 |     #                                           p=probs/probs.sum(),
 86 |     #                                           replace=False)
 87 |     #             self.binMask[ind_pos[sorted(masked_idx)],g] = False
 88 | 
 89 |     def generate(self):
 90 |         np.random.seed(self.seed)
 91 |         self.binMask = np.ones(self.shape).astype(bool)
 92 | 
 93 |         for c in range(self.shape[0]):
 94 |             cells_c = self.data[c, :]
 95 |             # Retrieve indices of positive values
 96 |             ind_pos = np.arange(self.shape[1])[cells_c > 0]
 97 |             cells_c_pos = cells_c[ind_pos]
 98 |             # Get masking probability of each value
 99 | 
100 |             if cells_c_pos.size > 5:
101 |                 probs = self.get_probs(cells_c_pos)
102 |                 n_masked = 1 + int(self.dropout * len(cells_c_pos))
103 |                 if n_masked >= cells_c_pos.size:
104 |                     print(
105 |                         "Warning: too many cells masked for gene {} ({}/{})".format(
106 |                             c, n_masked, cells_c_pos.size
107 |                         )
108 |                     )
109 |                     n_masked = 1 + int(0.5 * cells_c_pos.size)
110 | 
111 |                 masked_idx = np.random.choice(
112 |                     cells_c_pos.size, n_masked, p=probs / probs.sum(), replace=False
113 |                 )
114 |                 self.binMask[c, ind_pos[sorted(masked_idx)]] = False
115 | 


--------------------------------------------------------------------------------
/deepimpute/multinet.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import warnings
  3 | import tempfile
  4 | 
  5 | import pandas as pd
  6 | import numpy as np
  7 | from scipy.stats import pearsonr
  8 | 
  9 | import tensorflow.keras as keras
 10 | from keras import backend as K
 11 | from keras.models import Model,model_from_json
 12 | from keras.layers import Dense,Dropout,Input
 13 | from keras.callbacks import EarlyStopping
 14 | import keras.losses
 15 | 
 16 | import tensorflow as tf
 17 | 
 18 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 19 | 
 20 | def get_distance_matrix(raw, n_pred=None):
 21 | 
 22 |     VMR = raw.std() / raw.mean()
 23 |     VMR[np.isinf(VMR)] = 0
 24 |     
 25 |     if n_pred is None:
 26 |         potential_pred = raw.columns[VMR > 0]
 27 |     else:
 28 |         print("Using {} predictors".format(n_pred))
 29 |         potential_pred = VMR.sort_values(ascending=False).index[:n_pred]
 30 | 
 31 |     covariance_matrix = pd.DataFrame(np.abs(np.corrcoef(raw.T.loc[potential_pred])),
 32 |                                      index=potential_pred,
 33 |                                      columns=potential_pred).fillna(0)
 34 |     return covariance_matrix
 35 | 
 36 | def wMSE(y_true, y_pred, binary=False):
 37 |     if binary:
 38 |         weights = tf.cast(y_true>0, tf.float32)
 39 |     else:
 40 |         weights = y_true
 41 |     return tf.reduce_mean(weights*tf.square(y_true-y_pred))
 42 | 
 43 | def inspect_data(data):
 44 |     # Check if there area any duplicated cell/gene labels
 45 |     
 46 |     if sum(data.index.duplicated()):
 47 |         print("ERROR: duplicated cell labels. Please provide unique cell labels.")
 48 |         exit(1)
 49 |         
 50 |     if sum(data.columns.duplicated()):
 51 |         print("ERROR: duplicated gene labels. Please provide unique gene labels.")
 52 |         exit(1)
 53 |         
 54 |     max_value = np.max(data.values)
 55 |     if max_value < 10:
 56 |         print("ERROR: max value = {}. Is your data log-transformed? Please provide raw counts"
 57 |               .format(max_value))
 58 |         exit(1)
 59 |         
 60 |     print("Input dataset is {} cells (rows) and {} genes (columns)"
 61 |           .format(*data.shape))
 62 |     print("First 3 rows and columns:")
 63 |     print(data.iloc[:3,:3])
 64 | 
 65 | class MultiNet:
 66 | 
 67 |     def __init__(self,
 68 |                  learning_rate=1e-4,
 69 |                  batch_size=64,
 70 |                  max_epochs=500,
 71 |                  patience=5,
 72 |                  ncores=-1,
 73 |                  loss="wMSE",
 74 |                  output_prefix=tempfile.mkdtemp(),
 75 |                  sub_outputdim=512,
 76 |                  verbose=1,
 77 |                  seed=1234,
 78 |                  architecture=None
 79 |     ):
 80 |         self.NN_parameters = {"learning_rate": learning_rate,
 81 |                               "batch_size": batch_size,
 82 |                               "loss": loss,
 83 |                               "architecture": architecture,
 84 |                               "max_epochs": max_epochs,
 85 |                               "patience": patience}
 86 |         self.sub_outputdim = sub_outputdim
 87 |         self.outputdir = output_prefix
 88 |         self.verbose = verbose
 89 |         self.seed = seed
 90 |         self.setCores(ncores)
 91 | 
 92 |     def setCores(self, ncores):
 93 |         if ncores > 0:
 94 |             self.ncores = ncores
 95 |         else:
 96 |             self.ncores = os.cpu_count()
 97 |             print("Using all the cores ({})".format(self.ncores))
 98 |  
 99 |     def loadDefaultArchitecture(self):
100 |         self.NN_parameters['architecture'] = [
101 |                 {"type": "dense", "neurons": self.sub_outputdim//2, "activation": "relu"},
102 |                 {"type": "dropout", "rate": 0.2},
103 |             ]
104 |         
105 |     def save(self, model):
106 |         os.system("mkdir -p {}".format(self.outputdir))
107 |         
108 |         model_json = model.to_json()
109 |                 
110 |         with open("{}/model.json".format(self.outputdir), "w") as json_file:
111 |             json_file.write(model_json)
112 |             
113 |         # serialize weights to HDF5
114 |         model.save_weights("{}/model.h5".format(self.outputdir))
115 |         print("Saved model to disk in {}".format(self.outputdir))
116 | 
117 |     def load(self):
118 |         json_file = open('{}/model.json'.format(self.outputdir), 'r')
119 |         loaded_model_json = json_file.read()
120 |         json_file.close()
121 |         model = model_from_json(loaded_model_json)
122 |         model.load_weights('{}/model.h5'.format(self.outputdir))
123 | 
124 |         return model
125 |         
126 |     def build(self, inputdims):
127 |         if self.NN_parameters['architecture'] is None:
128 |             self.loadDefaultArchitecture()
129 | 
130 |         print(self.NN_parameters['architecture'])
131 | 
132 |         inputs = [ Input(shape=(inputdim,)) for inputdim in inputdims ]
133 |         outputs = inputs
134 | 
135 |         for layer in self.NN_parameters['architecture']:
136 |             if layer['type'].lower() == 'dense':
137 |                 outputs = [ Dense(layer['neurons'], activation=layer['activation'])(output)
138 |                             for output in outputs ]
139 |             elif layer['type'].lower() == 'dropout':
140 |                 outputs = [ Dropout(layer['rate'], seed=self.seed)(output)
141 |                             for output in outputs] 
142 |             else:
143 |                 print("Unknown layer type.")
144 | 
145 |         outputs = [Dense(self.sub_outputdim, activation="softplus")(output)
146 |                    for output in outputs]
147 |                 
148 |         model = Model(inputs=inputs, outputs=outputs)
149 | 
150 |         loss = self.NN_parameters['loss']
151 | 
152 |         if loss in [k for k, v in globals().items() if callable(v)]:
153 |             # if loss is a defined function
154 |             loss = eval(self.NN_parameters['loss'])
155 |             
156 |         if not callable(loss):
157 |             # it is defined in Keras
158 |             if hasattr(keras.losses, loss):
159 |                 loss = getattr(keras.losses, loss)                
160 |             else:
161 |                 print('Unknown loss: {}. Aborting.'.format(loss))
162 |                 exit(1)
163 | 
164 |         model.compile(optimizer=keras.optimizers.Adam(lr=self.NN_parameters['learning_rate']),
165 |                       loss=loss)
166 | 
167 |         return model
168 | 
169 |     def fit(self,
170 |             raw,
171 |             cell_subset=1,
172 |             NN_lim=None,
173 |             genes_to_impute=None,
174 |             n_pred=None,
175 |             ntop=5,
176 |             minVMR=0.5,
177 |             mode='random',
178 |     ):
179 |         
180 |         inspect_data(raw)
181 |         
182 |         if self.seed is not None:
183 |             np.random.seed(self.seed)
184 | 
185 |         if cell_subset != 1:
186 |             if cell_subset < 1:
187 |                 raw = raw.sample(frac=cell_subset)
188 |             else:
189 |                 raw = raw.sample(cell_subset)
190 | 
191 |         gene_metric = (raw.var()/(1+raw.mean())).sort_values(ascending=False)
192 |         gene_metric = gene_metric[gene_metric > 0]
193 | 
194 |         if genes_to_impute is None:
195 |             genes_to_impute = self.filter_genes(gene_metric, minVMR, NN_lim=NN_lim)
196 |         else:
197 |             # Make the number of genes to impute a multiple of the network output dim
198 |             n_genes = len(genes_to_impute)
199 |             if n_genes % self.sub_outputdim != 0:
200 |                 print("The number of input genes is not a multiple of {}. Filling with other genes.".format(n_genes))
201 |                 fill_genes = gene_metric.index[:self.sub_outputdim-n_genes]
202 | 
203 |                 if len(fill_genes) < self.sub_outputdim-n_genes:
204 |                     # Not enough genes in gene_metric. Sample with replacement
205 |                     rest = self.sub_outputdim - n_genes - len(fill_genes)
206 |                     fill_genes = np.concatenate([fill_genes,
207 |                                                  np.random.choice(gene_metric.index, rest, replace=True)])
208 | 
209 |                 genes_to_impute = np.concatenate([genes_to_impute, fill_genes])
210 | 
211 |         covariance_matrix = get_distance_matrix(raw, n_pred=n_pred)
212 | 
213 |         self.setTargets(raw.reindex(columns=genes_to_impute), mode=mode)
214 |         self.setPredictors(covariance_matrix, ntop=ntop)
215 | 
216 |         print("Normalization")
217 |         norm_data = np.log1p(raw).astype(np.float32) # normalizer.transform(raw)
218 | 
219 |         np.random.seed(self.seed)
220 |         tf.random.set_seed(self.seed)
221 |         
222 |         tf.config.threading.set_inter_op_parallelism_threads(self.ncores)
223 |         tf.config.threading.set_intra_op_parallelism_threads(self.ncores)
224 | 
225 |         print("Building network")
226 |         model = self.build([len(genes) for genes in self.predictors])
227 | 
228 |         test_cells = np.random.choice(norm_data.index, int(0.05 * norm_data.shape[0]), replace=False)
229 |         train_cells = np.setdiff1d(norm_data.index, test_cells)
230 | 
231 |         X_train = [norm_data.loc[train_cells, inputgenes].values for inputgenes in self.predictors]
232 |         Y_train = [norm_data.loc[train_cells, targetgenes].values for targetgenes in self.targets]
233 |         
234 |         X_test = [norm_data.loc[test_cells, inputgenes].values for inputgenes in self.predictors]
235 |         Y_test = [norm_data.loc[test_cells, targetgenes].values for targetgenes in self.targets]
236 | 
237 |         print("Fitting with {} cells".format(norm_data.shape[0]))
238 |         result = model.fit(X_train, Y_train,
239 |                            validation_data=(X_test,Y_test),
240 |                            epochs=self.NN_parameters["max_epochs"],
241 |                            batch_size=self.NN_parameters["batch_size"],
242 |                            callbacks=[EarlyStopping(monitor='val_loss',
243 |                                                     patience=self.NN_parameters["patience"])],
244 |                            verbose=self.verbose)
245 | 
246 |         self.trained_epochs = len(result.history['loss'])
247 |         print("Stopped fitting after {} epochs".format(self.trained_epochs))
248 | 
249 |         self.save(model)
250 | 
251 |         # Save some metrics on test data
252 |         Y_test_raw = np.hstack(Y_test).flatten()
253 |         Y_test_imputed = np.hstack(model.predict(X_test)).flatten()
254 | 
255 |         # Keep only positive values (since negative values could be dropouts)
256 |         Y_test_imputed = Y_test_imputed[Y_test_raw>0]
257 |         Y_test_raw = Y_test_raw[Y_test_raw>0]
258 | 
259 |         self.test_metrics = {
260 |             'correlation': pearsonr(Y_test_raw,Y_test_imputed)[0],
261 |             'MSE': np.sum((Y_test_raw-Y_test_imputed)**2)/len(Y_test_raw)
262 |         }        
263 | 
264 |         return self
265 | 
266 |     def predict(self,
267 |                 raw,
268 |                 imputed_only=False,
269 |                 policy="restore"):
270 | 
271 |         norm_raw = np.log1p(raw)
272 | 
273 |         inputs = [ norm_raw.loc[:,predictors].values.astype(np.float32)
274 |                    for predictors in self.predictors ]
275 | 
276 |         model = self.load()
277 | 
278 |         predicted = model.predict(inputs)
279 |         if len(inputs)>1:
280 |             predicted = np.hstack(predicted)
281 |         
282 |         predicted = pd.DataFrame(predicted, index=raw.index, columns=self.targets.flatten())
283 | 
284 |         predicted = predicted.groupby(by=predicted.columns, axis=1).mean()
285 |         not_predicted = norm_raw.drop(self.targets.flatten(), axis=1)
286 | 
287 |         imputed = (pd.concat([predicted,not_predicted],axis=1)
288 |                    .loc[raw.index, raw.columns]
289 |                    .values)
290 |         
291 |         # To prevent overflow
292 |         imputed[ (imputed > 2*norm_raw.values.max()) | (np.isnan(imputed)) ] = 0
293 |         # Convert back to counts
294 |         imputed = np.expm1(imputed)
295 | 
296 |         if policy == "restore":
297 |             print("Filling zeros")
298 |             mask = (raw.values > 0)
299 |             imputed[mask] = raw.values[mask]
300 |         elif policy == "max":
301 |             print("Imputing data with 'max' policy")
302 |             mask = (raw.values > imputed)
303 |             imputed[mask] = raw.values[mask]
304 | 
305 |         imputed = pd.DataFrame(imputed, index=raw.index, columns=raw.columns)
306 | 
307 |         if imputed_only:
308 |             return imputed.loc[:, predicted.columns]
309 |         else:
310 |             return imputed
311 |         
312 |     def filter_genes(self,
313 |                     gene_metric, # assumes gene_metric is sorted
314 |                     threshold,
315 |                     NN_lim=None
316 |     ):
317 |         if not str(NN_lim).isdigit():
318 |             NN_lim = (gene_metric > threshold).sum()
319 | 
320 |         n_subsets = int(np.ceil(NN_lim / self.sub_outputdim))
321 |         genes_to_impute = gene_metric.index[:n_subsets*self.sub_outputdim]
322 | 
323 |         rest = self.sub_outputdim - (len(genes_to_impute) % self.sub_outputdim)
324 | 
325 |         if rest > 0:
326 |             fill_genes = np.random.choice(gene_metric.index, rest)
327 |             genes_to_impute = np.concatenate([genes_to_impute, fill_genes])
328 | 
329 |         print("{} genes selected for imputation".format(len(genes_to_impute)))
330 | 
331 |         return genes_to_impute
332 | 
333 |     def setTargets(self,data, mode='random'):
334 |         
335 |         n_subsets = int(data.shape[1]/self.sub_outputdim)
336 | 
337 |         if mode == 'progressive':
338 |             self.targets = data.columns.values.reshape([n_subsets, self.sub_outputdim])
339 |         else:
340 |             self.targets = np.random.choice(data.columns,
341 |                                             [n_subsets, self.sub_outputdim],
342 |                                             replace=False)
343 |         
344 |     def setPredictors(self, covariance_matrix, ntop=5):
345 |         self.predictors = []
346 |         
347 |         for i,targets in enumerate(self.targets):
348 | 
349 |             genes_not_in_target = np.setdiff1d(covariance_matrix.columns, targets)
350 | 
351 |             if genes_not_in_target.size == 0:
352 |                 warnings.warn('Warning: number of target genes lower than output dim. Consider lowering down the sub_outputdim parameter',
353 |                               UserWarning)
354 |                 genes_not_in_target = covariance_matrix.columns
355 |             
356 |             subMatrix = ( covariance_matrix
357 |                           .loc[targets, genes_not_in_target]
358 |                           )
359 |             sorted_idx = np.argsort(-subMatrix.values, axis=1)
360 |             predictors = subMatrix.columns[sorted_idx[:,:ntop].flatten()]
361 | 
362 |             self.predictors.append(predictors.unique())
363 | 
364 |             print("Net {}: {} predictors, {} targets"
365 |                   .format(i,len(np.unique(predictors)),len(targets)))
366 | 
367 |     def score(self, data, policy=None):
368 |         warnings.warn(
369 |             "This method is deprecated. Please use model.test_metrics to measure model accuracy instead",
370 |             DeprecationWarning)
371 |         Y_hat = self.predict(data, policy=policy)
372 |         Y = data.loc[Y_hat.index, Y_hat.columns]
373 | 
374 |         return pearsonr(Y_hat.values.reshape(-1), Y.values.reshape(-1))
375 |         
376 | 
377 | 
378 | 
379 |     
380 | 


--------------------------------------------------------------------------------
/deepimpute/parser.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | 
 3 | def parse_args():
 4 |     parser = argparse.ArgumentParser(
 5 |         description="scRNA-seq data imputation using DeepImpute."
 6 |     )
 7 |     parser.add_argument("inputFile", type=str, help="Path to input data.")
 8 |     parser.add_argument(
 9 |         "-o", "--output",
10 |         type=str,
11 |         default="./imputed.csv",
12 |         help="Path to output data counts. Default: ./imputed.csv",
13 |     )
14 |     parser.add_argument(
15 |         "--cores", type=int, default=-1, help="Number of cores. Default: all available cores"
16 |     )
17 |     parser.add_argument(
18 |         "--cell-axis",
19 |         type=str,
20 |         choices=["rows", "columns"],
21 |         default="rows",
22 |         help="Cell dimension in the matrix. Default: rows",
23 |     )
24 |     parser.add_argument(
25 |         "--limit",
26 |         type=str,
27 |         default="auto",
28 |         help="Genes to impute (e.g. first 2000 genes). Default: auto",
29 |     )
30 |     parser.add_argument(
31 |         "--minVMR",
32 |         type=float,
33 |         default="0.5",
34 |         help="Min Variance over mean ratio for gene exclusion. Gene with a VMR below ${minVMR} are discarded. Used if --limit is set to 'auto'. Default: 0.5",
35 |     )
36 |     parser.add_argument(
37 |         "--subset",
38 |         type=float,
39 |         default=1,
40 |         help="Cell subset to speed up training. \
41 |                         Either a ratio (0<x<1) or a cell number (int). Default: 1 (all)",
42 |     )
43 |     parser.add_argument(
44 |         "--learning-rate",
45 |         type=float,
46 |         default=0.0005,
47 |         help="Learning rate. Default: 0.0001"
48 |     )
49 |     parser.add_argument(
50 |         "--batch-size",
51 |         type=int,
52 |         default=64,
53 |         help="Batch size. Default: 64"
54 |     )
55 |     parser.add_argument(
56 |         "--max-epochs",
57 |         type=int,
58 |         default=300,
59 |         help="Maximum number of epochs. Default: 500"
60 |     )
61 |     parser.add_argument(
62 |         "--hidden-neurons",
63 |         type=int,
64 |         default=300,
65 |         help="Number of neurons in the hidden dense layer. Default: 256"
66 |     )
67 |     parser.add_argument(
68 |         "--dropout-rate",
69 |         type=float,
70 |         default=0.2,
71 |         help="Dropout rate for the hidden dropout layer (0<rate<1). Default: 0.2"
72 |     )
73 |     parser.add_argument(
74 |         "--output-neurons",
75 |         type=int,
76 |         default=512,
77 |         help="Number of output neurons per sub-network. Default: 512"
78 |     )
79 |     parser.add_argument(
80 |         "--n_pred",
81 |         type=int,
82 |         default=None,
83 |         help="Number of predictors to consider. Consider using this parameter if your RAM is limited or if you have a high number of features. Default: All genes with nonzero VMR"
84 |     )
85 | 
86 |     parser.add_argument(
87 |         "--policy",
88 |         type=str,
89 |         default='restore',
90 |         help="Whether to restore positive values from the raw dataset or keep the max between the imputed values and the raw values. Choices are ['restore', 'max']. Default: restore"
91 |     )
92 | 
93 |     args = parser.parse_args()
94 | 
95 |     return args
96 | 
97 | 


--------------------------------------------------------------------------------
/deepimpute/util.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import pandas as pd
  3 | 
  4 | from deepimpute.maskedArrays import MaskedArray
  5 | 
  6 | """ Preprocessing functions """
  7 | 
  8 | 
  9 | def log1x(x):
 10 |     return np.log(1 + x)
 11 | 
 12 | 
 13 | def exp1x(x):
 14 |     return np.exp(x) - 1
 15 | 
 16 | 
 17 | def libNorm(scale=10000):
 18 |     def _libNorm(x):
 19 |         return scale / np.sum(x)
 20 |     return _libNorm
 21 | 
 22 | def set_int(name):
 23 | 
 24 |     def setter_wrapper(self, value):
 25 |         if type(np.prod(value)) is not np.int64:
 26 |             print("Wrong value for {}={}. Converting to integer.".format(name, value))
 27 |             if np.array(value).size == 1:
 28 |                 setattr(self, name, int(value))
 29 |             else:
 30 |                 setattr(self, name, [el for el in map(int, list(value))])
 31 |         else:
 32 |             setattr(self, name, value)
 33 | 
 34 |     return setter_wrapper
 35 | 
 36 | 
 37 | def get_int(name):
 38 | 
 39 |     def getter_wrapper(self):
 40 |         out = getattr(self, name)
 41 |         if np.array(out).size == 1:
 42 |             return int(out)
 43 |         else:
 44 |             return [el for el in map(int, out)]
 45 | 
 46 |     return getter_wrapper
 47 | 
 48 | # def get_input_genes(
 49 | #         dataframeToImpute, dims, distanceMatrix=None, targets=None, seed=1234 #predictorDropoutLimit=.99,seed=1234
 50 | # ):
 51 | #     #geneDropoutRate = (dataframeToImpute==0).mean()
 52 | #     potential_predictors = dataframeToImpute.columns #geneDropoutRate.index[geneDropoutRate < predictorDropoutLimit]
 53 | 
 54 | #     print("Keeping {} potential predictors.".format(len(potential_predictors)))
 55 |     
 56 | #     if targets is None:
 57 | #         np.random.seed(seed)
 58 | #         targets = [np.random.choice(dataframeToImpute.columns, dims[1], replace=False)]
 59 | 
 60 | #     if distanceMatrix is None:
 61 | #         distanceMatrix = pd.DataFrame(
 62 | #             np.abs(np.corrcoef(dataframeToImpute.T)),
 63 | #             index=dataframeToImpute.columns, columns=dataframeToImpute.columns
 64 | #         )[potential_predictors]
 65 | #     in_out_genes = []
 66 | 
 67 | #     max_limit = dims[0]
 68 | #     for genes in targets:
 69 | #         pred_to_rmv = np.setdiff1d(potential_predictors,targets)
 70 | #         subMatrix = distanceMatrix.loc[genes].drop(pred_to_rmv,axis=1)
 71 | #         sorted_idx = np.argsort(-subMatrix.values,axis=1)
 72 | #         predictorGenes = subMatrix.columns[sorted_idx[:,:max_limit]].values.flatten()        
 73 | #         in_out_genes.append((predictorGenes, genes))
 74 | #     return in_out_genes
 75 | 
 76 | 
 77 | # def _get_target_genes(geneQuantiles, minExpressionLevel, maxNumOfGenes):
 78 | #     if maxNumOfGenes == "auto":
 79 | #         targetGenes = geneQuantiles[geneQuantiles > minExpressionLevel].index
 80 | #         print("Minimum gene count for imputation set to {}, leaving {} genes for imputation."
 81 | #               .format(minExpressionLevel,len(targetGenes)))
 82 | 
 83 | #     else:
 84 | #         if maxNumOfGenes is None:
 85 | #             maxNumOfGenes = len(geneQuantiles)
 86 | #         maxNumOfGenes = min(maxNumOfGenes, len(geneQuantiles))
 87 | #         targetGenes = geneQuantiles.sort_values(ascending=False).index[:maxNumOfGenes]
 88 | #         print("Gene prediction limit set to {} genes".format(len(targetGenes)))
 89 | 
 90 | #     return targetGenes.tolist()
 91 | 
 92 | 
 93 | def score_model(model, data, metric, cols=None):
 94 |     # Create masked array
 95 |     if cols is None:
 96 |         cols = data.columns
 97 | 
 98 |     maskedData = MaskedArray(data=data)
 99 |     maskedData.generate()
100 |     maskedDf = pd.DataFrame(
101 |         maskedData.getMaskedMatrix(), index=data.index, columns=data.columns
102 |     )
103 |     # Predict
104 |     # model.fit(maskedDf)
105 |     imputed = model.predict(maskedDf)
106 | 
107 |     imputedGenes = np.intersect1d(cols, imputed.columns)
108 | 
109 |     # Compare imputed masked array and input
110 |     maskedIdx = maskedDf[imputedGenes].values != data[imputedGenes].values
111 |     score_res = metric(
112 |         data[imputedGenes].values[maskedIdx], imputed[imputedGenes].values[maskedIdx]
113 |     )
114 |     return score_res
115 | 
116 | # def wMSE(y_true,y_pred):
117 | #     return tf.reduce_mean(y_true*tf.square(y_true-y_pred))
118 | 


--------------------------------------------------------------------------------
/examples/notebook_example.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Single-cell RNA-seq imputation using DeepImpute\n",
  8 |     "\n",
  9 |     "Here is a comprehensive tutorial to understand the functionnalities of DeepImpute."
 10 |    ]
 11 |   },
 12 |   {
 13 |    "cell_type": "code",
 14 |    "execution_count": null,
 15 |    "metadata": {},
 16 |    "outputs": [],
 17 |    "source": [
 18 |     "from deepimpute.multinet import MultiNet\n",
 19 |     "import pandas as pd\n",
 20 |     "\n",
 21 |     "# Load dataset using pandas\n",
 22 |     "data = pd.read_csv('test.csv',index_col=0)\n",
 23 |     "print('Working on {} cells and {} genes'.format(*data.shape))"
 24 |    ]
 25 |   },
 26 |   {
 27 |    "cell_type": "markdown",
 28 |    "metadata": {},
 29 |    "source": [
 30 |     "## Create a DeepImpute multinet"
 31 |    ]
 32 |   },
 33 |   {
 34 |    "cell_type": "code",
 35 |    "execution_count": null,
 36 |    "metadata": {},
 37 |    "outputs": [],
 38 |    "source": [
 39 |     "# Using default parameters\n",
 40 |     "multinet = MultiNet() "
 41 |    ]
 42 |   },
 43 |   {
 44 |    "cell_type": "code",
 45 |    "execution_count": null,
 46 |    "metadata": {},
 47 |    "outputs": [],
 48 |    "source": [
 49 |     "# Using custom parameters\n",
 50 |     "NN_params = {\n",
 51 |     "        'learning_rate': 1e-4,\n",
 52 |     "        'batch_size': 64,\n",
 53 |     "        'max_epochs': 200,\n",
 54 |     "        'ncores': 5,\n",
 55 |     "        'sub_outputdim': 512,\n",
 56 |     "        'architecture': [\n",
 57 |     "            {\"type\": \"dense\", \"activation\": \"relu\", \"neurons\": 200},\n",
 58 |     "            {\"type\": \"dropout\", \"activation\": \"dropout\", \"rate\": 0.3}]\n",
 59 |     "    }\n",
 60 |     "\n",
 61 |     "multinet = MultiNet(**NN_params)"
 62 |    ]
 63 |   },
 64 |   {
 65 |    "cell_type": "markdown",
 66 |    "metadata": {},
 67 |    "source": [
 68 |     "## Fit the networks"
 69 |    ]
 70 |   },
 71 |   {
 72 |    "cell_type": "code",
 73 |    "execution_count": null,
 74 |    "metadata": {},
 75 |    "outputs": [],
 76 |    "source": [
 77 |     "# Using all the data\n",
 78 |     "multinet.fit(data,cell_subset=1,minVMR=0.5)"
 79 |    ]
 80 |   },
 81 |   {
 82 |    "cell_type": "code",
 83 |    "execution_count": null,
 84 |    "metadata": {},
 85 |    "outputs": [],
 86 |    "source": [
 87 |     "# Using 80% of the data\n",
 88 |     "multinet.fit(data,cell_subset=0.5)"
 89 |    ]
 90 |   },
 91 |   {
 92 |    "cell_type": "code",
 93 |    "execution_count": null,
 94 |    "metadata": {},
 95 |    "outputs": [],
 96 |    "source": [
 97 |     "# Using 200 cells (randomly selected)\n",
 98 |     "multinet.fit(data,cell_subset=200)"
 99 |    ]
100 |   },
101 |   {
102 |    "cell_type": "code",
103 |    "execution_count": null,
104 |    "metadata": {},
105 |    "outputs": [],
106 |    "source": [
107 |     "# Custom fit\n",
108 |     "trainingData = data.iloc[100:250,:]\n",
109 |     "multinet.fit(trainingData)"
110 |    ]
111 |   },
112 |   {
113 |    "cell_type": "markdown",
114 |    "metadata": {},
115 |    "source": [
116 |     "## Imputation\n",
117 |     "\n",
118 |     "The imputation can be done on any dataset as long as the gene labels are the same"
119 |    ]
120 |   },
121 |   {
122 |    "cell_type": "code",
123 |    "execution_count": null,
124 |    "metadata": {},
125 |    "outputs": [],
126 |    "source": [
127 |     "imputedData = multinet.predict(data)"
128 |    ]
129 |   },
130 |   {
131 |    "cell_type": "markdown",
132 |    "metadata": {},
133 |    "source": [
134 |     "## Visualization"
135 |    ]
136 |   },
137 |   {
138 |    "cell_type": "code",
139 |    "execution_count": null,
140 |    "metadata": {},
141 |    "outputs": [],
142 |    "source": [
143 |     "import matplotlib.pyplot as plt\n",
144 |     "import numpy as np\n",
145 |     "\n",
146 |     "limits = [0,100]\n",
147 |     "\n",
148 |     "fig,ax = plt.subplots()\n",
149 |     "\n",
150 |     "jitter = np.random.normal(0,1,data.size) # Add some jittering to better see the point density\n",
151 |     "ax.scatter(data.values.flatten()+jitter,imputedData.values.flatten(),s=2)\n",
152 |     "ax.plot(limits,limits,'r-.',linewidth=2)\n",
153 |     "ax.set_xlim(limits)\n",
154 |     "ax.set_ylim(limits)\n",
155 |     "\n",
156 |     "plt.show()\n"
157 |    ]
158 |   },
159 |   {
160 |    "cell_type": "markdown",
161 |    "metadata": {},
162 |    "source": [
163 |     "## Scoring\n",
164 |     "Display training metrics (MSE and Pearson's correlation on the test data)"
165 |    ]
166 |   },
167 |   {
168 |    "cell_type": "code",
169 |    "execution_count": null,
170 |    "metadata": {},
171 |    "outputs": [],
172 |    "source": [
173 |     "multinet.test_metrics"
174 |    ]
175 |   }
176 |  ],
177 |  "metadata": {
178 |   "kernelspec": {
179 |    "display_name": "Python 3",
180 |    "language": "python",
181 |    "name": "python3"
182 |   },
183 |   "language_info": {
184 |    "codemirror_mode": {
185 |     "name": "ipython",
186 |     "version": 3
187 |    },
188 |    "file_extension": ".py",
189 |    "mimetype": "text/x-python",
190 |    "name": "python",
191 |    "nbconvert_exporter": "python",
192 |    "pygments_lexer": "ipython3",
193 |    "version": "3.7.3"
194 |   }
195 |  },
196 |  "nbformat": 4,
197 |  "nbformat_minor": 2
198 | }
199 | 


--------------------------------------------------------------------------------
/makefile:
--------------------------------------------------------------------------------
 1 | PYTHON_VERSION = 3
 2 | 
 3 | reformat:
 4 | 	# Requires black: pip3 install black
 5 | 	black deepimpute
 6 | 	black tests
 7 | 	black setup.py
 8 | 
 9 | pytest:
10 | 	pytest
11 | 
12 | test:
13 | 	# We run the tests separately instead of with unittest to
14 | 	# avoid TensorFlow issue #8220 related to multiprocessing hanging.
15 | 	python${PYTHON_VERSION} ./tests/util_test.py
16 | 	python${PYTHON_VERSION} ./tests/maskedArrays_test.py
17 | 	python${PYTHON_VERSION} ./tests/multinet_test.py
18 | 	python${PYTHON_VERSION} ./tests/deepImpute_test.py
19 | 
20 | coverage:
21 | 	coverage${PYTHON_VERSION} run -p --source ./deepimpute/ ./tests/deepImpute_test.py
22 | 	coverage${PYTHON_VERSION} run -p --source ./deepimpute/ ./tests/util_test.py
23 | 	coverage${PYTHON_VERSION} run -p --source ./deepimpute/ ./tests/maskedArrays_test.py
24 | 	coverage${PYTHON_VERSION} run -p --source ./deepimpute/ ./tests/multinet_test.py
25 | 	coverage${PYTHON_VERSION} combine
26 | 	coverage${PYTHON_VERSION} html
27 | 	echo "Results in: file://${CURDIR}/htmlcov/index.html"
28 | 
29 | publish:
30 | 	pip install .
31 | 	python setup.py sdist
32 | 	twine upload dist/*
33 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | from setuptools import setup, find_packages
 2 | 
 3 | VERSION = "1.2"
 4 | 
 5 | setup(
 6 |     name="deepimpute",
 7 |     version=VERSION,
 8 |     description="scRNA-seq imputation",
 9 |     long_description=""" Deep learning (Tensorflow) based method for single cell RNA-seq data imputation. """,
10 |     classifiers=[],
11 |     keywords="Neural network, Deep Learning, scRNA-seq, single-cell, imputation",
12 |     author="Cedric Aridakessian",
13 |     author_email="carisdak@hawaii.edu",
14 |     url="",
15 |     entry_points={'console_scripts': ['deepImpute=deepimpute.deepImpute:deepImpute']},
16 |     license="MIT",
17 |     packages=find_packages(exclude=["examples", "tests"]),
18 |     include_package_data=True,
19 |     zip_safe=False,
20 |     install_requires=[
21 |         "numpy",
22 |         "pandas>=1.0",
23 |         "scipy",
24 |         "sklearn",
25 |         "tensorflow>=2.0",
26 |         "configparser",
27 |         "keras"
28 |     ],
29 | )
30 | 


--------------------------------------------------------------------------------
/tests/deepImpute_test.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | from unittest import mock
 3 | import argparse
 4 | 
 5 | import test_data
 6 | from deepimpute.deepImpute import deepImpute
 7 | 
 8 | args = {
 9 |     'inputFile': test_data.PATH+"csv",
10 |     'cell_axis': "rows",
11 |     'cores': 1,
12 |     'learning_rate': 1e-4,
13 |     'batch_size': 64,
14 |     'max_epochs': 300,
15 |     'output_neurons': 512,
16 |     'hidden_neurons': 300,
17 |     "dropout_rate": 0.2,
18 |     "subset": 1,
19 |     "limit": 1000,
20 |     "minVMR": 0.5,
21 |     "n_pred": None,
22 |     "policy": "restore",
23 |     "output": None
24 | }
25 | 
26 | class TestDeepImpute(unittest.TestCase):
27 |     """ """
28 | 
29 |     @mock.patch('argparse.ArgumentParser.parse_args',
30 |                 return_value=argparse.Namespace(**args))
31 |     def test_all(self, mock_args):
32 |         _ = deepImpute()
33 | 
34 | if __name__ == "__main__":
35 |     unittest.main()
36 | 


--------------------------------------------------------------------------------
/tests/maskedArrays_test.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | 
 3 | import numpy as np
 4 | import pandas as pd
 5 | 
 6 | import test_data
 7 | from deepimpute.maskedArrays import MaskedArray
 8 | 
 9 | 
10 | class TestNet(unittest.TestCase):
11 |     """ """
12 | 
13 |     def test_generate(self):
14 |         rawData = test_data.rawData
15 |         m_df = MaskedArray(data=rawData, dropout=0.1)
16 |         m_df.generate()
17 | 
18 |     def test_getValues(self):
19 |         data = pd.DataFrame(
20 |             {"r1": np.arange(5), "r2": np.arange(5) ** 2, "r3": np.arange(5) ** 3}
21 |         ).T
22 |         binMask = [
23 |             [True, True, False, False, True],
24 |             [True, True, False, False, False],
25 |             [False, True, True, True, True],
26 |         ]
27 |         maskedData = MaskedArray(data=data, mask=binMask)
28 | 
29 |         res_rows = [masked for masked in maskedData.getMasked(rows=True)]
30 |         self.assertEqual(res_rows, [[2, 3], [4, 9, 16], [0]])
31 | 
32 |         res_cols = [masked for masked in maskedData.getMasked(rows=False)]
33 |         self.assertEqual(res_cols, [[0], [], [2, 4], [3, 9], [16]])
34 | 
35 | 
36 | if __name__ == "__main__":
37 |     unittest.main()
38 | 


--------------------------------------------------------------------------------
/tests/multinet_test.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | 
 3 | import test_data
 4 | from deepimpute.multinet import MultiNet
 5 | 
 6 | # from utils_plot import train_test_scatter
 7 | 
 8 | 
 9 | class TestMultinet(unittest.TestCase):
10 |     """ """
11 | 
12 |     def test_all(self):
13 |         rawData = test_data.rawData
14 |         idx = rawData.quantile(.99).sort_values(ascending=False).index[0:1300]
15 |         rawData = rawData[idx]
16 | 
17 |         hyperparams = {
18 |             "architecture": [
19 |                 {"type": "dense", "activation": "relu", "neurons": 150},
20 |                 {"type": "dropout", "activation": "dropout", "rate": 0.2},
21 |             ],
22 |             "loss": "wMSE",
23 |             "sub_outputdim": 512,
24 |             "seed": 123,
25 |             "ncores": 2,
26 |             "verbose": 1
27 |         }
28 | 
29 |         model = MultiNet(**hyperparams)
30 |         model.fit(rawData)
31 |         _ = model.predict(rawData,policy="restore")
32 | 
33 |         print(model.test_metrics)
34 | 
35 | 
36 | if __name__ == "__main__":
37 |     unittest.main()
38 | 


--------------------------------------------------------------------------------
/tests/test_data.py:
--------------------------------------------------------------------------------
1 | import os
2 | import pandas as pd
3 | 
4 | PATH = os.path.dirname(os.path.realpath(__file__)) + "/../examples/test."
5 | rawData = pd.read_csv(PATH + "csv", index_col=0)
6 | 


--------------------------------------------------------------------------------
/tests/util_test.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | 
 3 | # import pandas as pd
 4 | # import numpy as np
 5 | 
 6 | from deepimpute import util
 7 | 
 8 | 
 9 | class TestUtil(unittest.TestCase):
10 |     """ """
11 | 
12 |     def test_log1x(self):
13 |         self.assertEqual(util.log1x(0), 0)
14 | 
15 |     def test_exp1x(self):
16 |         self.assertEqual(util.exp1x(0), 0)
17 | 
18 |     # def test_get_input_genes(self):
19 |     #     data = pd.DataFrame(np.identity(10))
20 |     #     res = util.get_input_genes(data, [2, 5], targets=[[0, 1]])
21 | 
22 |     # def test_get_target_genes(self):
23 |     #     genes_quantiles = pd.Series([1, 15, 9, 10, 12])
24 |     #     res = util._get_target_genes(genes_quantiles, minExpressionLevel=5, maxNumOfGenes=2)
25 |     #     self.assertEqual(sorted(res), [1, 4])
26 | 
27 | 
28 | if __name__ == "__main__":
29 |     unittest.main()
30 | 


--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
1 | [tox]
2 | envlist = py27,py36
3 | 
4 | [testenv]
5 | commands =
6 |     make test


--------------------------------------------------------------------------------