├── README.rst ├── README.md ├── LICENSE └── sparse_nmf.py /README.rst: -------------------------------------------------------------------------------- 1 | sparseNMF 2 | ========= 3 | 4 | Sample Python implementation of the algorithm presented in the following 5 | paper: 6 | 7 | :: 8 | 9 | Block Coordinate Descent for Sparse NMF 10 | Vamsi K. Potluru, Sergey M. Plis, Jonathan Le Roux, Barak A. Pearlmutter, Vince D. Calhoun, Thomas P. Hayes 11 | ICLR 2013. 12 | http://arxiv.org/abs/1301.3527 13 | 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | sparseNMF 2 | ========= 3 | 4 | Sample Python implementation of the algorithm presented in the following paper: 5 | 6 | Block Coordinate Descent for Sparse NMF 7 | Vamsi K. Potluru, Sergey M. Plis, Jonathan Le Roux, Barak A. Pearlmutter, Vince D. Calhoun, Thomas P. Hayes 8 | ICLR 2013. 9 | http://arxiv.org/abs/1301.3527 10 | 11 | Documentation 12 | ------------- 13 | 14 | http://ismav.github.com/sparseNMF 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (BSD License) 2 | 3 | Copyright (c) 2014, Vamsi Krishna Potluru (Rutgers University) 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | * Neither the name of Rutgers University nor the 14 | names of its contributors may be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL VAMSI KRISHNA POTLURU BE LIABLE FOR ANY 21 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 24 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /sparse_nmf.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import numpy as np 3 | import scipy.io 4 | import pylab as plt 5 | 6 | def sparse_nmf(X, r, maxiter, spar, W = None, H = None): 7 | """Input data and the rank 8 | 9 | Learns a sparse NMF model given data X and the rank r. 10 | 11 | Parameters 12 | ---------- 13 | X : {array}, shape = [n_features, n_samples] 14 | r : rank of factorization 15 | maxiter : number of updates of the factor matrices 16 | spar : sparsity of the features given by measure sp(x)= (sqrt(n)-|x|_1/|x|_2 )/(sqrt(n)-1) 17 | 18 | Returns 19 | ------- 20 | W : {array} 21 | Feature matrix to the sparse NMF problem. 22 | 23 | Reference 24 | --------- 25 | 26 | Block Coordinate Descent for Sparse NMF 27 | Vamsi K. Potluru, Sergey M. Plis, Jonathan Le Roux, Barak A. Pearlmutter, Vince D. Calhoun, Thomas P. Hayes 28 | ICLR 2013. 29 | http://arxiv.org/abs/1301.3527 30 | """ 31 | m, n = np.shape(X) 32 | if not W and not H: 33 | W, H = init_nmf(X, r, spar) 34 | Obj = np.zeros(maxiter) 35 | for i in range(maxiter): 36 | Obj[i] = np.linalg.norm(X - np.dot(W, H), 'fro') 37 | print('iter: {} Obj: {}'.format(i + 1, Obj[i])) 38 | W = update_W(X, W, H, spar) 39 | H = update_H(X, W, H) 40 | 41 | return W 42 | 43 | 44 | def init_nmf(X, r, spar): 45 | """ Initialize the matrix factors for NMF. 46 | 47 | Use Gaussian random numbers in [-1,1] to initialize 48 | 49 | Parameters 50 | ---------- 51 | 52 | X: {array}, shape = [n_features, n_samples] 53 | r: rank of factorization 54 | 55 | Returns 56 | ------- 57 | 58 | W : {array} 59 | Feature matrix of the factorization 60 | H : {array} 61 | Weight matrix of the factorization 62 | 63 | where X ~ WH 64 | """ 65 | m, n = np.shape(X) 66 | W = np.zeros((m, r)) 67 | k = np.sqrt(m) - spar * (np.sqrt(m) - 1) 68 | for i in range(r): 69 | W[:, i] = sparse_opt(np.sort(np.random.rand(m))[::-1], k) 70 | 71 | W = np.random.rand(m, r) 72 | H = np.random.rand(r, n) 73 | return (W, H) 74 | 75 | 76 | def update_W(X, W, H, spar): 77 | """Update the feature matrix based on user-defined sparsity""" 78 | m, n = np.shape(X) 79 | m, r = np.shape(W) 80 | cach = np.zeros((m, r)) 81 | HHt = np.dot(H, H.T) 82 | cach = -np.dot(X, H.T) + np.dot(W, np.dot(H, H.T)) 83 | for i in range(r): 84 | W, cach = W_sparse_ith(W, HHt, cach, spar, i) 85 | 86 | return W 87 | 88 | 89 | def update_H(X, W, H): 90 | """Update the weight matrix using the regular multiplicative updates""" 91 | m, n = np.shape(X) 92 | WtX = np.dot(W.T, X) 93 | WtW = np.dot(W.T, W) 94 | for j in range(10): 95 | H = H * WtX / (np.dot(WtW, H) + np.spacing(1)) 96 | 97 | return H 98 | 99 | 100 | def W_sparse_ith(W, HHt, cach, spar, i): 101 | """ Update the columns sequentially""" 102 | m, r = np.shape(W) 103 | C = cach[:, i] - W[:, i] * HHt[i, i] 104 | V = np.zeros(m) 105 | k = np.sqrt(m) - spar * (np.sqrt(m) - 1) 106 | a = sparse_opt(np.sort(-C)[::-1], k) 107 | ind = np.argsort(-C)[::-1] 108 | V[ind] = a 109 | cach = cach + np.outer(V - W[:, i], HHt[i, :]) 110 | W[:, i] = V 111 | return (W, cach) 112 | 113 | 114 | def sparse_opt(b, k): 115 | """ Project a vector onto a sparsity constraint 116 | 117 | Solves the projection problem by taking into account the 118 | symmetry of l1 and l2 constraints. 119 | 120 | Parameters 121 | ---------- 122 | b : sorted vector in decreasing value 123 | k : Ratio of l1/l2 norms of a vector 124 | 125 | Returns 126 | ------- 127 | z : closest vector satisfying the required sparsity constraint. 128 | 129 | """ 130 | n = len(b) 131 | sumb = np.cumsum(b) 132 | normb = np.cumsum(b * b) 133 | pnormb = np.arange(1, n + 1) * normb 134 | y = (pnormb - sumb * sumb) / (np.arange(1, n + 1) - k * k) 135 | bot = np.int(np.ceil(k * k)) 136 | z = np.zeros(n) 137 | if bot > n: 138 | print('Looks like the sparsity measure is not between 0 and 1\n') 139 | return 140 | obj = (-np.sqrt(y) * (np.arange(1, n + 1) + k) + sumb) / np.arange(1, n + 1) 141 | indx = np.argmax(obj[bot:n]) 142 | p = indx + bot - 1 143 | p = min(p, n - 1) 144 | p = max(p, bot) 145 | lam = np.sqrt(y[p]) 146 | mue = -sumb[p] / (p + 1) + k / (p + 1) * lam 147 | z[:p + 1] = (b[:p + 1] + mue) / lam 148 | return z 149 | 150 | 151 | if __name__ == '__main__': 152 | r = 25 153 | spar = 0.5 154 | maxiter = 200 155 | X = scipy.io.loadmat('../data/orlfaces.mat') 156 | W = sparse_nmf(X['V'], r, maxiter, spar) 157 | for i in range(r): 158 | plt.subplot(np.sqrt(r), np.sqrt(r), i + 1) 159 | plt.imshow(np.reshape(W.T[i], [92, 112]).T) 160 | plt.axis('off') 161 | plt.ion() 162 | 163 | plt.show(True) 164 | #import ipdb 165 | #ipdb.set_trace() 166 | --------------------------------------------------------------------------------