├── ImageDenoising
    ├── SimCo.pyc
    ├── KSVDDL.py
    ├── MODDL.py
    ├── GDDL.py
    ├── PSimCo.py
    ├── RSimCo.py
    ├── README.md
    ├── im2col.py
    ├── omperr.py
    ├── GDBTLS.py
    ├── RGDBTLS.py
    ├── KSVD.py
    └── DenoiseImage.py
├── SimCo
    ├── README.md
    ├── fg_tilde_eval01.py
    ├── DictUpdate03.py
    └── DictLineSearch03.py
├── KSVD
    ├── README.md
    ├── omp.py
    └── KSVD.py
├── README.md
├── omp.py
├── FindDistanceBetweenDictionaries.py
└── Dictionary_learning_v2.py


/ImageDenoising/SimCo.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Rehan-Ahmad/Dictionary-Learning-Algorithms/HEAD/ImageDenoising/SimCo.pyc


--------------------------------------------------------------------------------
/SimCo/README.md:
--------------------------------------------------------------------------------
1 | Reference:
2 | W. Dai, T. Xu, and W. Wang, "Simultaneous Codeword Optimization (SimCO) for Dictionary Update and Learning," submitted to IEEE Transactions on Signal Processing, October 2011.
3 | Full text is available at http://arxiv.org/abs/1109.5302
4 | 


--------------------------------------------------------------------------------
/KSVD/README.md:
--------------------------------------------------------------------------------
1 | Reference: 
2 |  "The K-SVD: An Algorithm for Designing of Overcomplete Dictionaries for Sparse Representation", written by M. Aharon, M. Elad, and A.M. Bruckstein and 
3 |   appeared in the IEEE Trans. On Signal Processing, Vol. 54, no. 11, pp. 4311-4322, November 2006. 
4 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Dictionary-Learning-Algorithms
2 | Main File: **Dictionary_Learning_v2.py** <br />
3 | 
4 | This file execute all the algorithms consisting of Gradient Descent based Dictionary Learning (variants), KSVD, MOD and SimCo. <br />
5 | It will plot the error curve after comparing the original and learned Dictionary on the synthetic Dataset. <br />
6 | 
7 | Implementation of Dictionary Learning algorithms on real data can be found in DenoiseImage folder. It uses learned dictionary for image denoising.
8 | 


--------------------------------------------------------------------------------
/ImageDenoising/KSVDDL.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Wed Aug 09 16:13:55 2017
 4 | 
 5 | KSVD based Dictionary Learning algorithm.
 6 | 
 7 | @author: Rehan
 8 | """
 9 | import numpy as np
10 | from copy import deepcopy
11 | from omperr import omperr
12 | from KSVD import KSVD
13 | 
14 | def KSVDDL(Y,param):
15 |     D = deepcopy(param.initialDictionary)
16 |     iterations = param.itN
17 |     errglobal = param.errorGoal
18 | 
19 |     for j in range(iterations):
20 |         X = omperr(D,Y,errglobal)
21 |         D,X = KSVD(Y,D,X)
22 | 
23 |     return D


--------------------------------------------------------------------------------
/ImageDenoising/MODDL.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Wed Aug 09 16:10:33 2017
 4 | 
 5 | MOD algorithm for Dictionary Learning
 6 | 
 7 | @author: Rehan
 8 | """
 9 | import numpy as np
10 | from copy import deepcopy
11 | from omperr import omperr
12 | from sklearn import preprocessing
13 | 
14 | def MODDL(Y,param):
15 |     D = deepcopy(param.initialDictionary)
16 |     iterations = param.itN
17 |     errglobal = param.errorGoal
18 | 
19 |     for j in range(iterations):
20 |         X = omperr(D,Y,errglobal)
21 |         D = np.dot(Y,np.linalg.pinv(X))
22 |         D = preprocessing.normalize(D,norm='l2',axis=0)
23 |     return D
24 | 


--------------------------------------------------------------------------------
/ImageDenoising/GDDL.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Wed Aug 09 14:00:03 2017
 4 | 
 5 | @author: Rehan
 6 | 
 7 | Gradient Descent Dictionary Learning (GDDL) with Momentum term  
 8 | 
 9 | """
10 | import numpy as np
11 | from copy import deepcopy
12 | from omperr import omperr
13 | from sklearn import preprocessing
14 | 
15 | def GDDL(Y,param):
16 |     iterations = param.itN
17 |     D = deepcopy(param.initialDictionary)
18 |     errglobal = param.errorGoal
19 |     gamma = param.MomentumGamma
20 |     alpha = param.alpha
21 |     
22 |     v = np.zeros(D.shape)
23 |     for j in range(iterations):
24 |         X = omperr(D,Y,errglobal)
25 |         v = gamma*v - alpha*np.dot(Y-np.dot(D,X),X.T)
26 |         D = D - v
27 |         D = preprocessing.normalize(D,norm='l2',axis=0)
28 |     return D


--------------------------------------------------------------------------------
/ImageDenoising/PSimCo.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Sun Aug 06 11:34:26 2017
 4 | 
 5 | @author: Rehan
 6 | """
 7 | from copy import deepcopy
 8 | from DictUpdate03 import DictUpdate03
 9 | from omperr import omperr
10 | 
11 | def PSimCo(Y,param):
12 |     class IPar():
13 |         pass
14 |     IPara = IPar()
15 |     itN = param.itN
16 |     D = deepcopy(param.initialDictionary)
17 |     errglobal = param.errorGoal
18 |     IPara.mu = 0
19 |     IPara.I = param.I
20 |     IPara.dispN = 20
21 |     IPara.DebugFlag = 0
22 |     IPara.itN = 1
23 |     IPara.gmin = 1e-5 # the minimum value of gradient
24 |     IPara.Lmin = 1e-6 # t4-t1 should be larger than Lmin
25 |     IPara.t4 = 1e-2   # the initial value of t4
26 |     IPara.rNmax = 3   # the number of iterative refinement in Part B in DictLineSearch03.m
27 |     
28 |     for itn in range(itN):
29 |         X = omperr(D,Y,errglobal)
30 |         D,X,_ = DictUpdate03(Y,D,X,IPara)
31 |     
32 |     return D


--------------------------------------------------------------------------------
/ImageDenoising/RSimCo.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Sun Aug 06 11:34:26 2017
 4 | 
 5 | @author: Rehan
 6 | """
 7 | from copy import deepcopy
 8 | from DictUpdate03 import DictUpdate03
 9 | from omperr import omperr
10 | 
11 | def RSimCo(Y,param):
12 |     class IPar():
13 |         pass
14 |     IPara = IPar()
15 |     itN = param.itN
16 |     D = deepcopy(param.initialDictionary)
17 |     errglobal = param.errorGoal
18 |     IPara.mu = 0.05
19 |     IPara.I = param.I
20 |     IPara.dispN = 20
21 |     IPara.DebugFlag = 0
22 |     IPara.itN = 1
23 |     IPara.gmin = 1e-5 # the minimum value of gradient
24 |     IPara.Lmin = 1e-6 # t4-t1 should be larger than Lmin
25 |     IPara.t4 = 1e-2   # the initial value of t4
26 |     IPara.rNmax = 3   # the number of iterative refinement in Part B in DictLineSearch03.m
27 |     
28 |     for itn in range(itN):
29 | #        X = omp(D,Y,param.sparsity)
30 |         X = omperr(D,Y,errglobal)
31 |         D,X,_ = DictUpdate03(Y,D,X,IPara)
32 |     
33 |     return D


--------------------------------------------------------------------------------
/omp.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Mon Jun 12 12:08:22 2017
 4 | 
 5 | @author: Rehan Ahmad
 6 | 
 7 | Orthogonal Matching Persuit (OMP) algorithm for sparse representation
 8 | 
 9 |  D: Dictionary, columns must be normalized (by l2 norm)
10 |  X: input signal to represent
11 |  L: max. no. of coefficients for each signal
12 |  A: Sparse coefficient Matrix
13 |  
14 | """
15 | import numpy as np
16 | from copy import deepcopy
17 | 
18 | def omp(D,X,L):
19 |     n,P = X.shape
20 |     n,K = D.shape
21 |     A = np.ndarray((D.shape[1],X.shape[1]))
22 |     for k in range(P):
23 |         a = 0
24 |         x = deepcopy(X[:,k])
25 |         residual = deepcopy(x)
26 |         indx = np.zeros((L,),dtype = int)
27 |         for j in range(L):
28 |             proj = np.dot(D.T,residual)
29 |             pos = np.argmax(np.abs(proj))
30 |             indx[j] = int(pos)
31 |             a = np.dot(np.linalg.pinv(D[:,indx[0:j+1]]),x)
32 |             residual = x-np.dot(D[:,indx[0:j+1]],a)
33 |             if np.sum(residual**2) < 1e-6:
34 |                 break
35 |         temp = np.zeros((K,))
36 |         temp[indx[0:j+1]] = deepcopy(a)
37 |         A[:,k] = temp
38 |     return A
39 | 


--------------------------------------------------------------------------------
/KSVD/omp.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Mon Jun 12 12:08:22 2017
 4 | 
 5 | @author: Rehan Ahmad
 6 | 
 7 | Orthogonal Matching Persuit (OMP) algorithm for sparse representation
 8 | 
 9 |  D: Dictionary, columns must be normalized (by l2 norm)
10 |  X: input signal to represent
11 |  L: max. no. of coefficients for each signal
12 |  A: Sparse coefficient Matrix
13 |  
14 | """
15 | import numpy as np
16 | from copy import deepcopy
17 | 
18 | def omp(D,X,L):
19 |     n,P = X.shape
20 |     n,K = D.shape
21 |     A = np.ndarray((D.shape[1],X.shape[1]))
22 |     for k in range(P):
23 |         a = 0
24 |         x = deepcopy(X[:,k])
25 |         residual = deepcopy(x)
26 |         indx = np.zeros((L,),dtype = int)
27 |         for j in range(L):
28 |             proj = np.dot(D.T,residual)
29 |             pos = np.argmax(np.abs(proj))
30 |             indx[j] = int(pos)
31 |             a = np.dot(np.linalg.pinv(D[:,indx[0:j+1]]),x)
32 |             residual = x-np.dot(D[:,indx[0:j+1]],a)
33 |             if np.sum(residual**2) < 1e-6:
34 |                 break
35 |         temp = np.zeros((K,))
36 |         temp[indx[0:j+1]] = deepcopy(a)
37 |         A[:,k] = temp
38 |     return A
39 | 


--------------------------------------------------------------------------------
/ImageDenoising/README.md:
--------------------------------------------------------------------------------
 1 | # Image Dnoising using Dictionary learning techniques.
 2 | 
 3 | Main File: **DenoiseImage.py** <br />
 4 | 
 5 | **GDDL**: Simple Gradient Descent with Momentum accelearation for Dictionary Learning .<br />
 6 | **GDBTLS**: Gradient Descent based Back Tracking Line Search Algorithm for Dictionary Learning.<br />
 7 | **RGDBTLS**: Regularized Gradient Descent based Back Tracking Line Search Algorithm for Dictionary Learning.<br />
 8 | **MODDL**: Method of Optimal Directions for Dictionary Learning.<br />
 9 | 
10 | Gradient Descent based dictionary learning algorithms are self written. While implementation of KSVD and SimCo are taken from following references.<br />
11 | Some required files for RSimCo and PSimCo can be found in SimCo folder.<br />
12 | 
13 | ## References: 
14 | 1. W. Dai, T. Xu, and W. Wang, "Simultaneous Codeword Optimization (SimCO) for Dictionary Update and Learning," submitted to IEEE Transactions on Signal Processing, October 2011. Full text is available at http://arxiv.org/abs/1109.5302.<br />
15 | 2. "The K-SVD: An Algorithm for Designing of Overcomplete Dictionaries for Sparse Representation", written by M. Aharon, M. Elad, and A.M. Bruckstein and appeared in the IEEE Trans. On Signal Processing, Vol. 54, no. 11, pp. 4311-4322, November 2006.<br />
16 | 


--------------------------------------------------------------------------------
/SimCo/fg_tilde_eval01.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Fri Jul 07 17:13:53 2017
 4 | 
 5 | @author: Rehan Ahmad
 6 | 
 7 |  fg_tilde_eval01 computes the gradient descent direction for LineSearch 
 8 |  in regularized SimCO version
 9 | 
10 |  References:
11 |  W. Dai, T. Xu, and W. Wang, 
12 |  "Simultaneous Codeword Optimization (SimCO) for Dictionary Update and Learning,"
13 |  submitted to IEEE Transactions on Signal Processing, October 2011.
14 |  Full text is available at http://arxiv.org/abs/1109.5302
15 | 
16 | """
17 | import numpy as np
18 | from copy import deepcopy
19 | import pdb
20 | 
21 | def fg_tilde_eval01(Y,D,Omega,IPara):
22 |     m,n = Y.shape
23 |     d = D.shape[1]
24 |     X = np.zeros((d,n))
25 |     OmegaL = np.sum(Omega,axis = 0)
26 |     mu = IPara.mu #the parameter of regularized item
27 |     mu_sqrt = np.sqrt(mu)
28 |     for cn in range(n):
29 |         L = deepcopy(OmegaL[cn])
30 |         X[Omega[:,cn],cn] = np.linalg.lstsq(np.append(D[:,Omega[:,cn]],np.diag(mu_sqrt*np.ones((L,))),axis=0),\
31 |          np.append(Y[:,cn],np.zeros((L,)),axis=0))[0]
32 | 
33 |     Yr = Y - np.dot(D,X)
34 |     # the cost function with regularized term
35 |     f = np.sum(Yr*Yr) + mu*np.sum(X*X)
36 |     freal = np.sum(Yr*Yr)
37 |     
38 |     g = -2*np.dot(Yr,X.T)
39 |     # additional steps to make sure the orthoganilty
40 |     DGcorr = np.sum(D*g, axis = 0)
41 |     g = g - D*np.tile(DGcorr,(m,1))
42 |     
43 |     return f,X,g,freal
44 | 


--------------------------------------------------------------------------------
/FindDistanceBetweenDictionaries.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Fri Jul 07 11:42:02 2017
 4 | 
 5 | @author: Rehan Ahmad
 6 | 
 7 |  D   : Original Dictionary
 8 |  Dhat: Predicted/Estimated Dictionary 
 9 |  
10 |  Finding the matched atoms between original and predicted Dictionary.
11 |  Sweeping through the columns of original Dictionary and finding closest
12 |  column in predicted Dictionary.
13 | 
14 |  Reference: 
15 |   "The K-SVD: An Algorithm for Designing of Overcomplete Dictionaries for Sparse
16 |   Representation", written by M. Aharon, M. Elad, and A.M. Bruckstein and 
17 |   appeared in the IEEE Trans. On Signal Processing, Vol. 54, no. 11, 
18 |   pp. 4311-4322, November 2006. 
19 | 
20 | """
21 | import numpy as np
22 | from copy import deepcopy
23 | def FindDistanceBetweenDictionaries(D,Dhat):
24 |     catchCounter = 0 
25 |     totalDistances = 0
26 |     Dnew = np.ndarray((D.shape[0],D.shape[1]))
27 |     for i in range(Dhat.shape[1]):
28 |         Dnew[:,i] = deepcopy(np.sign(Dhat[0,i])*Dhat[:,i])
29 |     for i in range(Dhat.shape[1]):
30 |         d = deepcopy(np.sign(D[0,i])*D[:,i])
31 |         distances = np.sum((Dnew-np.tile(np.reshape(d,(-1,1)),(1,Dhat.shape[1])))**2,axis=0)
32 |         index = np.argmin(distances)
33 |         errorOfElement = 1 - np.abs(np.dot(Dnew[:,index],d))
34 |         totalDistances = totalDistances + errorOfElement
35 |         catchCounter = catchCounter + (errorOfElement < 0.01)
36 |     return catchCounter
37 | 


--------------------------------------------------------------------------------
/ImageDenoising/im2col.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Mon Aug 07 23:56:42 2017
 4 |  
 5 |  Implementation taken from Matlab im2col.py
 6 |  This version contains sliding order only.
 7 | 
 8 | @author: Rehan
 9 | """
10 | import numpy as np
11 | from copy import deepcopy
12 | 
13 | def im2col(a,block):
14 |     ma,na = a.shape
15 |     m = block[0] 
16 |     n = block[1]
17 |     
18 |     if (ma<m or na<n): # if neighborhood is larger than image
19 |         b = np.zeros((m*n,))
20 |         return b
21 |     
22 |     # Create Hankel-like indexing sub matrix.
23 |     mc = block[0]
24 |     nc = ma-m+1
25 |     nn = na-n+1
26 |     cidx = np.reshape(range(mc),(-1,1))
27 |     ridx = np.reshape(range(1,nc+1),(1,-1))
28 |     t = cidx[:,np.zeros((nc,),dtype=int)] + ridx[np.zeros((mc,),dtype=int),:] # Hankel Subscripts
29 |     tt = np.zeros((mc*n,nc))
30 |     rows = np.array(range(mc))
31 |     for i in range(n):
32 |         tt[i*mc+rows,:] = t+ma*i
33 | 
34 |     ttt = np.zeros((mc*n,nc*nn),dtype=int)
35 |     cols = np.array(range(nc))
36 |     for j in range(nn):
37 |         ttt[:,j*nc+cols] = tt+ma*j
38 |     
39 |     # If a is a row vector, change it to a column vector. This change is
40 |     # necessary when A is a row vector and [M N] = size(A).
41 | 
42 |     b = np.zeros((ttt.shape[0],ttt.shape[1]))
43 |     for i in range(ttt.shape[0]):
44 |         b[i,:] = deepcopy(a[np.unravel_index(ttt[i,:]-1,(int(a.shape[0]),int(a.shape[1])),order='F')])
45 |     #b = deepcopy(a[ttt-1])
46 |     return b
47 | 
48 | 


--------------------------------------------------------------------------------
/ImageDenoising/omperr.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Tue Aug 08 15:00:03 2017
 4 | ============================================================================
 5 |  Sparse coding of a group of signals based on a given 
 6 |  dictionary and specified number of atoms to use. 
 7 |  input arguments: D - the dictionary
 8 |                   X - the signals to represent
 9 |                   errorGoal - the maximal allowed representation error for
10 |                   each siganl.
11 |  output arguments: A - sparse coefficient matrix.
12 | ============================================================================
13 | 
14 | @author: Rehan
15 | """
16 | import numpy as np
17 | from copy import deepcopy
18 | from numpy.linalg import pinv
19 | 
20 | def omperr(D,X,errorGoal):
21 |     n,P = X.shape
22 |     n,K = D.shape
23 |     E2 = (errorGoal**2)*n
24 |     maxNumCoef = n/2
25 |     A = np.zeros((D.shape[1],X.shape[1]))
26 |     for k in range(P):
27 |         a = 0
28 |         x = deepcopy(X[:,k])
29 |         residual = deepcopy(x)
30 |         indx = np.array([],dtype=int)
31 |         currResNorm2 = np.sum(residual**2)
32 |         j = 0
33 |         while currResNorm2 > E2 and j < maxNumCoef:
34 |             j = j+1
35 |             proj = np.dot(D.T,residual)
36 |             pos = np.argmax(np.abs(proj))
37 |             indx = np.append(indx,int(pos))
38 |             a = np.dot(pinv(D[:,indx[0:j]]),x)
39 |             residual = x-np.dot(D[:,indx[0:j]],a)
40 |             currResNorm2 = np.sum(residual**2)
41 |         if (len(indx) > 0):
42 |            A[indx,k] = deepcopy(a)
43 |     return A
44 | 


--------------------------------------------------------------------------------
/ImageDenoising/GDBTLS.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Wed Aug 09 14:24:07 2017
 4 | Gradient Descent Back Tracking Line Search (GDBTLS) Algorithm
 5 | @author: Rehan
 6 | """
 7 | import numpy as np
 8 | from omperr import omperr
 9 | from copy import deepcopy
10 | from sklearn import preprocessing
11 | 
12 | def GDBTLS(Y,param):
13 |     D = deepcopy(param.initialDictionary)
14 |     errglobal = param.errorGoal
15 |     iterations = param.itN
16 |     np.random.seed(3)
17 |     beta = np.random.rand()
18 |     np.random.seed(3)
19 |     eta = np.random.rand()*0.5    
20 |     Grad = np.zeros(D.shape)
21 |     
22 |     for j in range(iterations):
23 |         alpha = 1
24 |         X = omperr(D,Y,errglobal)
25 |         Dhat_GDtemp = deepcopy(D)
26 |         
27 |         #################################################################
28 |         # Back Tracking line search Algorithm (BTLS) to find optimal    #
29 |         # value of alpha                                                #
30 |         #################################################################
31 |         Grad = -np.dot(Y-np.dot(D,X),X.T)
32 |         oldfunc = np.linalg.norm(Y-np.dot(D,X),'fro')**2 
33 |         newfunc = np.linalg.norm(Y-np.dot(Dhat_GDtemp,X),'fro')**2 
34 |         while(~(newfunc <= oldfunc-eta*alpha*np.sum(Grad**2))):
35 |             alpha = beta*alpha
36 |             Dhat_GDtemp = deepcopy(D)            
37 |             Dhat_GDtemp = Dhat_GDtemp + alpha*np.dot(Y-np.dot(Dhat_GDtemp,X),X.T)
38 |             Dhat_GDtemp = preprocessing.normalize(Dhat_GDtemp,norm='l2',axis=0)
39 |             newfunc = np.linalg.norm(Y-np.dot(Dhat_GDtemp,X),'fro')**2
40 |             if(alpha < 1e-9):
41 |                 break
42 |         #################################################################
43 |         #################################################################
44 |         D = D + alpha*np.dot(Y-np.dot(D,X),X.T)
45 |         D = preprocessing.normalize(D,norm='l2',axis=0)
46 |         
47 |     return D


--------------------------------------------------------------------------------
/ImageDenoising/RGDBTLS.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Wed Aug 09 14:35:43 2017
 4 | 
 5 | Regularized Gradient Descent based Back Tracking Line Search (RGDBTLS) 
 6 | Algorithm. Regularization on Sparse matrix X.
 7 | 
 8 | @author: Rehan
 9 | """
10 | import numpy as np
11 | from sklearn import preprocessing
12 | from omperr import omperr
13 | from copy import deepcopy
14 | 
15 | def RGDBTLS(Y,param):
16 |     mu = param.mu
17 |     D = deepcopy(param.initialDictionary)
18 |     iterations = param.itN
19 |     errglobal = param.errorGoal
20 |     np.random.seed(3)
21 |     beta = np.random.rand()
22 |     np.random.seed(3)
23 |     eta = np.random.rand()*0.5
24 |     Grad = np.zeros(D.shape)
25 | 
26 |     for j in range(iterations):                        
27 |         alpha = 1
28 |         X = omperr(D,Y,errglobal)
29 |         Dhat_RGDtemp = deepcopy(D)
30 |         
31 |         #################################################################
32 |         # Back Tracking line search Algorithm (BTLS) to find optimal    #
33 |         # value of alpha                                                #
34 |         #################################################################
35 |         Grad = -np.dot(Y-np.dot(D,X),X.T)
36 |         oldfunc = np.linalg.norm(Y-np.dot(D,X),'fro')**2 + mu*np.linalg.norm(X,'fro')**2
37 |         newfunc = np.linalg.norm(Y-np.dot(Dhat_RGDtemp,X),'fro')**2 + mu*np.linalg.norm(X,'fro')**2
38 |         while(~(newfunc <= oldfunc-eta*alpha*np.sum(Grad**2))):
39 |             alpha = beta*alpha
40 |             Dhat_RGDtemp = deepcopy(D)            
41 |             Dhat_RGDtemp = Dhat_RGDtemp + alpha*np.dot(Y-np.dot(Dhat_RGDtemp,X),X.T)
42 |             Dhat_RGDtemp = preprocessing.normalize(Dhat_RGDtemp,norm='l2',axis=0)
43 |             newfunc = np.linalg.norm(Y-np.dot(Dhat_RGDtemp,X),'fro')**2 + mu*np.linalg.norm(X,'fro')**2
44 |             if(alpha < 1e-9):
45 |                 break
46 |         #################################################################
47 |         #################################################################
48 |         D = D + alpha*np.dot(Y-np.dot(D,X),X.T)
49 |         D = preprocessing.normalize(D,norm='l2',axis=0)
50 |         ########## Update X Considering same sparsity pattern ###########
51 |         Omega = X!=0
52 |         ColUpdate = np.sum(Omega,axis=0)!=0
53 |         YI = deepcopy(Y[:,ColUpdate])
54 |         DI = deepcopy(D)
55 |         OmegaI = deepcopy(Omega[:,ColUpdate])
56 |         OmegaL = np.sum(OmegaI,axis=0)
57 |         mu_sqrt = np.sqrt(mu)
58 |         
59 |         for cn in range(YI.shape[1]):
60 |             L = deepcopy(OmegaL[cn])
61 |             X[OmegaI[:,cn],cn] = np.linalg.lstsq(np.append(DI[:,OmegaI[:,cn]],\
62 |               np.diag(mu_sqrt*np.ones((L,))),axis=0),\
63 |                 np.append(YI[:,cn],np.zeros((L,)),axis=0))[0]
64 |     
65 |     return D


--------------------------------------------------------------------------------
/SimCo/DictUpdate03.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Fri Jul 07 12:23:12 2017
 4 | 
 5 | @author: Rehan Ahmad
 6 | 
 7 |  DictUpdate03 is the dictionary update function in SimCO.
 8 |  Given the initial dictionary D, initial sparse coefficient matrix X and
 9 |  the traning data matrix Y, this function produces the updated D and X
10 |  through itN iterations of line search algorithm in DictLineSearch03
11 | 
12 |  References:
13 |  W. Dai, T. Xu, and W. Wang, 
14 |  "Simultaneous Codeword Optimization (SimCO) for Dictionary Update and Learning,"
15 |  submitted to IEEE Transactions on Signal Processing, October 2011.
16 |  Full text is available at http://arxiv.org/abs/1109.5302
17 | 
18 | """
19 | import numpy as np
20 | from copy import deepcopy
21 | from DictLineSearch03 import DictLineSearch03
22 | import pdb
23 | 
24 | def DictUpdate03(Y, Dhat, Xhat, IPara):
25 |     D = deepcopy(Dhat)
26 |     X = deepcopy(Xhat)
27 |     
28 |     class OPara():
29 |         pass
30 |     OPara = OPara()
31 |     I = deepcopy(IPara.I)
32 |     itN = deepcopy(IPara.itN)
33 |     OPara.Flag = np.zeros((itN,))
34 |     OPara.f0 = np.zeros((itN,))
35 |     OPara.f1 = np.zeros((itN,))
36 |     OPara.f0real = np.zeros((itN,))
37 |     OPara.f1real = np.zeros((itN,))
38 |     OPara.gn2 = np.zeros((itN,))
39 |     OPara.topt = np.zeros((itN,))
40 |     d = X.shape[0]
41 |     m = Y.shape[0]
42 | 
43 |     #Ic is the complementary set of I
44 |     I = np.intersect1d(range(d),I)
45 |     Ic = np.setdiff1d(range(d),I)
46 |     Yp = Y - np.dot(D[:,Ic],X[Ic,:])
47 |     Omega = deepcopy(X!=0)
48 |     ColUpdate = np.sum(Omega[I,:],axis=0)!=0
49 |     YI = deepcopy(Yp[:,ColUpdate])
50 |     DI = deepcopy(D[:,I])
51 |     XI = deepcopy(X[:,ColUpdate][I,:])   # XI = X[I,ColUpdate]
52 |     OmegaI = deepcopy(Omega[:,ColUpdate][I,:])
53 |     f_YIComp = np.linalg.norm(Yp[:,~ColUpdate],'fro')**2
54 |     
55 |     # gradient descent line search
56 | 
57 |     for itn in range(itN):
58 |         if itn == 0:
59 |             OPara.f0real[itn] = np.linalg.norm(Y-np.dot(D,X),'fro')**2
60 |         else:
61 |             OPara.f0real[itn] = OPara.f1real[itn-1]
62 |         
63 |         # use the line search mechanism for dictionary update
64 |         DI,XI,OParaLS = DictLineSearch03(YI,DI,OmegaI,IPara)
65 |         D[:,I] = deepcopy(DI)
66 |         X[np.ix_(I,ColUpdate)] = deepcopy(XI)
67 |         OPara.Flag[itn] = OParaLS.Flag
68 |         OPara.f0[itn] = OParaLS.f0 + f_YIComp
69 |         OPara.f1[itn] = OParaLS.f1 + f_YIComp
70 |         OPara.f1real[itn] = np.linalg.norm(Y-np.dot(D,X),'fro')**2
71 |         OPara.gn2[itn] = deepcopy(OParaLS.gn2)
72 |         OPara.topt[itn] = deepcopy(OParaLS.topt)
73 |         
74 |         if OParaLS.Flag != 0:
75 |             OPara.Flag = OPara.Flag[0:itn]
76 |             OPara.f0 = OPara.f0[0:itn]
77 |             OPara.f1 = OPara.f1[0:itn]            
78 |             OPara.f0real = OPara.f0real[0:itn]
79 |             OPara.f1real = OPara.f1real[0:itn]
80 |             OPara.gn2 = OPara.gn2[0:itn]
81 |             OPara.topt = OPara.topt[0:itn]
82 |             break;
83 | 
84 |         IPara.t4 = deepcopy(OParaLS.topt)
85 |             
86 |     # finalize
87 |     D[:,I] = deepcopy(DI)
88 |     X[np.ix_(I,ColUpdate)] = deepcopy(XI)
89 | 
90 |     return D,X,OPara
91 |     
92 | 


--------------------------------------------------------------------------------
/KSVD/KSVD.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Mon Jun 12 23:23:38 2017
 4 | 
 5 | @author: Rehan Ahmad
 6 | 
 7 | K-SVD Algorithm for Dictionary Learning.
 8 | 
 9 |  Data (nxk)      : Input Data / Signal.
10 |  Dictionary (nxm): Dictionary as an input which will be updated by the algorithm.
11 |                    Dictionary columns must be normalized by l2 norm. 
12 |  CoefMat (mxk)   : Coefficient Matrix as an input generated by sparse algorithm
13 |                    (OMP/BP/MP etc).
14 |                    
15 |  This implementation is taken from matlab code KSVD.m by:
16 |  "The K-SVD: An Algorithm for Designing of Overcomplete Dictionaries for Sparse
17 |   Representation", written by M. Aharon, M. Elad, and A.M. Bruckstein and 
18 |   appeared in the IEEE Trans. On Signal Processing, Vol. 54, no. 11, 
19 |   pp. 4311-4322, November 2006. 
20 |   
21 | """
22 | import numpy as np
23 | from scipy.sparse.linalg import svds
24 | from numpy.linalg import svd
25 | from copy import deepcopy
26 | import pdb
27 | 
28 | def KSVD(Data, Dictionary, CoefMat):
29 | 
30 |     Dict = deepcopy(Dictionary)        
31 |     CoefMatrix = deepcopy(CoefMat)    
32 |     rPerm = np.random.permutation(Dict.shape[1])
33 |     
34 |     for j in rPerm: 
35 |         # the data indices that uses the j'th dictionary element.
36 |         relevantDataIndices = np.nonzero(CoefMatrix[j,:])[0]
37 |         if (len(relevantDataIndices)<1):
38 |             ErrorMat = Data-np.dot(Dict,CoefMatrix)
39 |             ErrorNormVec = np.sum(ErrorMat**2, axis = 0)
40 |             d,i = np.max(ErrorNormVec),np.argmax(ErrorNormVec)
41 |             betterDictionaryElement = deepcopy(Data[:,i])
42 |             betterDictionaryElement = betterDictionaryElement/np.sqrt(\
43 |                 np.dot(betterDictionaryElement.T,betterDictionaryElement))
44 |             betterDictionaryElement = betterDictionaryElement*\
45 |                 np.sign(betterDictionaryElement[0])
46 |             CoefMatrix[j,:] = np.zeros((CoefMatrix.shape[1],))
47 |             Dict[:,j] = deepcopy(np.reshape(betterDictionaryElement,(-1,)))            
48 |             return Dict,CoefMatrix
49 |     
50 |         tmpCoefMatrix = deepcopy(CoefMatrix[:,relevantDataIndices]) 
51 |         # the coeffitients of the element we now improve are not relevant.
52 |         tmpCoefMatrix[j,:] = np.zeros((tmpCoefMatrix.shape[1]))
53 |         # vector of errors that we want to minimize with the new element
54 |         errors = Data[:,relevantDataIndices] - np.dot(Dict,tmpCoefMatrix) 
55 |         # the better dictionary element and the values of beta are found using svd.
56 |         # This is because we would like to minimize || errors - beta*element ||_F^2. 
57 |         # that is, to approximate the matrix 'errors' with a one-rank matrix. This
58 |         # is done using the largest singular value.
59 |         try:
60 |             uu,ss,vv = svd(errors)
61 |             betterDictionaryElement,singularValue,betaVector = uu[:,0],ss[0],vv[0,:]
62 |         except: 
63 |             print "Error has occured in svd...You are now in pdb mode"
64 |             pdb.set_trace()
65 |                 
66 |         CoefMatrix[j,relevantDataIndices] = np.dot(singularValue,betaVector)
67 |         Dict[:,j] = deepcopy(np.reshape(betterDictionaryElement,(-1,)))
68 |         
69 |     return Dict, CoefMatrix
70 | 


--------------------------------------------------------------------------------
/ImageDenoising/KSVD.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | Created on Mon Jun 12 23:23:38 2017
 4 | 
 5 | @author: Rehan Ahmad
 6 | 
 7 | K-SVD Algorithm for Dictionary Learning.
 8 | 
 9 |  Data (nxk)      : Input Data / Signal.
10 |  Dictionary (nxm): Dictionary as an input which will be updated by the algorithm.
11 |                    Dictionary columns must be normalized by l2 norm. 
12 |  CoefMat (mxk)   : Coefficient Matrix as an input generated by sparse algorithm
13 |                    (OMP/BP/MP etc).
14 |                    
15 |  This implementation is taken from matlab code KSVD.m by:
16 |  "The K-SVD: An Algorithm for Designing of Overcomplete Dictionaries for Sparse
17 |   Representation", written by M. Aharon, M. Elad, and A.M. Bruckstein and 
18 |   appeared in the IEEE Trans. On Signal Processing, Vol. 54, no. 11, 
19 |   pp. 4311-4322, November 2006. 
20 |   
21 | """
22 | import numpy as np
23 | from scipy.sparse.linalg import svds
24 | from numpy.linalg import svd
25 | from copy import deepcopy
26 | import pdb
27 | 
28 | def KSVD(Data, Dictionary, CoefMat):
29 | 
30 |     Dict = deepcopy(Dictionary)        
31 |     CoefMatrix = deepcopy(CoefMat)    
32 |     rPerm = np.random.permutation(Dict.shape[1])
33 |     
34 |     for j in rPerm: 
35 |         # the data indices that uses the j'th dictionary element.
36 |         relevantDataIndices = np.nonzero(CoefMatrix[j,:])[0]
37 |         if (len(relevantDataIndices)<1):
38 |             ErrorMat = Data-np.dot(Dict,CoefMatrix)
39 |             ErrorNormVec = np.sum(ErrorMat**2, axis = 0)
40 |             d,i = np.max(ErrorNormVec),np.argmax(ErrorNormVec)
41 |             betterDictionaryElement = deepcopy(Data[:,i])
42 |             betterDictionaryElement = betterDictionaryElement/np.sqrt(\
43 |                 np.dot(betterDictionaryElement.T,betterDictionaryElement))
44 |             betterDictionaryElement = betterDictionaryElement*\
45 |                 np.sign(betterDictionaryElement[0])
46 |             CoefMatrix[j,:] = np.zeros((CoefMatrix.shape[1],))
47 |             Dict[:,j] = deepcopy(np.reshape(betterDictionaryElement,(-1,)))            
48 |             return Dict,CoefMatrix
49 |     
50 |         tmpCoefMatrix = deepcopy(CoefMatrix[:,relevantDataIndices]) 
51 |         # the coeffitients of the element we now improve are not relevant.
52 |         tmpCoefMatrix[j,:] = np.zeros((tmpCoefMatrix.shape[1]))
53 |         # vector of errors that we want to minimize with the new element
54 |         errors = Data[:,relevantDataIndices] - np.dot(Dict,tmpCoefMatrix) 
55 |         # the better dictionary element and the values of beta are found using svd.
56 |         # This is because we would like to minimize || errors - beta*element ||_F^2. 
57 |         # that is, to approximate the matrix 'errors' with a one-rank matrix. This
58 |         # is done using the largest singular value.
59 |         try:
60 |             uu,ss,vv = svd(errors)
61 |             betterDictionaryElement,singularValue,betaVector = uu[:,0],ss[0],vv[0,:]
62 |         except: 
63 |             print "Error has occured in svd...You are now in pdb mode"
64 |             pdb.set_trace()
65 |                 
66 |         CoefMatrix[j,relevantDataIndices] = np.dot(singularValue,betaVector)
67 |         Dict[:,j] = deepcopy(np.reshape(betterDictionaryElement,(-1,)))
68 |         
69 |     return Dict, CoefMatrix
70 | 


--------------------------------------------------------------------------------
/SimCo/DictLineSearch03.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | """
  3 | Created on Fri Jul 07 15:10:42 2017
  4 | 
  5 | @author: Rehan Ahmad
  6 | 
  7 |  DictUpdate03 is the dictionary update function in SimCO.
  8 |  Use line search mechanism to update the dictionary D. This is one
  9 |  iteration of the line search algorithm for dictionary update
 10 |  
 11 |  References:
 12 |  W. Dai, T. Xu, and W. Wang, 
 13 |  "Simultaneous Codeword Optimization (SimCO) for Dictionary Update and Learning,"
 14 |  submitted to IEEE Transactions on Signal Processing, October 2011.
 15 |  Full text is available at http://arxiv.org/abs/1109.5302
 16 | 
 17 | """
 18 | import numpy as np
 19 | from copy import deepcopy
 20 | from fg_tilde_eval01 import fg_tilde_eval01
 21 | 
 22 | def DictLineSearch03(Y,Dhat,Omega,IPara):
 23 |     D = deepcopy(Dhat)
 24 |     class OPara():
 25 |         pass
 26 |     OPara = OPara()
 27 |     c = (np.sqrt(5)-1)/2.0
 28 |     fv = np.zeros((100,))
 29 |     tv = np.zeros((100,))
 30 |     f4v = np.zeros((4,))
 31 |     t4v = np.zeros((4,))
 32 |     m,n = Y.shape
 33 |     d = D.shape[1]
 34 | 
 35 |     gmin = IPara.gmin
 36 |     Lmin = IPara.Lmin
 37 |     rNmax = IPara.rNmax
 38 |     t4v[3] = IPara.t4
 39 |     
 40 |     # compute the direction and corresponding gradient
 41 |     f,X,g,_ = fg_tilde_eval01(Y,D,Omega,IPara)
 42 |     evaln = 0
 43 |     fv[evaln] = deepcopy(f)
 44 |     tv[evaln] = 0
 45 |     
 46 |     # look at the magnitude of the gradient
 47 |     OPara.gn2 = np.linalg.norm(g,'fro')/np.linalg.norm(Y,'fro')**2
 48 |     gColn2 = np.sqrt(np.sum(g*g,axis = 0))
 49 |     gZero = gColn2 < gmin*np.linalg.norm(Y,'fro')**2/n
 50 |     # if the the magnitude of the gradient is less than the minimum threshold 
 51 |     # value, then quit and return D and X
 52 |     if np.sum(gZero) == D.shape[1]: 
 53 |         OPara.Flag = 1
 54 |         OPara.fv = fv[0:evaln]
 55 |         OPara.tv = tv[0:evaln]
 56 |         OPara.topt = 0
 57 |         OPara.f0 = deepcopy(f)
 58 |         OPara.f1 = deepcopy(f)
 59 |         return D,X,OPara
 60 | 
 61 |     gColn2[gZero] = 0
 62 |     H = np.zeros((m,d))
 63 |     H[0,gZero] = 1
 64 |     H[1:m,gZero] = 0
 65 |     H[:,~gZero] = g[:,~gZero]*np.tile(-1/gColn2[~gZero],(m,1))
 66 |     Step = gColn2/np.mean(gColn2)
 67 |     
 68 |     # Part A : find a good t4
 69 |     # set t4v and f4v; 
 70 |     t4v[2] = t4v[3]*c
 71 |     t4v[1] = t4v[3]*(1-c)
 72 |     f4v[0] = fv[0]
 73 |     for evaln in range(1,4):
 74 |         t = t4v[evaln]
 75 |         Dt = D*np.tile(np.cos(Step*t),(m,1)) + H*np.tile(np.sin(Step*t),(m,1))
 76 |         f4v[evaln],_,_,_ = fg_tilde_eval01(Y,Dt,Omega,IPara)
 77 | 
 78 |     fv[1:4] = f4v[1:4]
 79 |     tv[1:4] = t4v[1:4]
 80 |     # loop to find a good t4
 81 |     while t4v[3]-t4v[0] >= Lmin:
 82 |         # if f(D(t1)) is not greater than f(D(t2)), then t4=t2, t3=c*t4,
 83 |         #   t2=(1-c)*t4
 84 |         if f4v[0] <= f4v[1]:
 85 |             t4v[3] = t4v[1]
 86 |             t4v[2] = t4v[3]*c
 87 |             t4v[1] = t4v[3]*(1-c)
 88 |             f4v[3] = f4v[1]
 89 |             evaln = evaln + 1
 90 |             t = t4v[1]
 91 |             tv[evaln] = t
 92 |             Dt = D*np.tile(np.cos(Step*t),(m,1)) + H*np.tile(np.sin(Step*t),(m,1))
 93 |             ft,_,_,_ = fg_tilde_eval01(Y,Dt,Omega,IPara) 
 94 |             f4v[1] = ft
 95 |             fv[evaln] = ft
 96 |             evaln = evaln + 1
 97 |             t = t4v[2]
 98 |             tv[evaln] = t
 99 |             Dt = D*np.tile(np.cos(Step*t),(m,1)) + H*np.tile(np.sin(Step*t),(m,1))
100 |             ft,_,_,_ = fg_tilde_eval01(Y,Dt,Omega,IPara)
101 |             f4v[2] = ft
102 |             fv[evaln] = ft
103 |               
104 |             # if f(D(t2)) is not greater than f(D(t3)), then t4=t3, t3=t2,
105 |             #   t2=(1-c)*t4 
106 |         elif f4v[1] <= f4v[2]:
107 |             t4v[3] = t4v[2]; t4v[2] = t4v[1]; t4v[1] = t4v[3]*(1-c);
108 |             f4v[3] = f4v[2]; f4v[2] = f4v[1]
109 |             evaln = evaln + 1
110 |             t = t4v[1]; tv[evaln] = t
111 |             Dt = D*np.tile(np.cos(Step*t),(m,1)) + H*np.tile(np.sin(Step*t),(m,1))
112 |             ft,_,_,_ = fg_tilde_eval01(Y,Dt,Omega,IPara)
113 |             f4v[1] = ft; fv[evaln] = ft;
114 |             # if f(D(t3)) is greater than f(D(t4)), then t2=t3, t3=t4
115 |             #   t4=t3/c 
116 |         elif f4v[2]>f4v[3]:
117 |             t4v[1] = t4v[2]; t4v[2] = t4v[3]; t4v[3] = t4v[2]/c;
118 |             f4v[1] = f4v[2]; f4v[2] = f4v[3]; 
119 |             evaln = evaln + 1
120 |             t = t4v[3]; tv[evaln] = t;
121 |             Dt = D*np.tile(np.cos(Step*t),(m,1)) + H*np.tile(np.sin(Step*t),(m,1))
122 |             ft,_,_,_ = fg_tilde_eval01(Y,Dt,Omega,IPara);
123 |             f4v[3] = ft; fv[evaln] = ft;
124 |         else:
125 |             #quit
126 |             break
127 | 
128 |     # if t is too small, fet Flag to minus 1
129 | #    if t4v[3]-t4v[0] < Lmin:
130 | #        Flag = -1
131 |     
132 |     # Part B: refine the segment
133 |     evalN = evaln
134 |     # iterate until t4-t1 is small enough
135 |     while (t4v[3]-t4v[0]) >= Lmin and evaln-evalN <= rNmax:
136 |         # if f(D(t1))>f(D(t2))>f(D(t3)), then t1=t2, t2=t3, t3=t1+c*(t4-t1)
137 |         if f4v[0]>f4v[1] and f4v[1]>f4v[2]:
138 |             t4v[0] = t4v[1]; t4v[1] = t4v[2]; t4v[2] = t4v[0]+c*(t4v[3]-t4v[0]);
139 |             f4v[0] = f4v[1]; f4v[1] = f4v[2]
140 |             evaln = evaln + 1
141 |             t = t4v[2]; tv[evaln] = t;
142 |             Dt = D*np.tile(np.cos(Step*t),(m,1)) + H*np.tile(np.sin(Step*t),(m,1))
143 |             ft,_,_,_ = fg_tilde_eval01(Y,Dt,Omega,IPara);
144 |             f4v[2] = ft; fv[evaln] = ft;
145 |         # otherwise, t4=43, t3=t2, t2=t1+(1-c)(t4-t1)    
146 |         else:
147 |             t4v[3] = t4v[2]; t4v[2] = t4v[1]; t4v[1] = t4v[0]+(1-c)*(t4v[3]-t4v[0]);
148 |             f4v[3] = f4v[2]; f4v[2] = f4v[1];
149 |             evaln = evaln + 1;
150 |             t = t4v[1]; tv[evaln] = t;
151 |             Dt = D*np.tile(np.cos(Step*t),(m,1)) + H*np.tile(np.sin(Step*t),(m,1))
152 |             ft,_,_,_ = fg_tilde_eval01(Y,Dt,Omega,IPara)
153 |             f4v[1] = ft; fv[evaln] = ft;
154 |     
155 |     # finalize
156 |     fv = fv[0:evaln]
157 |     tv = tv[0:evaln]
158 |     findex = np.argmin(fv)
159 |     t = tv[findex]
160 |     D = D*np.tile(np.cos(Step*t),(m,1)) + H*np.tile(np.sin(Step*t),(m,1))
161 |     # compute X
162 |     f,X,_,_ = fg_tilde_eval01(Y,D,Omega,IPara)
163 |     
164 |     OPara.f0 = fv[0]
165 |     OPara.f1 = deepcopy(f)
166 |     OPara.fv = deepcopy(fv)
167 |     OPara.tv = deepcopy(tv)
168 |     OPara.topt = deepcopy(t)
169 |     OPara.Flag = 0
170 |     
171 |     return D,X,OPara
172 | 


--------------------------------------------------------------------------------
/ImageDenoising/DenoiseImage.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | """
  3 | Created on Sat Aug 05 16:34:13 2017
  4 | 
  5 |  References:
  6 | 1. W. Dai, T. Xu, and W. Wang, 
  7 |    "Simultaneous Codeword Optimization (SimCO) for Dictionary Update and Learning,"
  8 |    submitted to IEEE Transactions on Signal Processing, October 2011.
  9 |    Full text is available at http://arxiv.org/abs/1109.5302
 10 | 
 11 | 2. The methods implemented here are the same one as described in 
 12 |    "Image Denoising Via Sparse and Redundant representations over Learned Dictionaries",
 13 |    (IEEE Trans. on Image Processing, Vol. 15, no. 12, December 2006).
 14 |  
 15 | @author: Rehan
 16 | """
 17 | import cv2
 18 | import numpy as np
 19 | from RSimCo import RSimCo
 20 | from PSimCo import PSimCo
 21 | import matplotlib.pylab as plt
 22 | from time import time
 23 | from im2col import im2col
 24 | from omperr import omperr
 25 | from GDDL import GDDL
 26 | from GDBTLS import GDBTLS
 27 | from RGDBTLS import RGDBTLS
 28 | from MODDL import MODDL
 29 | from KSVDDL import KSVDDL
 30 | 
 31 | def DenoiseImage(Image,Param,seed):
 32 |     NN1,NN2 = Image.shape
 33 |     C = 1.15
 34 |     bb = 8
 35 |     maxNumBlocksToTrainOn = 1000
 36 |     sigma = Param.noise
 37 |     K = Param.k
 38 |     class Par():
 39 |         pass
 40 |     param=Par()
 41 |     param.K = K
 42 |     param.I = range(K)
 43 |     param.itN = 10
 44 |     param.errorGoal = sigma*C
 45 | 
 46 |     # first, train a dictionary on blocks from the noisy image
 47 |     if np.prod(np.array([NN1,NN2])-bb+1) > maxNumBlocksToTrainOn:
 48 |         np.random.seed(seed)
 49 |         randPermutation =  np.random.permutation(np.prod(np.array([NN1,NN2])-bb+1))
 50 |         selectedBlocks = randPermutation[0:maxNumBlocksToTrainOn]
 51 |         blkMatrix = np.zeros((bb**2,maxNumBlocksToTrainOn))
 52 |         for i in range(maxNumBlocksToTrainOn):
 53 |             row,col = np.unravel_index(selectedBlocks[i],tuple(np.array(Image.shape)-bb+1),order='F')
 54 |             currBlock = Image[row:row+bb,col:col+bb]
 55 |             blkMatrix[:,i] = np.reshape(currBlock,(-1,),order='F')
 56 |     else:
 57 |         blkMatrix = im2col(Image,(bb,bb))
 58 |         
 59 |     ######## Make initial dictionary from DCT ###########
 60 |     Pn = int(np.ceil(np.sqrt(K)))
 61 |     DCT = np.zeros((bb,Pn))
 62 |     for k in range(Pn):
 63 |         V = np.cos(np.array(range(bb))*k*np.pi/Pn)
 64 |         if k > 0:
 65 |             V = V-np.mean(V)
 66 |         DCT[:,k] = V/np.linalg.norm(V)
 67 |     DCT = np.kron(DCT,DCT)
 68 |     #####################################################
 69 |     param.initialDictionary = DCT[:,0:param.K]
 70 |     
 71 |     # reducedc
 72 |     vecOfMeans = np.mean(blkMatrix,axis=0)
 73 |     blkMatrix = blkMatrix-np.dot(np.ones((blkMatrix.shape[0],1)),np.reshape(vecOfMeans,(1,-1),order='F'))
 74 |     
 75 |     if(Param.method == 'RSimCo'):
 76 |         print 'Executing RSimCo...'
 77 |         Dictionary = RSimCo(blkMatrix,param)
 78 | 
 79 |     elif(Param.method == 'PSimCo'):
 80 |         print 'Executing PSimCo...'
 81 |         Dictionary = PSimCo(blkMatrix,param)
 82 | 
 83 |     elif(Param.method == 'GDDL'):
 84 |         print 'Executing GDDL...'
 85 |         param.MomentumGamma = 0.5
 86 |         param.alpha = 0.005
 87 |         Dictionary = GDDL(blkMatrix,param)
 88 | 
 89 |     elif(Param.method == 'GDBTLS'):
 90 |         print 'Executing GDBTLS...'
 91 |         param.alpha = 0.005
 92 |         Dictionary = GDBTLS(blkMatrix,param)
 93 | 
 94 |     elif(Param.method == 'RGDBTLS'):
 95 |         print 'Executing RGDBTLS...'
 96 |         param.alpha = 0.005
 97 |         param.mu = 0.05
 98 |         Dictionary = RGDBTLS(blkMatrix,param)
 99 | 
100 |     elif(Param.method == 'MODDL'):
101 |         print 'Executing MODDL...'
102 |         Dictionary = MODDL(blkMatrix,param)
103 | 
104 |     elif(Param.method == 'KSVDDL'):
105 |         print 'Executing KSVDDL...'
106 |         Dictionary = KSVDDL(blkMatrix,param)
107 |     else:
108 |         raise('No Method Defined')
109 | 
110 |     #denoise the image using the resulted dictionary
111 |     errT = sigma*C    
112 |     blocks = im2col(Image,(bb,bb))
113 |     idx = range(blocks.shape[1])
114 |     # go with jumps of 30000
115 |     for jj in range(0,blocks.shape[1],30000):
116 |         jumpSize = min(jj+30000,blocks.shape[1])
117 |         #reduceDC
118 |         vecOfMeans = np.mean(blocks[:,jj:jumpSize],axis=0)
119 |         blocks[:,jj:jumpSize] = blocks[:,jj:jumpSize] - np.tile(vecOfMeans,(blocks.shape[0],1))
120 |         Coefs = omperr(Dictionary,blocks[:,jj:jumpSize],errT)
121 |         #reducedc
122 |         blocks[:,jj:jumpSize] = np.dot(Dictionary,Coefs) + \
123 |         np.dot(np.ones((blocks.shape[0],1)),np.reshape(vecOfMeans,(1,-1),order='F'))
124 |     
125 |     count = 0
126 |     Weight = np.zeros((NN1,NN2))
127 |     IMout = np.zeros((NN1,NN2))
128 |     rows,cols = np.unravel_index(idx,tuple(np.array(Image.shape)-bb+1),order='F')
129 |     for i in range(len(cols)):
130 |         col = cols[i]
131 |         row = rows[i]
132 |         block = np.reshape(blocks[:,count],(bb,bb),order='F')
133 |         IMout[row:row+bb,col:col+bb] = IMout[row:row+bb,col:col+bb] + block
134 |         Weight[row:row+bb,col:col+bb] = Weight[row:row+bb,col:col+bb] + np.ones((bb,bb))
135 |         count = count+1
136 |     
137 |     IOut = (Image+0.034*sigma*IMout)/(1.0+0.034*sigma*Weight)
138 |     return IOut
139 | 
140 | if __name__ == "__main__":
141 |     tic = time()
142 |     class Para():
143 |         pass
144 |     Param = Para()
145 |     Param.k = 256 #dictionary atoms
146 |     Param.noise = 25
147 |     Param.method = 'PSimCo'
148 |     ImageName = 'camera.png'
149 |     seed = 1
150 |     
151 |     Img = cv2.imread(ImageName,0)
152 |     OriginalImage = np.float64(Img)
153 |     np.random.seed(seed)
154 |     NoisedImage = OriginalImage + Param.noise*np.random.standard_normal(OriginalImage.shape)
155 | 
156 |     # Denoise the corrupted image using learned dicitionary from corrupted image
157 |     DenoisedImage = DenoiseImage(NoisedImage, Param,seed)
158 | 
159 |     NoisedPSNR = 20.0*np.log10(255.0/np.sqrt(np.mean((NoisedImage-OriginalImage)**2)))
160 |     DenoisedPSNR = 20.0*np.log10(255.0/np.sqrt(np.mean((DenoisedImage-OriginalImage)**2)))
161 | 
162 |     # Display the results
163 |     plt.close('all')
164 |     plt.figure()
165 |     plt.subplot(131); plt.imshow(OriginalImage,cmap='gray')
166 |     plt.title('Original Image')
167 |     plt.subplot(132); plt.imshow(NoisedImage,cmap='gray')
168 |     plt.title('Noisy image with PSNR %f' %NoisedPSNR)
169 |     plt.subplot(133); plt.imshow(DenoisedImage,cmap='gray'); 
170 |     plt.title('Denoised Image by trained dictionary PSNR %f' %DenoisedPSNR)
171 |     print 'Total Time: %.2f min' %((time()-tic)/60.0)


--------------------------------------------------------------------------------
/Dictionary_learning_v2.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | """
  3 | Created on Mon Jun 05 23:31:51 2017
  4 | 
  5 | @author: Rehan Ahmad
  6 | 
  7 | Back Tracking Line Search taken from:
  8 | http://users.ece.utexas.edu/~cmcaram/EE381V_2012F/Lecture_4_Scribe_Notes.final.pdf
  9 | 
 10 | """
 11 | import numpy as np
 12 | from sklearn import preprocessing
 13 | import matplotlib.pylab as plt
 14 | from copy import deepcopy
 15 | import time
 16 | from omp import omp
 17 | from KSVD import KSVD
 18 | from FindDistanceBetweenDictionaries import FindDistanceBetweenDictionaries
 19 | from DictUpdate03 import DictUpdate03
 20 | import pdb
 21 | 
 22 | def awgn(x,snr_db):
 23 |     L = len(x)
 24 |     Es = np.sum(np.abs(x)**2)/L
 25 |     snr_lin = 10**(snr_db/10.0)
 26 |     noise = np.sqrt(Es/snr_lin)*np.random.randn(L)
 27 |     y = x + noise
 28 |     return y
 29 | 
 30 | if __name__ == "__main__":
 31 |     tic = time.time()
 32 |     
 33 |     FlagPGD = True; FlagPGDMom = True; FlagMOD = False; FlagKSVD = True;
 34 |     FlagRSimCo = True; FlagPSimCo = True; FlagGDBTLS = True; FlagRGDBTLS = True
 35 |     
 36 |     drows = 16 #20 #16
 37 |     dcols = 32 #50 #32
 38 |     ycols = 78 #1500 #78
 39 |     alpha = 0.005
 40 | 
 41 |     iterations = 1000
 42 |     SNR = 20
 43 |     epochs = 1
 44 |     sparsity = 4
 45 |     
 46 |     count_success = np.ndarray((iterations,epochs))
 47 |     count_success_momen = np.ndarray((iterations,epochs))
 48 |     count_success_MOD = np.ndarray((iterations,epochs))
 49 |     count_success_KSVD = np.ndarray((iterations,epochs))
 50 |     count_success_RSimCo = np.ndarray((iterations,epochs))
 51 |     count_success_PSimCo = np.ndarray((iterations,epochs))
 52 |     count_success_GDBTLS = np.ndarray((iterations,epochs))
 53 |     count_success_RGDBTLS = np.ndarray((iterations,epochs))
 54 | 
 55 |     e = np.ndarray((iterations,epochs))
 56 |     e_momen = np.ndarray((iterations,epochs))
 57 |     e_GDBTLS = np.ndarray((iterations,epochs))
 58 |     e_MOD = np.ndarray((iterations,epochs))
 59 |     e_KSVD = np.ndarray((iterations,epochs))
 60 |     e_RSimCo = np.ndarray((iterations,epochs))
 61 |     e_PSimCo = np.ndarray((iterations,epochs))
 62 |     e_RGDBTLS = np.ndarray((iterations,epochs))
 63 |     
 64 |     for epoch in range(epochs):
 65 |         alpha = 0.005
 66 | #        np.random.seed(epoch)
 67 | 
 68 |          ################# make initial dictionary #############################
 69 | #        Pn=ceil(sqrt(K));
 70 | #        DCT=zeros(bb,Pn);
 71 | #        for k=0:1:Pn-1,
 72 | #            V=cos([0:1:bb-1]'*k*pi/Pn);
 73 | #            if k>0, V=V-mean(V); end;
 74 | #            DCT(:,k+1)=V/norm(V);
 75 | #        end;
 76 | #        DCT=kron(DCT,DCT);
 77 |         ######################################################################
 78 | 
 79 |         # Creating dictionary from uniform iid random distribution
 80 |         # and normalizing atoms by l2-norm                
 81 |         D = np.random.rand(drows,dcols)
 82 |         D = preprocessing.normalize(D,norm='l2',axis=0)            
 83 |         # Creating data Y by linear combinations of randomly selected
 84 |         # atoms and iid uniform coefficients         
 85 |         Y = np.ndarray((drows,ycols))
 86 |         for i in range(ycols):
 87 |             PermIndx = np.random.permutation(dcols)
 88 |             Y[:,i] = np.random.rand()*D[:,PermIndx[0]] + \
 89 |              np.random.rand()*D[:,PermIndx[1]] + \
 90 |              np.random.rand()*D[:,PermIndx[2]] + \
 91 |              np.random.rand()*D[:,PermIndx[3]]
 92 |              
 93 |         # Add awgn noise in data Y 
 94 | #        for i in range(ycols):
 95 | #            Y[:,i] = awgn(Y[:,i],SNR)
 96 |         
 97 |         Dhat = np.ndarray((drows,dcols))
 98 |         Dhat = deepcopy(Y[:,np.random.permutation(ycols)[0:dcols]])
 99 |         Dhat = preprocessing.normalize(Dhat,norm='l2',axis=0)
100 |         Dhat_momen = deepcopy(Dhat)
101 |         Dhat_MOD = deepcopy(Dhat)
102 |         Dhat_KSVD = deepcopy(Dhat)
103 |         Dhat_RSimCo = deepcopy(Dhat)
104 |         Dhat_PSimCo = deepcopy(Dhat)
105 |         Dhat_GDBTLS = deepcopy(Dhat)
106 |         Dhat_RGDBTLS = deepcopy(Dhat)
107 |     
108 |         ########################################################
109 |         # Applying Projected Gradient Descent without momentum #
110 |         ########################################################
111 |         if(FlagPGD==True):
112 |             X = omp(D,Y,sparsity)
113 |             for j in range(iterations):
114 | #                X = omp(Dhat,Y,sparsity)
115 |     #            for i in range(dcols):
116 |     #                R = Y-np.dot(Dhat,X)
117 |     #                Dhat[:,i] = Dhat[:,i] + alpha*np.dot(R,X[i,:])
118 |                 Dhat = Dhat + alpha*np.dot(Y-np.dot(Dhat,X),X.T) #Parallel dictionary update...
119 |                 Dhat = preprocessing.normalize(Dhat,norm='l2',axis=0)
120 |                 
121 |                 e[j,epoch] = np.linalg.norm(Y-np.dot(Dhat,X),'fro')**2
122 |                 count = FindDistanceBetweenDictionaries(D,Dhat)
123 |                 count_success[j,epoch] = count
124 |         #####################################################
125 |         # Applying Projected Gradient Descent with momentum #
126 |         #####################################################
127 |         if(FlagPGDMom==True):
128 |             v = np.zeros((drows,dcols))
129 |             gamma = 0.5
130 |             X = omp(D,Y,sparsity)            
131 |             for j in range(iterations):
132 | #                X = omp(Dhat_momen,Y,sparsity)
133 |     #            for i in range(dcols):
134 |     #                R = Y-np.dot(Dhat_momen,X)
135 |     #                v[:,i] = gamma*v[:,i] + alpha*np.dot(R,X[i,:])
136 |     #                Dhat_momen[:,i] = Dhat_momen[:,i] + v[:,i]
137 |                 v = gamma*v - alpha*np.dot(Y-np.dot(Dhat_momen,X),X.T)
138 |                 Dhat_momen = Dhat_momen - v
139 |                     
140 |                 Dhat_momen = preprocessing.normalize(Dhat_momen,norm='l2',axis=0)        
141 |                 e_momen[j,epoch] = np.linalg.norm(Y-np.dot(Dhat_momen,X),'fro')**2
142 |                 count_momen = FindDistanceBetweenDictionaries(D,Dhat_momen)
143 |                 count_success_momen[j,epoch] = count_momen
144 |         #####################################################
145 |         # Applying Gradient Descent with back tracking line #
146 |         # search algorithm                                  #
147 |         #####################################################
148 |         if(FlagGDBTLS==True):
149 |             alpha = 1
150 |             beta = np.random.rand()
151 |             eta = np.random.rand()*0.5
152 |             Grad = np.zeros((drows,dcols))
153 |             
154 |             X = omp(D,Y,sparsity)
155 |             for j in range(iterations):
156 |                 alpha = 1
157 | #                X = omp(Dhat_GDBTLS,Y,sparsity)
158 |                 Dhat_GDtemp = deepcopy(Dhat_GDBTLS)
159 |                 
160 |                 #################################################################
161 |                 # Back Tracking line search Algorithm (BTLS) to find optimal    #
162 |                 # value of alpha                                                #
163 |                 #################################################################
164 |                 Grad = -np.dot(Y-np.dot(Dhat_GDBTLS,X),X.T)
165 |                 oldfunc = np.linalg.norm(Y-np.dot(Dhat_GDBTLS,X),'fro')**2 
166 |                 newfunc = np.linalg.norm(Y-np.dot(Dhat_GDtemp,X),'fro')**2 
167 |                 while(~(newfunc <= oldfunc-eta*alpha*np.sum(Grad**2))):
168 |                     alpha = beta*alpha
169 |                     Dhat_GDtemp = deepcopy(Dhat_GDBTLS)            
170 |                     Dhat_GDtemp = Dhat_GDtemp + alpha*np.dot(Y-np.dot(Dhat_GDtemp,X),X.T)
171 |                     Dhat_GDtemp = preprocessing.normalize(Dhat_GDtemp,norm='l2',axis=0)
172 |                     newfunc = np.linalg.norm(Y-np.dot(Dhat_GDtemp,X),'fro')**2
173 |                     if(alpha < 1e-9):
174 |                         break
175 |                 #################################################################
176 |                 #################################################################
177 |                 Dhat_GDBTLS = Dhat_GDBTLS + alpha*np.dot(Y-np.dot(Dhat_GDBTLS,X),X.T)
178 |                 Dhat_GDBTLS = preprocessing.normalize(Dhat_GDBTLS,norm='l2',axis=0)
179 |                 
180 |                 e_GDBTLS[j,epoch] = np.linalg.norm(Y-np.dot(Dhat_GDBTLS,X),'fro')**2
181 |                 count_GDBTLS = FindDistanceBetweenDictionaries(D,Dhat_GDBTLS)
182 |                 count_success_GDBTLS[j,epoch] = count_GDBTLS
183 |                 
184 |         #####################################################
185 |         # Applying Gradient Descent with back tracking line #
186 |         # search algorithm with regularization on X         #
187 |         #####################################################
188 |         if(FlagRGDBTLS==True):
189 |             alpha = 1
190 |             mu = 0.01 
191 | #            beta = np.random.rand()
192 | #            eta = np.random.rand()*0.5
193 | #            Grad = np.zeros((drows,dcols))
194 | #            mu = 0.01 
195 |             
196 |             X = omp(D,Y,sparsity)
197 |             for j in range(iterations):
198 |                 alpha = 1
199 | #                X = omp(Dhat_RGDBTLS,Y,sparsity)
200 |                 Dhat_RGDtemp = deepcopy(Dhat_RGDBTLS)
201 |                 
202 |                 #################################################################
203 |                 # Back Tracking line search Algorithm (BTLS) to find optimal    #
204 |                 # value of alpha                                                #
205 |                 #################################################################
206 |                 Grad = -np.dot(Y-np.dot(Dhat_RGDBTLS,X),X.T)
207 |                 oldfunc = np.linalg.norm(Y-np.dot(Dhat_RGDBTLS,X),'fro')**2 + mu*np.linalg.norm(X,'fro')**2
208 |                 newfunc = np.linalg.norm(Y-np.dot(Dhat_RGDtemp,X),'fro')**2 + mu*np.linalg.norm(X,'fro')**2
209 |                 while(~(newfunc <= oldfunc-eta*alpha*np.sum(Grad**2))):
210 |                     alpha = beta*alpha
211 |                     Dhat_RGDtemp = deepcopy(Dhat_RGDBTLS)            
212 |                     Dhat_RGDtemp = Dhat_RGDtemp + alpha*np.dot(Y-np.dot(Dhat_RGDtemp,X),X.T)
213 |                     Dhat_RGDtemp = preprocessing.normalize(Dhat_RGDtemp,norm='l2',axis=0)
214 |                     newfunc = np.linalg.norm(Y-np.dot(Dhat_RGDtemp,X),'fro')**2 + mu*np.linalg.norm(X,'fro')**2
215 |                     if(alpha < 1e-9):
216 |                         break
217 |                 #################################################################
218 |                 #################################################################
219 |                 Dhat_RGDBTLS = Dhat_RGDBTLS + alpha*np.dot(Y-np.dot(Dhat_RGDBTLS,X),X.T)
220 |                 Dhat_RGDBTLS = preprocessing.normalize(Dhat_RGDBTLS,norm='l2',axis=0)
221 |                 ########## Update X Considering same sparsity pattern############
222 |                 Omega = X!=0
223 |                 ColUpdate = np.sum(Omega,axis=0)!=0
224 |                 YI = deepcopy(Y[:,ColUpdate])
225 |                 DI = deepcopy(Dhat_RGDBTLS)
226 |                 XI = deepcopy(X[:,ColUpdate])
227 |                 OmegaI = deepcopy(Omega[:,ColUpdate])
228 |                 OmegaL = np.sum(Omega,axis=0)
229 |                 mu_sqrt = np.sqrt(mu)
230 |                 
231 |                 for cn in range(ycols):
232 |                     L = deepcopy(OmegaL[cn])
233 |                     X[OmegaI[:,cn],cn] = np.linalg.lstsq(np.append(DI[:,OmegaI[:,cn]],\
234 |                       np.diag(mu_sqrt*np.ones((L,))),axis=0),\
235 |                         np.append(YI[:,cn],np.zeros((L,)),axis=0))[0]
236 |                 #################################################################
237 |                 e_RGDBTLS[j,epoch] = np.linalg.norm(Y-np.dot(Dhat_RGDBTLS,X),'fro')**2
238 |                 count_RGDBTLS = FindDistanceBetweenDictionaries(D,Dhat_RGDBTLS)
239 |                 count_success_RGDBTLS[j,epoch] = count_RGDBTLS
240 |         ############################################
241 |         # Applying MOD Algorithm                   #
242 |         ############################################     
243 |         if(FlagMOD==True):
244 |             X = omp(D,Y,sparsity)
245 |             for j in range(iterations):
246 |     #            X = omp(Dhat_MOD,Y,sparsity)
247 |                 Dhat_MOD = np.dot(Y,np.linalg.pinv(X))
248 |                 Dhat_MOD = preprocessing.normalize(Dhat_MOD,norm='l2',axis=0)
249 |         
250 |                 count_MOD = FindDistanceBetweenDictionaries(D,Dhat_MOD)
251 |                 count_success_MOD[j,epoch] = count_MOD
252 |                 e_MOD[j,epoch] = np.linalg.norm(Y-np.dot(Dhat_MOD,X),'fro')**2
253 |         ############################################
254 |         # Applying KSVD Algorithm                  #
255 |         ############################################        
256 |         if(FlagKSVD==True):
257 |             X = omp(D,Y,sparsity)
258 |             for j in range(iterations):
259 |     #            X = omp(Dhat_KSVD,Y,sparsity)
260 |                 Dhat_KSVD,X = KSVD(Y,Dhat_KSVD,X)
261 |         
262 |                 count_KSVD = FindDistanceBetweenDictionaries(D,Dhat_KSVD)
263 |                 count_success_KSVD[j,epoch] = count_KSVD
264 |                 e_KSVD[j,epoch] = np.linalg.norm(Y-np.dot(Dhat_KSVD,X),'fro')**2
265 | 
266 |         #############################################
267 |         # Applying Regularized SimCo Algorithm      #
268 |         #############################################
269 |         if(FlagRSimCo==True):
270 |             class IPara():
271 |                 pass
272 |             IPara = IPara()
273 |             IPara.I = range(D.shape[1])
274 |             IPara.mu = 0.01
275 |             IPara.dispN = 20 
276 |             IPara.DebugFlag = 0
277 |             IPara.itN = 1
278 |             IPara.gmin = 1e-5; # the minimum value of gradient
279 |             IPara.Lmin = 1e-6; # t4-t1 should be larger than Lmin
280 |             IPara.t4 = 1e-2;   # the initial value of t4
281 |             IPara.rNmax = 3;   # the number of iterative refinement in Part B in DictLineSearch03.m
282 |         
283 |             X = omp(D,Y,sparsity)
284 |             for j in range(iterations):
285 |     #            X = omp(Dhat_RSimCo,Y,sparsity)
286 |                 Dhat_RSimCo,X,_ = DictUpdate03(Y,Dhat_RSimCo,X,IPara)
287 |         
288 |                 count_RSimCo = FindDistanceBetweenDictionaries(D,Dhat_RSimCo)
289 |                 count_success_RSimCo[j,epoch] = count_RSimCo
290 |                 e_RSimCo[j,epoch] = np.linalg.norm(Y-np.dot(Dhat_RSimCo,X),'fro')**2
291 |         #############################################
292 |         # Applying Primitive SimCo Algorithm        #
293 |         #############################################    
294 |         if(FlagPSimCo==True):
295 |             IPara.mu = 0
296 |             X = omp(D,Y,sparsity)
297 |             for j in range(iterations):
298 |     #            X = omp(Dhat_PSimCo,Y,sparsity)
299 |                 Dhat_PSimCo,X,_ = DictUpdate03(Y,Dhat_PSimCo,X,IPara)
300 |         
301 |                 count_PSimCo = FindDistanceBetweenDictionaries(D,Dhat_PSimCo)
302 |                 count_success_PSimCo[j,epoch] = count_PSimCo
303 |                 e_PSimCo[j,epoch] = np.linalg.norm(Y-np.dot(Dhat_PSimCo,X),'fro')**2
304 |         #############################################
305 |         #############################################
306 |         print 'epoch: ',epoch,'completed'
307 | 
308 |     plt.close('all')
309 |     if FlagPGD==True:     plt.plot(np.sum(count_success,axis=1)/epochs,'b',label = 'PGD')
310 |     if FlagPGDMom==True:  plt.plot(np.sum(count_success_momen,axis=1)/epochs,'r',label = 'PGD_Momentum')
311 |     if FlagMOD==True:     plt.plot(np.sum(count_success_MOD,axis=1)/epochs,'g',label = 'MOD')
312 |     if FlagKSVD==True:    plt.plot(np.sum(count_success_KSVD,axis=1)/epochs,'y',label = 'KSVD')
313 |     if FlagRSimCo==True:  plt.plot(np.sum(count_success_RSimCo,axis=1)/epochs,'m',label = 'RSimCo')
314 |     if FlagPSimCo==True:  plt.plot(np.sum(count_success_PSimCo,axis=1)/epochs,'c',label = 'PSimCo')
315 |     if FlagGDBTLS==True:  plt.plot(np.sum(count_success_GDBTLS,axis=1)/epochs,':',label = 'GDBTLS')
316 |     if FlagRGDBTLS==True: plt.plot(np.sum(count_success_RGDBTLS,axis=1)/epochs,'--',label = 'R_GDBTLS')
317 | 
318 |     plt.legend()
319 |     plt.xlabel('iteration number')
320 |     plt.ylabel('Success Counts in iteration')
321 |     plt.title('Dictionary Learning Algorithms applied on Syhthetic data')
322 |     
323 |     plt.figure()
324 |     if FlagPGD==True:     plt.plot(np.sum(e,axis=1)/epochs,'b',label = 'PGD')
325 |     if FlagPGDMom==True:  plt.plot(np.sum(e_momen,axis=1)/epochs,'r',label = 'PGD_Momentum')
326 |     if FlagMOD==True:     plt.plot(np.sum(e_MOD,axis=1)/epochs,'g',label = 'MOD')
327 |     if FlagKSVD==True:    plt.plot(np.sum(e_KSVD,axis=1)/epochs,'y',label = 'KSVD')
328 |     if FlagRSimCo==True:  plt.plot(np.sum(e_RSimCo,axis=1)/epochs,'m',label = 'RSimCo')
329 |     if FlagPSimCo==True:  plt.plot(np.sum(e_PSimCo,axis=1)/epochs,'c',label = 'PSimCo')
330 |     if FlagGDBTLS==True:  plt.plot(np.sum(e_GDBTLS,axis=1)/epochs,':',label = 'GDBTLS')
331 |     if FlagRGDBTLS==True: plt.plot(np.sum(e_RGDBTLS,axis=1)/epochs,'--',label = 'R_GDBTLS')
332 |     
333 |     plt.legend()
334 |     plt.xlabel('iteration number')
335 |     plt.ylabel('Error: Sum of squares')
336 |             
337 |     toc = time.time()
338 |     print 'Total Time Taken by code: ','%.2f' %((toc-tic)/60.0),'min'
339 | 


--------------------------------------------------------------------------------