├── self-conv.png ├── 2DPatch ├── demo_SAIST │ ├── tmp.tif │ ├── soft.m │ ├── Data │ │ └── Denoising_test_images │ │ │ └── barbara.png │ ├── Add_noise.m │ ├── Im2Patch.m │ ├── csnr.m │ ├── Denoising_Main.m │ ├── exemplar_matching.m │ ├── SAIST_param.m │ ├── Image_LASSC_Denoising.m │ ├── Block_matching.m │ ├── Image_LASSC_Denoising2.m │ ├── self_convolution_2d.m │ ├── Image_Denoising.m │ ├── LASSC_Denoising.m │ └── cal_ssim.m ├── README.md └── self_convolution_2d.m ├── 3DPatch ├── demo_Self-MM │ ├── data │ │ └── RGB-NIR │ │ │ └── 016.mat │ ├── PSNR3D.m │ ├── estimateSigma.m │ ├── sparse_l0.m │ ├── module_SVDl0.m │ ├── module_image2patch.m │ ├── module_patch2image.m │ ├── TLORTHOpenalty_func.m │ ├── onlineUTLupdate_analysis.m │ ├── module_LRapprox.m │ ├── module_TLapprox.m │ ├── self_convolution_2d.m │ ├── demo_rgbnir_denoising.m │ └── getparam.m ├── README.md └── self_convolution_2d.m └── README.md /self-conv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GuoLanqing/Self-Convolution/HEAD/self-conv.png -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/tmp.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GuoLanqing/Self-Convolution/HEAD/2DPatch/demo_SAIST/tmp.tif -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/data/RGB-NIR/016.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GuoLanqing/Self-Convolution/HEAD/3DPatch/demo_Self-MM/data/RGB-NIR/016.mat -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/soft.m: -------------------------------------------------------------------------------- 1 | % define the soft threshold function, which is used above. 2 | function y = soft(x,tau) 3 | 4 | y = sign(x).*max(abs(x)-tau,0); 5 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/PSNR3D.m: -------------------------------------------------------------------------------- 1 | function [ t ] = PSNR3D(X) 2 | [aa,bb, cc]=size(X); 3 | t=20*log10((sqrt(aa*bb*cc))*255/(norm(X(:),'fro'))); 4 | end 5 | 6 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/Data/Denoising_test_images/barbara.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GuoLanqing/Self-Convolution/HEAD/2DPatch/demo_SAIST/Data/Denoising_test_images/barbara.png -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/estimateSigma.m: -------------------------------------------------------------------------------- 1 | function sigma = estimateSigma(input, noisyY, sigma0, lambda) 2 | 3 | sigma = lambda * sqrt(abs(sigma0^2 - mean((input(:) - noisyY(:)).^2))); 4 | 5 | end 6 | 7 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/Add_noise.m: -------------------------------------------------------------------------------- 1 | function im_o = Add_noise( im, v ) 2 | 3 | seed = 0; 4 | randn( 'state', seed ); 5 | noise = randn( size(im) ); 6 | % noise = noise/sqrt(mean2(noise.^2)); 7 | im_o = double(im) + v*noise; -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/sparse_l0.m: -------------------------------------------------------------------------------- 1 | function [X, nonZeromap] = sparse_l0(X, threshold) 2 | [~, maxInd] = max(abs(X)); 3 | [n, N] = size(X); 4 | % maxVal = X(maxInd); 5 | nonZeromap = (bsxfun(@ge, abs(X), threshold)); 6 | base = 0 : n : n*(N - 1); 7 | maxInd = maxInd + base; 8 | nonZeromap(maxInd) = true; 9 | X = X .* nonZeromap; 10 | nonZeromap = sum(nonZeromap)'; 11 | end 12 | 13 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/Im2Patch.m: -------------------------------------------------------------------------------- 1 | function X = Im2Patch( im, par ) 2 | f = par.win; 3 | N = size(im,1)-f+1; 4 | M = size(im,2)-f+1; 5 | L = N*M; 6 | X = zeros(f*f, L, 'single'); 7 | k = 0; 8 | for i = 1:f 9 | for j = 1:f 10 | k = k+1; 11 | blk = im(i:end-f+i,j:end-f+j); 12 | X(k,:) = blk(:)'; 13 | end 14 | end -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/module_SVDl0.m: -------------------------------------------------------------------------------- 1 | function [X] = module_SVDl0( Y, thr0, sigma, param) 2 | % SVD 3 | [U,SigmaY,V] = svd((Y),'econ'); 4 | % threshold 5 | thr = thr0 * sigma * (param.PatchNOsqrt + param.PatchSize); 6 | idx = find(SigmaY>thr); 7 | 8 | svp = length(idx); 9 | SigmaX = SigmaY(idx); 10 | X = U(:,1:svp)*diag(SigmaX)*V(:,1:svp)'; 11 | end -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/csnr.m: -------------------------------------------------------------------------------- 1 | function s=csnr(A,B,row,col) 2 | 3 | [n,m,ch]=size(A); 4 | 5 | if ch==1 6 | e=A-B; 7 | e=e(row+1:n-row,col+1:m-col); 8 | me=mean(mean(e.^2)); 9 | s=10*log10(255^2/me); 10 | else 11 | e=A-B; 12 | e=e(row+1:n-row,col+1:m-col,:); 13 | e1=e(:,:,1);e2=e(:,:,2);e3=e(:,:,3); 14 | me1=mean(mean(e1.^2)); 15 | me2=mean(mean(e2.^2)); 16 | me3=mean(mean(e3.^2)); 17 | mse=(me1+me2+me3)/3; 18 | s = 10*log10(255^2/mse); 19 | % s(1)=10*log10(255^2/me1); 20 | % s(2)=10*log10(255^2/me2); 21 | % s(3)=10*log10(255^2/me3); 22 | end 23 | 24 | 25 | return; -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/Denoising_Main.m: -------------------------------------------------------------------------------- 1 | % clc; 2 | clear; 3 | %addpath('Utilities'); 4 | Test_image_dir = 'Data\Denoising_test_images'; 5 | 6 | 7 | %---------------------------------------------------------- 8 | % Experiment 1 9 | %---------------------------------------------------------- 10 | Out_dir = 'Results\Denoising_results\'; 11 | levels = [20]; 12 | 13 | for idx = 1 : length(levels) 14 | 15 | pre = sprintf('nsig_%d', idx); 16 | Res_dir = strcat(Out_dir, pre); 17 | nSig = levels(idx); 18 | tic; 19 | par = Image_Denoising(nSig, Res_dir, Test_image_dir); 20 | denoisetime=toc; 21 | end 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/module_image2patch.m: -------------------------------------------------------------------------------- 1 | function [Z, ZT] = module_image2patch(Xr,PatchSize) 2 | 3 | [H, W, c] = size(Xr); 4 | PS2 = PatchSize*PatchSize; 5 | TolPat = (H-PatchSize+1)*(W-PatchSize+1); 6 | Z = zeros(PS2*c, TolPat, 'single'); 7 | if nargout==2, ZT = zeros(PatchSize, PatchSize,d, TolPat, 'single'); end 8 | 9 | L = 0; 10 | for i = 1:PatchSize 11 | for j = 1:PatchSize 12 | L = L + 1; 13 | for t = 1:c 14 | tmp = Xr(i:end-PatchSize+i, j:end-PatchSize+j, t); 15 | Z((t-1)*PS2+L,:) = tmp(:)'; 16 | if nargout == 2, ZT(i,j,t,:) = tmp(:)'; end 17 | end 18 | end 19 | end 20 | 21 | 22 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/module_patch2image.m: -------------------------------------------------------------------------------- 1 | function [Y, weight] = module_patch2image(Z, weightZ, PatchSize, H, W, c) 2 | 3 | H2 = H - PatchSize + 1; 4 | W2 = W - PatchSize + 1; 5 | OffR = 0:H2-1; 6 | OffC = 0:W2-1; 7 | 8 | 9 | Y = zeros(H, W, c); 10 | weight = zeros(H, W, c); 11 | L = 0; 12 | weightZ = reshape(weightZ, [H2, W2]); 13 | 14 | for channel = 1 : c 15 | for i = 1:PatchSize 16 | for j = 1:PatchSize 17 | L = L + 1; 18 | tmp = Z(L,:); %row, should be rearrange as image 19 | Y(OffR+i,OffC+j, channel) = Y(OffR+i,OffC+j, channel) + reshape(tmp, [H2 W2]); 20 | weight(OffR+i,OffC+j, channel) = weight(OffR+i,OffC+j, channel) + weightZ; 21 | end 22 | end 23 | end 24 | 25 | 26 | -------------------------------------------------------------------------------- /2DPatch/README.md: -------------------------------------------------------------------------------- 1 | # 2D Patch Self-Convolution 2 | 3 | Description 4 | ----- 5 | 6 | The package includes: 7 | 8 | 1. the original SAIST program (directory `demo_SAIST`) 9 | 10 | 2. 2D image patch and 2D search window version of self-convolution `self_convolution_2d.m` 11 | 12 | ``` 13 | % Goal : self-convolution to replace original block matching (searching K most similar patches of each reference patch) 14 | % Inputs: 15 | % 1. im : image, Height*Width 16 | % 2. par : parameters 17 | % - win : size of search winodw 18 | % - step : step of extracting patches 19 | % - nblk : number of similar patches (K) 20 | % Output: 21 | % 1. pos_arr : K most similar patches for every reference patch 22 | % within search window, nblk * num_patches 23 | ``` 24 | 25 | Running the Code 26 | ----- 27 | 28 | - **A Simple Example**
29 | For a gray-scale image denoising example run the `Denoising_Main.m` file. 30 | 31 | Requirements and Dependencies 32 | ----- 33 | 34 | - Matlab with an Image Processing Toolbox. 35 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/TLORTHOpenalty_func.m: -------------------------------------------------------------------------------- 1 | function updateBuffer= TLORTHOpenalty_func(thr, buffer, blocks, param) 2 | YXT = buffer.YXT; 3 | D = buffer.D; 4 | alpha = param.alpha; 5 | % thr = param.TLthr0; % sparse coding threshold 6 | sparseWeight = param.sparseWeight; 7 | 8 | % (1) sparse coding 9 | X1 = D * blocks; 10 | % X2 = X1.*(bsxfun(@ge,abs(X1),thr)); 11 | X2 = sparse_l0(X1, thr); 12 | % (2) accumulate YX' 13 | YXT = alpha * YXT + (blocks * X2'); 14 | % (3) svd 15 | [U, ~, V] = svd(YXT); 16 | % (4) Update D 17 | D = V * U'; 18 | if isfield(param, 'isRecon') && param.isRecon 19 | % sparse coding with updated D 20 | X1 = D * blocks; 21 | % enforce sparsity >= 1 22 | [X2, scores] = sparse_l0(X1, thr); 23 | updateBuffer.sparsity = scores; 24 | scores = sparseWeight ./ scores; 25 | updateBuffer.TLaproxError = sum((X2 - X1).^2); 26 | % recon 27 | blocks = D' * X2; 28 | updateBuffer.blocks = blocks; % instantaneous recon. 29 | updateBuffer.scores = scores; 30 | end 31 | 32 | updateBuffer.YXT = YXT; 33 | updateBuffer.D = D; 34 | end 35 | 36 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/onlineUTLupdate_analysis.m: -------------------------------------------------------------------------------- 1 | function updateBuffer = onlineUTLupdate_analysis(buffer, param, blocks) 2 | 3 | % buffers 4 | YXT = buffer.YXT; 5 | D = buffer.D; 6 | alpha = param.alpha; 7 | thr = param.TLthr; % sparse coding threshold 8 | sparseWeight = param.sparseWeight; 9 | 10 | % (1) sparse coding 11 | X1 = D * blocks; 12 | % X2 = X1.*(bsxfun(@ge,abs(X1),thr)); 13 | X2 = sparse_l0(X1, thr); 14 | % (2) accumulate YX' 15 | YXT = alpha * YXT + (blocks * X2'); 16 | % (3) svd 17 | [U, ~, V] = svd(YXT); 18 | % (4) Update D 19 | D = V * U'; 20 | if isfield(param, 'isRecon') && param.isRecon 21 | % sparse coding with updated D 22 | X1 = D * blocks; 23 | % enforce sparsity >= 1 24 | [X2, scores] = sparse_l0(X1, thr); 25 | updateBuffer.sparsity = scores; 26 | scores = sparseWeight ./ scores; 27 | updateBuffer.TLaproxError = sum((X2 - X1).^2); 28 | % recon 29 | blocks = D' * X2; 30 | updateBuffer.blocks = blocks; % instantaneous recon. 31 | updateBuffer.scores = scores; 32 | end 33 | 34 | updateBuffer.YXT = YXT; 35 | updateBuffer.D = D; 36 | end 37 | 38 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/exemplar_matching.m: -------------------------------------------------------------------------------- 1 | function pos_arr = exemplar_matching(im, par) 2 | %S = 41; 3 | f = par.win; 4 | f2 = f^2; 5 | s = par.step; 6 | 7 | N = size(im,1)-f+1; 8 | M = size(im,2)-f+1; 9 | r = [1:s:N]; 10 | r = [r r(end)+1:N]; 11 | c = [1:s:M]; 12 | c = [c c(end)+1:M]; 13 | L = N*M; 14 | X = zeros(f*f, L, 'single'); 15 | 16 | k = 0; 17 | for i = 1:f 18 | for j = 1:f 19 | k = k+1; 20 | blk = im(i:end-f+i,j:end-f+j); 21 | X(k,:) = blk(:)'; 22 | end 23 | end 24 | 25 | % Index image 26 | I = (1:L); 27 | I = reshape(I, N, M); 28 | N1 = length(r); 29 | M1 = length(c); 30 | pos_arr = zeros(par.nblk, N1*M1 ); 31 | X = X'; 32 | 33 | for i = 1 : N1 34 | for j = 1 : M1 35 | 36 | row = r(i); 37 | col = c(j); 38 | off = (col-1)*N + row; 39 | off1 = (j-1)*N1 + i; 40 | 41 | template=X(off,:); 42 | temp=repmat(template,L,1); 43 | dif=(temp-X)'; 44 | dis=mean(dif.*dif); 45 | [val,ind] = sort(dis); 46 | pos_arr(:,off1) = ind(1:par.nblk) ; 47 | end 48 | end -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/SAIST_param.m: -------------------------------------------------------------------------------- 1 | %---------------------------------------------------- 2 | % Blur: 9x9 Uniform kernel; AGWN std Varance = 2^0.5 3 | % Data: May 20th, 2010 4 | % Author: Weisheng Dong, wsdong@mail.xidian.edu.cn 5 | %---------------------------------------------------- 6 | function par = SAIST_param(nSig) 7 | par.nSig = nSig; 8 | if nSig<=15 9 | par.win = 6; 10 | par.nblk = 40; 11 | par.c1 = 2.8*sqrt(2); % 1.7 12 | par.lamada = 0.63; 13 | par.w = 0.23; 14 | par.K = 10; 15 | elseif nSig <= 30 16 | par.win = 6; 17 | par.nblk = 60; 18 | par.c1 = 2.9*sqrt(2); % 2.6 19 | par.lamada = 0.65; 20 | par.w = 0.23; 21 | par.K = 11; 22 | elseif nSig<=50 23 | par.win = 8; 24 | par.nblk = 75; 25 | par.c1 = 3.0*sqrt(2); % 2.6 26 | par.lamada = 0.67; 27 | par.w = 0.23; 28 | par.K = 12; 29 | else 30 | par.win = 9; 31 | par.nblk = 90; 32 | par.c1 = 3.1*sqrt(2); % 1.6 33 | par.lamada = 0.64; 34 | par.w = 0.23; 35 | par.K = 14; 36 | end 37 | par.step=1; 38 | % par.step = min(6, par.win-1); 39 | end 40 | 41 | 42 | -------------------------------------------------------------------------------- /3DPatch/README.md: -------------------------------------------------------------------------------- 1 | # 3D Patch Self-Convolution 2 | 3 | Description 4 | ----- 5 | 6 | The package includes: 7 | 8 | 1. directory `demo_Self-MM` (the original Self-MM program) 9 | 10 | [Self-MM](https://arxiv.org/abs/2006.13714) is a multi-modality image denoising framework based on self-covolution and online sparse and group low-rank model learning scheme. 11 | 12 | 2. 3D image patch and 2D search window version of self-convolution `self_convolution_2d.m` 13 | 14 | ``` 15 | % Goal : self-convolution to replace original block matching (searching K most similar patches of each reference patch) 16 | % Inputs: 17 | % 1. Xr : image, Height*Width*Channels 18 | % 2. X : extracted patches, patch_size*num_patches 19 | % 3. param : parameters 20 | % - PatchNO : number of similar patches (K) 21 | % - PatchSize : size of image patch 22 | % - SearchWin : size of search window 23 | % - step : step of extracting patches 24 | % Output: 25 | % 1. NLBLK : K most similar patches for every reference patch 26 | % within search window, PatchNO * num_patches 27 | ``` 28 | 29 | 30 | Running the Code 31 | ----- 32 | 33 | - **A Simple Example**
34 | For a RGB-NIR image denoising example run the `demo_rgbnir_denoising.m` file. 35 | 36 | Requirements and Dependencies 37 | ----- 38 | 39 | - Matlab with an Image Processing Toolbox. 40 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/Image_LASSC_Denoising.m: -------------------------------------------------------------------------------- 1 | %---------------------------------------------------- 2 | % Blur: 9x9 Uniform kernel; AGWN std Varance = 2^0.5 3 | % Data: May 20th, 2010 4 | % Author: Weisheng Dong, wsdong@mail.xidian.edu.cn 5 | %---------------------------------------------------- 6 | function [xx,r]=Image_LASSC_Denoising(y, x, nSig) 7 | pre = 'LASSC_'; 8 | par.nSig = nSig; 9 | if nSig<=15 10 | par.win = 6; 11 | par.nblk = 40; 12 | par.c1 = 2.8*sqrt(2); % 1.7 13 | par.lamada = 0.63; 14 | par.w = 0.23; 15 | par.K = 10; 16 | elseif nSig <= 30 17 | par.win = 7; 18 | par.nblk = 60; 19 | par.c1 = 2.9*sqrt(2); % 2.6 20 | par.lamada = 0.65; 21 | par.w = 0.23; 22 | par.K = 11; 23 | elseif nSig<=50 24 | par.win = 8; 25 | par.nblk = 75; 26 | par.c1 = 3.0*sqrt(2); % 2.6 27 | par.lamada = 0.67; 28 | par.w = 0.23; 29 | par.K = 12; 30 | else 31 | par.win = 9; 32 | par.nblk = 90; 33 | par.c1 = 3.1*sqrt(2); % 1.6 34 | par.lamada = 0.64; 35 | par.w = 0.23; 36 | par.K = 14; 37 | end 38 | par.step = min(6, par.win-1); 39 | 40 | 41 | par.I = x; 42 | par.nim = y; 43 | 44 | [xx,r] = LASSC_Denoising( par ); 45 | 46 | %disp( sprintf('%s: PSNR = %3.2f SSIM = %f\n', im_dir(i).name, PSNR, SSIM) ); 47 | 48 | 49 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/Block_matching.m: -------------------------------------------------------------------------------- 1 | function pos_arr = Block_matching(im, par) 2 | S = 30; 3 | f = par.win; 4 | f2 = f^2; 5 | s = par.step; 6 | 7 | N = size(im,1)-f+1; 8 | M = size(im,2)-f+1; 9 | r = [1:s:N]; 10 | r = [r r(end)+1:N]; 11 | c = [1:s:M]; 12 | c = [c c(end)+1:M]; 13 | L = N*M; 14 | X = zeros(f*f, L, 'single'); 15 | 16 | k = 0; 17 | for i = 1:f 18 | for j = 1:f 19 | k = k+1; 20 | blk = im(i:end-f+i,j:end-f+j); 21 | X(k,:) = blk(:)'; 22 | end 23 | end 24 | 25 | % Index image 26 | I = (1:L); 27 | I = reshape(I, N, M); 28 | N1 = length(r); 29 | M1 = length(c); 30 | pos_arr = zeros(par.nblk, N1*M1 ); 31 | X = X'; 32 | 33 | for i = 1 : N1 34 | for j = 1 : M1 35 | 36 | row = r(i); 37 | col = c(j); 38 | off = (col-1)*N + row; 39 | off1 = (j-1)*N1 + i; 40 | 41 | rmin = max( row-S, 1 ); 42 | rmax = min( row+S, N ); 43 | cmin = max( col-S, 1 ); 44 | cmax = min( col+S, M ); 45 | 46 | idx = I(rmin:rmax, cmin:cmax); 47 | idx = idx(:); 48 | B = X(idx, :); 49 | v = X(off, :); 50 | 51 | 52 | dis = (B(:,1) - v(1)).^2; 53 | for k = 2:f2 54 | dis = dis + (B(:,k) - v(k)).^2; 55 | end 56 | dis = dis./f2; 57 | [val,ind] = sort(dis); 58 | pos_arr(:,off1) = idx( ind(1:par.nblk) ); 59 | end 60 | end -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/module_LRapprox.m: -------------------------------------------------------------------------------- 1 | function [output, weights] = ... 2 | module_LRapprox(input, idxBM, noisy, param, GridX, GridY) 3 | 4 | output = zeros(size(input)); 5 | numMatch = size(idxBM, 1); 6 | patchDim = param.PatchSize * param.PatchSize; 7 | weights = zeros(1, size(input, 2), size(input,3)); 8 | LGridH = length(GridX); 9 | LGridW = length(GridY); 10 | pc = param.patchChannel; 11 | 12 | for i = 1 : LGridH 13 | for j = 1 : LGridW 14 | kk = (j-1)*LGridH + i; % index of key patches 15 | curMatrix = input(:, idxBM(:,kk)); 16 | noisyMat = noisy(:, idxBM(:,kk)); 17 | % vectorize the 3D space 18 | curMatrix = reshape(curMatrix, patchDim, pc * numMatch); 19 | noisyMat = reshape(noisyMat, patchDim, pc * numMatch); 20 | if param.noiseEstFlag 21 | sigma = param.lamada*sqrt(abs(param.nSig^2-mean((curMatrix(:)-noisyMat(:)).^2))); 22 | else 23 | sigma = param.nSig; 24 | end 25 | % de-mean 26 | MeanMat = mean(curMatrix, 2); 27 | curMatrix = bsxfun(@minus, curMatrix, MeanMat); 28 | 29 | ReconMat = module_SVDl0(curMatrix, param.LRthr0, sigma, param); 30 | 31 | % re-mean 32 | ReconMat = bsxfun(@plus, ReconMat, MeanMat); 33 | 34 | % revert to 3D image 35 | ReconMat = reshape(ReconMat, patchDim *pc, numMatch); 36 | 37 | output(:,idxBM(:,kk)) = output(:,idxBM(:,kk)) + ReconMat; 38 | weights(1, idxBM(:,kk)) = weights(1, idxBM(:,kk)) + 1; 39 | end 40 | end -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/Image_LASSC_Denoising2.m: -------------------------------------------------------------------------------- 1 | %---------------------------------------------------- 2 | % Blur: 9x9 Uniform kernel; AGWN std Varance = 2^0.5 3 | % Data: May 20th, 2010 4 | % Author: Weisheng Dong, wsdong@mail.xidian.edu.cn 5 | %---------------------------------------------------- 6 | function [xx,r]=Image_LASSC_Denoising2(y, x, nSig, win, iter) 7 | pre = 'LASSC_'; 8 | par.nSig = nSig; 9 | par.win=win; 10 | par.K=iter; 11 | if nSig<=15 12 | %par.win = 6; 13 | par.nblk = 40; 14 | par.c1 = 2.8*sqrt(2); % 1.7 15 | par.lamada = 0.63; 16 | par.w = 0.23; 17 | %par.K = 10; 18 | elseif nSig <= 30 19 | %par.win = 7; 20 | par.nblk = 60; 21 | par.c1 = 2.9*sqrt(2); % 2.6 22 | par.lamada = 0.65; 23 | par.w = 0.23; 24 | %par.K = 11; 25 | elseif nSig<=50 26 | %par.win = 8; 27 | par.nblk = 75; 28 | par.c1 = 3.0*sqrt(2); % 2.6 29 | par.lamada = 0.67; 30 | par.w = 0.23; 31 | %par.K = 12; 32 | else 33 | %par.win = 9; 34 | par.nblk = 90; 35 | par.c1 = 3.1*sqrt(2); % 1.6 36 | par.lamada = 0.64; 37 | par.w = 0.23; 38 | %par.K = 14; 39 | end 40 | par.step = min(4, par.win-1); 41 | 42 | 43 | par.I = [x fliplr(x);flipud(x) flipud(fliplr(x))]; 44 | par.nim = [y fliplr(y);flipud(y) flipud(fliplr(y))]; 45 | 46 | [xx_buf PSNR SSIM] = LASSC_Denoising( par ); 47 | xx=(xx_buf(1:end/2,1:end/2)+fliplr(xx_buf(1:end/2,end/2+1:end))... 48 | +flipud(xx_buf(end/2+1:end,1:end/2))+fliplr(flipud(xx_buf(end/2+1:end,end/2+1:end))))/4; 49 | 50 | %disp( sprintf('%s: PSNR = %3.2f SSIM = %f\n', im_dir(i).name, PSNR, SSIM) ); 51 | 52 | 53 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/self_convolution_2d.m: -------------------------------------------------------------------------------- 1 | function pos_arr = self_convolution_2d(im, par) 2 | 3 | %PROCESSSTATIC Summary of this function goes here 4 | % Detailed explanation goes here 5 | % Goal : self-convolution to replace original block matching (searching K most similar patches of each reference patch) 6 | % Inputs: 7 | % 1. im : image, Height*Width 8 | % 2. par : parameters 9 | % - win : search winodw size 10 | % - step : step of extracting patches 11 | % - nblk : number of similar patches (K) 12 | % Output: 13 | % 1. pos_arr : K most similar patches for every reference patch 14 | % within search window, par.nblk*num_patches 15 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 16 | 17 | S = 30; 18 | f = par.win; 19 | s = par.step; 20 | 21 | N = size(im,1)-f+1; 22 | M = size(im,2)-f+1; 23 | r = [1:s:N]; 24 | r = [r r(end)+1:N]; 25 | c = [1:s:M]; 26 | c = [c c(end)+1:M]; 27 | L = N*M; 28 | X = zeros(f*f, L, 'single'); 29 | 30 | k = 0; 31 | for i = 1:f 32 | for j = 1:f 33 | k = k+1; 34 | blk = im(i:end-f+i,j:end-f+j); 35 | X(k,:) = blk(:)'; 36 | end 37 | end 38 | 39 | % Index image 40 | I = (1:L); 41 | I = reshape(I, N, M); 42 | N1 = length(r); 43 | M1 = length(c); 44 | pos_arr = zeros(par.nblk, N1*M1 ); 45 | Patnorm = 0.5*(vecnorm(X)).^2; 46 | 47 | for i = 1 : N1 48 | for j = 1 : M1 49 | 50 | row = r(i); 51 | col = c(j); 52 | off = (col-1)*N + row; 53 | off1 = (j-1)*N1 + i; 54 | 55 | rmin = max( row-S, 1 ); 56 | rmax = min( row+S, N ); 57 | cmin = max( col-S, 1 ); 58 | cmax = min( col+S, M ); 59 | 60 | idx = I(rmin:rmax, cmin:cmax); 61 | Subimage = im(rmin:rmax+f-1, cmin:cmax+f-1); 62 | idx = idx(:); 63 | Subpatnorm = Patnorm(idx); 64 | D = conv2(Subimage,rot90(reshape(X(:,off),f,[])',2),"valid"); 65 | dis = Subpatnorm - D(:)'; 66 | 67 | [val,ind] = sort(dis); 68 | pos_arr(:,off1) = idx( ind(1:par.nblk) ); 69 | end 70 | end -------------------------------------------------------------------------------- /2DPatch/self_convolution_2d.m: -------------------------------------------------------------------------------- 1 | function pos_arr = self_convolution_2d(im, par) 2 | 3 | %PROCESSSTATIC Summary of this function goes here 4 | % Detailed explanation goes here 5 | % Goal : self-convolution to replace original block matching (searching K most similar patches of each reference patch) 6 | % Inputs: 7 | % 1. im : image, Height*Width 8 | % 2. par : parameters 9 | % - win : patch size 10 | % - nblk : number of similar patches (K) 11 | % - step : step of extracting patches 12 | % 3. S : search window size 13 | % Output: 14 | % 1. pos_arr : K most similar patches for every reference patch 15 | % within search window, par.nblk*num_patches 16 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 17 | 18 | S = 30; 19 | f = par.win; 20 | s = par.step; 21 | 22 | N = size(im,1)-f+1; 23 | M = size(im,2)-f+1; 24 | r = [1:s:N]; 25 | r = [r r(end)+1:N]; 26 | c = [1:s:M]; 27 | c = [c c(end)+1:M]; 28 | L = N*M; 29 | X = zeros(f*f, L, 'single'); 30 | 31 | k = 0; 32 | for i = 1:f 33 | for j = 1:f 34 | k = k+1; 35 | blk = im(i:end-f+i,j:end-f+j); 36 | X(k,:) = blk(:)'; 37 | end 38 | end 39 | 40 | % Index image 41 | I = (1:L); 42 | I = reshape(I, N, M); 43 | N1 = length(r); 44 | M1 = length(c); 45 | pos_arr = zeros(par.nblk, N1*M1 ); 46 | Patnorm = 0.5*(vecnorm(X)).^2; 47 | 48 | for i = 1 : N1 49 | for j = 1 : M1 50 | 51 | row = r(i); 52 | col = c(j); 53 | off = (col-1)*N + row; 54 | off1 = (j-1)*N1 + i; 55 | 56 | rmin = max( row-S, 1 ); 57 | rmax = min( row+S, N ); 58 | cmin = max( col-S, 1 ); 59 | cmax = min( col+S, M ); 60 | 61 | idx = I(rmin:rmax, cmin:cmax); 62 | Subimage = im(rmin:rmax+f-1, cmin:cmax+f-1); 63 | idx = idx(:); 64 | Subpatnorm = Patnorm(idx); 65 | D = conv2(Subimage,rot90(reshape(X(:,off),f,[])',2),"valid"); 66 | dis = Subpatnorm - D(:)'; 67 | 68 | [val,ind] = sort(dis); 69 | pos_arr(:,off1) = idx( ind(1:par.nblk) ); 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/module_TLapprox.m: -------------------------------------------------------------------------------- 1 | function [ patchRecon, Weight, W] = module_TLapprox(idxBM, keyPatch_arr, patches, Y, param, buffer ) 2 | 3 | patchRecon = zeros(size(patches)); 4 | Weight = zeros(1, size(patches, 2),size(patches, 3)); 5 | % Parameters 6 | func = param.func; % shrinkage 7 | learningIter = param.learningIter; 8 | TLthr0 = param.TLthr0; 9 | dimMatch = param.dimMatch; % transform dim on BM 10 | n2D = size(patches, 1); 11 | n = n2D * dimMatch; 12 | maxNumber=param.maxNumber; 13 | TLstart=1; 14 | currData=0; 15 | numTotal=length(keyPatch_arr); 16 | for k = 1 : numTotal 17 | currData=currData+1; 18 | curTensorInd=idxBM(1:dimMatch, k); 19 | curMatrix = patches(:, curTensorInd); 20 | if param.noiseEstFlag 21 | % noise estimate 22 | noisyMat = Y(:, idxBM(1:dimMatch,k)); 23 | sigma = param.lamada*sqrt(abs(param.nSig^2-mean((curMatrix(:)-noisyMat(:)).^2))); 24 | else 25 | sigma = param.nSig; 26 | end 27 | 28 | miniblocks(:,currData)=curMatrix(:); 29 | if((currData==maxNumber)||k==length(keyPatch_arr)) 30 | if(k==length(keyPatch_arr)) 31 | miniblocks=miniblocks(:,1:currData); 32 | end 33 | 34 | % TL approximate 35 | threshold = sigma * TLthr0; 36 | buffer = ... 37 | TLORTHOpenalty_func(threshold, buffer, miniblocks, param); 38 | denoisedBlock = buffer.blocks; 39 | denoisedBlock(denoisedBlock < 0) = 0; 40 | denoisedBlock(denoisedBlock > 255) = 255; 41 | TLscores = buffer.scores; 42 | denoisedBlock = bsxfun(@times, denoisedBlock, TLscores'); 43 | for jj = TLstart : k 44 | curTensorInd = idxBM(1:dimMatch,jj); 45 | idxMini = jj - TLstart + 1; 46 | curTensor = reshape(denoisedBlock(:, idxMini), n2D, dimMatch); 47 | curWeight = TLscores(idxMini); 48 | patchRecon(:, curTensorInd) = ...s 49 | patchRecon(:, curTensorInd) + curTensor; 50 | Weight(:, curTensorInd) = Weight(:, curTensorInd) + curWeight; 51 | end 52 | TLstart = k + 1; 53 | currData = 0; 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/Image_Denoising.m: -------------------------------------------------------------------------------- 1 | %---------------------------------------------------- 2 | % Blur: 9x9 Uniform kernel; AGWN std Varance = 2^0.5 3 | % Data: May 20th, 2010 4 | % Author: Weisheng Dong, wsdong@mail.xidian.edu.cn 5 | %---------------------------------------------------- 6 | function par = Image_Denoising( nSig, Out_dir, In_dir ) 7 | pre = 'LASSC_'; 8 | par.nSig = nSig; 9 | if nSig<=15 10 | par.win = 6; 11 | par.nblk = 40; 12 | par.c1 = 2.8*sqrt(2); % 1.7 13 | par.lamada = 0.63; 14 | par.w = 0.23; 15 | par.K = 10; 16 | elseif nSig <= 30 17 | par.win = 6; 18 | par.nblk = 60; 19 | par.c1 = 2.9*sqrt(2); % 2.6 20 | par.lamada = 0.65; 21 | par.w = 0.23; 22 | par.K = 6; 23 | elseif nSig<=50 24 | par.win = 8; 25 | par.nblk = 75; 26 | par.c1 = 3.0*sqrt(2); % 2.6 27 | par.lamada = 0.67; 28 | par.w = 0.23; 29 | par.K = 12; 30 | else 31 | par.win = 9; 32 | par.nblk = 90; 33 | par.c1 = 3.1*sqrt(2); % 1.6 34 | par.lamada = 0.64; 35 | par.w = 0.23; 36 | par.K = 14; 37 | end 38 | % par.step = min(6, par.win-1); 39 | par.step=1; 40 | fpath = fullfile(In_dir, '*.png'); 41 | im_dir = dir(fpath); 42 | im_num = length(im_dir); 43 | cnt = 0; 44 | sum_psnr = 0; 45 | sum_ssim = 0; 46 | time0 = clock; 47 | fn_txt = strcat( pre, 'PSNR_SSIM.txt' ); 48 | 49 | for i = 1:im_num 50 | 51 | par.I = double( imread(fullfile(In_dir, im_dir(i).name)) ); 52 | par.nim = par.I + nSig*Gen_noise(In_dir, im_dir, i); 53 | imwrite(par.nim./255,'tmp.tif'); 54 | 55 | [par im PSNR SSIM] = LASSC_Denoising( par ); 56 | sum_psnr = sum_psnr + PSNR; 57 | sum_ssim = sum_ssim + SSIM; 58 | 59 | fname = strcat(pre, im_dir(i).name); 60 | % imwrite(im./255, fullfile(Out_dir, fname)); 61 | disp( sprintf('%s: PSNR = %3.2f SSIM = %f\n', im_dir(i).name, PSNR, SSIM) ); 62 | fprintf('%s : PSNR = %2.2f SSIM = %2.4f\n', im_dir(i).name, PSNR, SSIM); 63 | cnt = cnt + 1; 64 | end 65 | fprintf('\n\nAverage : PSNR = %2.2f SSIM = %2.4f\n', sum_psnr/cnt, sum_ssim/cnt); 66 | disp(sprintf('Total elapsed time = %f min\n', (etime(clock,time0)/60) )); 67 | return; 68 | 69 | 70 | function nim = Gen_noise( In_dir, im_dir, i ) 71 | randn('seed',0); 72 | for ii=1:i 73 | im = imread(fullfile(In_dir, im_dir(ii).name)); 74 | nim = randn(size(im)); 75 | end 76 | return; 77 | 78 | -------------------------------------------------------------------------------- /3DPatch/self_convolution_2d.m: -------------------------------------------------------------------------------- 1 | function [NLBLK, GridX,GridY] = self_convolution_2d(Xr, X, Param, Height, Width) 2 | 3 | %PROCESSSTATIC Summary of this function goes here 4 | % Detailed explanation goes here 5 | % Goal : self-convolution to replace original block matching (searching K most similar patches of each reference patch) 6 | % Inputs: 7 | % 1. Xr : Height*Width*Channels 8 | % 2. X : extracted patches, patch_size*num_patches 9 | % 3. param : parameters 10 | % - PatchNO : number of similar patches (K) 11 | % - step : step of extracting patches 12 | % - PatchSize : size of image patch 13 | % - SearchWin : size of search window 14 | % Output: 15 | % 1. NLBLK : K most similar patches for every reference patch 16 | % within search window, param.PatchNO*num_patches 17 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 18 | 19 | PatchNO = 30; PatchSize = 8; SW = 20; s = 1; 20 | if isfield(Param,'PatchNO'), PatchNO = Param.PatchNO; end 21 | if isfield(Param,'PatchSize'), PatchSize = Param.PatchSize; end 22 | if isfield(Param,'SearchWin'), SW = Param.SearchWin; end 23 | if isfield(Param,'step'), s = Param.step; end 24 | ImClipH = Height - PatchSize +1; 25 | ImClipW = Width - PatchSize +1; 26 | 27 | GridX = 1:s:ImClipH; 28 | GridX = [GridX GridX(end)+1:ImClipH]; 29 | GridY = 1:s:ImClipW; 30 | GridY = [GridY GridY(end)+1:ImClipW]; 31 | 32 | Idx = (1:ImClipH*ImClipW); 33 | Idx = reshape(Idx, ImClipH, ImClipW); 34 | LGridH = length(GridX); 35 | LGridW = length(GridY); 36 | 37 | NLBLK = zeros(PatchNO, LGridH * LGridW); 38 | Patnorm=0.5*vecnorm(X).^2; 39 | for i = 1 : LGridH 40 | for j = 1 : LGridW 41 | x = GridX(i); y = GridY(j); 42 | top = max( x-SW, 1 ); button = min( x+SW, ImClipH ); 43 | left = max( y-SW, 1 ); right = min( y+SW, ImClipW ); 44 | 45 | kk = (j-1)*LGridH + i; 46 | NL_Idx = Idx(top:button, left:right); 47 | NL_Idx = NL_Idx(:); 48 | 49 | RefPatch = X(:,(y-1)*ImClipH + x); 50 | 51 | % 3d search window 52 | tempSW = Xr(top:button+PatchSize-1, left:right+PatchSize-1,:); 53 | 54 | % 3d reference patch 55 | RefPatch = rot90(flipud(reshape(RefPatch,PatchSize,PatchSize,[])),3); 56 | RefPatch = rot90(flip(RefPatch,3),2); 57 | 58 | D=convn(tempSW,RefPatch,"valid"); 59 | SubPatnorm=Patnorm(NL_Idx); 60 | D=SubPatnorm-D(:)'; 61 | 62 | [v, idx] = sort(D); 63 | NLBLK(:,kk) = NL_Idx(idx(1:PatchNO)); 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/self_convolution_2d.m: -------------------------------------------------------------------------------- 1 | function [NLBLK, GridX,GridY] = self_convolution_2d(Xr, X, Param, Height, Width) 2 | 3 | %PROCESSSTATIC Summary of this function goes here 4 | % Detailed explanation goes here 5 | % Goal : self-convolution to replace original block matching (searching K most similar patches of each reference patch) 6 | % Inputs: 7 | % 1. Xr : Height*Width*Channels 8 | % 2. X : extracted patches, patch_size*num_patches 9 | % 3. param : parameters 10 | % - PatchNO : number of similar patches (K) 11 | % - PatchSize : size of image patch 12 | % - SearchWin : size of search window 13 | % - step : step of extracting patches 14 | % Output: 15 | % 1. NLBLK : K most similar patches for every reference patch 16 | % within search window, param.PatchNO*num_patches 17 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 18 | 19 | PatchNO = 30; PatchSize = 8; SW = 20; s = 1; 20 | if isfield(Param,'PatchNO'), PatchNO = Param.PatchNO; end 21 | if isfield(Param,'PatchSize'), PatchSize = Param.PatchSize; end 22 | if isfield(Param,'SearchWin'), SW = Param.SearchWin; end 23 | if isfield(Param,'step'), s = Param.step; end 24 | ImClipH = Height - PatchSize +1; 25 | ImClipW = Width - PatchSize +1; 26 | 27 | GridX = 1:s:ImClipH; 28 | GridX = [GridX GridX(end)+1:ImClipH]; 29 | GridY = 1:s:ImClipW; 30 | GridY = [GridY GridY(end)+1:ImClipW]; 31 | 32 | Idx = (1:ImClipH*ImClipW); 33 | Idx = reshape(Idx, ImClipH, ImClipW); 34 | LGridH = length(GridX); 35 | LGridW = length(GridY); 36 | 37 | NLBLK = zeros(PatchNO, LGridH * LGridW); 38 | Patnorm=0.5*vecnorm(X).^2; 39 | for i = 1 : LGridH 40 | for j = 1 : LGridW 41 | x = GridX(i); y = GridY(j); 42 | top = max( x-SW, 1 ); button = min( x+SW, ImClipH ); 43 | left = max( y-SW, 1 ); right = min( y+SW, ImClipW ); 44 | 45 | kk = (j-1)*LGridH + i; 46 | NL_Idx = Idx(top:button, left:right); 47 | NL_Idx = NL_Idx(:); 48 | 49 | RefPatch = X(:,(y-1)*ImClipH + x); 50 | 51 | % 3d search window 52 | tempSW = Xr(top:button+PatchSize-1, left:right+PatchSize-1,:); 53 | 54 | % 3d reference patch 55 | RefPatch = rot90(flipud(reshape(RefPatch,PatchSize,PatchSize,[])),3); 56 | RefPatch = rot90(flip(RefPatch,3),2); 57 | 58 | D=convn(tempSW,RefPatch,"valid"); 59 | SubPatnorm=Patnorm(NL_Idx); 60 | D=SubPatnorm-D(:)'; 61 | 62 | [v, idx] = sort(D); 63 | NLBLK(:,kk) = NL_Idx(idx(1:PatchNO)); 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/demo_rgbnir_denoising.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | sigma = 10; 3 | load('./data/RGB-NIR/016.mat'); 4 | 5 | % test 6 | imcat_AB = imcat_AB(1:200,1:200,:); 7 | 8 | clean=double(imcat_AB); 9 | [Height, Width, Ch] = size(clean); 10 | 11 | % add noise 12 | randn('seed',0); 13 | noisy = double(clean) + sigma * randn(size(clean)); 14 | 15 | param.searchType = '0'; % 2d patch searching 16 | param.nSig = sigma; 17 | param.patchChannel = Ch; % patch channels 18 | param= getparam(param); 19 | 20 | param.noiseEstFlag = false; 21 | Xr = noisy; 22 | PatchSize = param.PatchSize; 23 | noisyPatch = module_image2patch(noisy, PatchSize); % noisy patches 24 | 25 | 26 | % TL threshold 27 | TLthrList = param.TLthrlist; 28 | 29 | param.learningIter = 20; 30 | param.func = @(X, threshold) sparse_l0(X, threshold); 31 | buffer.YXT=zeros(param.n2D*Ch,param.n2D*Ch); 32 | buffer.D=kron(... 33 | kron(kron(dctmtx(param.PatchSize), dctmtx(param.PatchSize)), dctmtx(Ch)), ... 34 | dctmtx(param.dimMatch)); 35 | tic; 36 | for iter = 1 : param.Iter 37 | % adding back noise -> Input 38 | Xr = Xr + param.delta*(noisy - Xr); 39 | inputPatch = module_image2patch(Xr, PatchSize); % input patch 40 | % estimate noise if iter > 1 41 | % use initial sigma, if iter == 1 42 | if iter > 1 43 | param.noiseEstFlag = true; 44 | end 45 | % every other Innerloop 46 | if (mod(iter-1,param.Innerloop)==0) 47 | param.PatchNO = param.PatchNO - param.PatchNOreduce; 48 | param.PatchNOsqrt = sqrt(param.PatchNO); 49 | [idxBM, GridX,GridY] = self_convolution_2d(Xr,inputPatch, param, Height, Width); 50 | keyPatch_arr = idxBM(1, :); 51 | end 52 | 53 | % LR approx 54 | [patch_LR, weight_LR] = ... 55 | module_LRapprox(inputPatch, idxBM ,noisyPatch, param, GridX, GridY); 56 | [Xr_LR, Weight_LR] = ... 57 | module_patch2image(patch_LR, weight_LR, PatchSize, Height, Width, Ch); 58 | Xr_LR = Xr_LR ./(Weight_LR + eps); 59 | psnr_LR = PSNR3D( clean - Xr_LR); 60 | 61 | % TL approx 62 | param.TLthr0 = TLthrList(iter); 63 | [ patch_TL, weight_TL] = ... 64 | module_TLapprox(idxBM, keyPatch_arr, inputPatch, noisyPatch, param, buffer); 65 | [Xr_TL, Weight_TL] = ... 66 | module_patch2image(patch_TL, weight_TL, PatchSize, Height, Width, Ch); 67 | Xr_TL = Xr_TL ./(Weight_TL + eps); 68 | psnr_TL = PSNR3D( clean - Xr_TL); 69 | 70 | % fusion 71 | Xr = 0.5 * (Xr_LR + Xr_TL); 72 | psnrXr = PSNR3D( clean - Xr); 73 | fprintf( 'Self-MM, iter = %2.2f, PSNR = %2.2f. \n', iter, psnrXr); 74 | end 75 | denoisetime=toc; 76 | %%%%%%%%%%%%%%%%%%%%%%% 77 | fprintf( 'Denoising finish, with iter = %2.2f, and psnr = %2.2f \n', iter, psnrXr); 78 | -------------------------------------------------------------------------------- /3DPatch/demo_Self-MM/getparam.m: -------------------------------------------------------------------------------- 1 | function param = getparam(param) 2 | sigma = param.nSig; 3 | param.SearchWin = 30; % search window size 4 | param.delta = 0.1; % noise adding back ratio 5 | param.Innerloop = 2; % inner loop (control BM freq) size 6 | if param.searchType == '0' % 2d block searching 7 | if sigma<=10 % sigma = 5,10 8 | param.PatchSize = 6; 9 | param.PatchNO = 20; 10 | param.lamada = 0.54; 11 | param.Iter = 1; 12 | param.dimMatch = 4; % TL used #matched 13 | param.TLthrlist = [3.3]; 14 | param.LRthr0 = 1.5; 15 | param.PatchNOreduce = 3; 16 | elseif sigma<=20 % sigma = 15,20 17 | param.PatchSize = 6; 18 | param.PatchNO = 20; 19 | param.lamada = 0.62; 20 | param.Iter = 4; 21 | param.dimMatch = 4; % TL used #matched 22 | param.TLthrlist = [3.0,2.4,2.0,0.8]; 23 | param.LRthr0 = 0.9; 24 | param.PatchNOreduce = 3; 25 | elseif sigma<=50 % sigma = 50 26 | param.PatchSize = 7; 27 | param.dimMatch = 4; 28 | param.PatchNO = 25; 29 | param.Iter = 6; 30 | param.lamada = 0.56; 31 | param.TLthrlist = [3.0, 2.4, 2.0, 1.2, 1, 0.8]; 32 | param.LRthr0 = 0.9; 33 | param.PatchNOreduce = 4; 34 | else 35 | param.PatchSize = 9; 36 | param.PatchNO = 30; 37 | param.Iter = 14; 38 | param.lamada = 0.72; 39 | param.dimMatch = 4; 40 | end 41 | else % 3d block searching 42 | if sigma<=20 % sigma = 5,10,15,20 43 | param.PatchSize = 6; 44 | param.PatchNO = 20; 45 | param.lamada = 0.56; 46 | param.Iter = 4; 47 | param.dimMatch = 3; % TL used #matched 48 | param.TLthrlist = [3.0, 2.4, 2.0, 0.8]; 49 | param.LRthr0 = 1.2; 50 | param.PatchNOreduce = 3; 51 | elseif sigma<=50 % sigma = 50 52 | param.PatchSize = 7; 53 | param.dimMatch = 4; 54 | param.PatchNO = 25; 55 | param.Iter = 6; 56 | param.lamada = 0.54; 57 | param.TLthrlist = [3.0, 2.4, 2.0, 1.2, 1, 0.8]; 58 | param.LRthr0 = 0.9; 59 | param.PatchNOreduce = 4; 60 | else 61 | param.PatchSize = 9; 62 | param.PatchNO = 30; 63 | param.Iter = 14; 64 | param.lamada = 0.72; 65 | param.dimMatch = 4; 66 | end 67 | end 68 | % Param.step = floor((Param.PatchSize)/2-1); 69 | param.step = 1; 70 | 71 | 72 | % TL parameters 73 | param.maxNumber=param.PatchSize*param.PatchSize*10; 74 | param.noisyWeight = 1e-4 / sigma; 75 | 76 | param.alpha=1; 77 | param.isMeanRemoved = true; 78 | param.isRecon = true; 79 | param.sparseWeight = 60; 80 | 81 | param.n2D = param.PatchSize*param.PatchSize*param.dimMatch; 82 | 83 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/LASSC_Denoising.m: -------------------------------------------------------------------------------- 1 | function [par im_out, PSNR, SSIM] = LASSC_Denoising( par ) 2 | time0 = clock; 3 | nim = par.nim; 4 | ori_im = par.I; 5 | b = par.win; 6 | [h w ch] = size(nim); 7 | par.BMtime = 0; 8 | N = h-b+1; 9 | M = w-b+1; 10 | r = [1:N]; 11 | c = [1:M]; 12 | disp(sprintf('PSNR of the noisy image = %f \n', csnr(nim, ori_im, 0, 0) )); 13 | 14 | im_out = nim; 15 | lamada = par.w; 16 | nsig = par.nSig; 17 | 18 | for iter = 1 : par.K 19 | 20 | im_out = im_out + lamada*(nim - im_out); 21 | dif = im_out-nim; 22 | vd = nsig^2-(mean(mean(dif.^2))); 23 | 24 | if (iter==1) 25 | par.nSig = sqrt(abs(vd)); 26 | else 27 | par.nSig = sqrt(abs(vd))*par.lamada; 28 | end 29 | 30 | if (mod(iter,6)==0) || (iter==1) 31 | tic; 32 | blk_arr = self_convolution_2d( im_out, par); 33 | temp=toc; 34 | par.BMtime=par.BMtime+temp; 35 | end 36 | X = Im2Patch( im_out, par ); 37 | 38 | Ys = zeros( size(X) ); 39 | W = zeros( size(X) ); 40 | L = size(blk_arr,2); 41 | for i = 1 : L 42 | B = X(:, blk_arr(:, i)); 43 | mB = repmat(mean( B, 2 ), 1, size(B, 2)); 44 | B = B-mB; 45 | [Ys(:, blk_arr(:,i)), W(:, blk_arr(:,i)), R(i)] = Low_rank_SSC( double(B), par.c1, par.nSig, mB ); 46 | end 47 | %R_save(iter,:)=R; 48 | im_out = zeros(h,w); 49 | im_wei = zeros(h,w); 50 | k = 0; 51 | for i = 1:b 52 | for j = 1:b 53 | k = k+1; 54 | im_out(r-1+i,c-1+j) = im_out(r-1+i,c-1+j) + reshape( Ys(k,:)', [N M]); 55 | im_wei(r-1+i,c-1+j) = im_wei(r-1+i,c-1+j) + reshape( W(k,:)', [N M]); 56 | end 57 | end 58 | im_out = im_out./(im_wei+eps); 59 | 60 | if isfield(par,'I') 61 | PSNR = csnr( im_out, par.I, 0, 0 ); 62 | SSIM = cal_ssim( im_out, par.I, 0, 0 ); 63 | end 64 | 65 | fprintf( 'Iteration %d : nSig = %2.2f, PSNR = %2.2f, SSIM = %2.4f\n', iter, par.nSig, PSNR, SSIM ); 66 | end 67 | if isfield(par,'I') 68 | PSNR = csnr( im_out, par.I, 0, 0 ); 69 | SSIM = cal_ssim( im_out, par.I, 0, 0 ); 70 | end 71 | disp(sprintf('Total elapsed time = %f min\n', (etime(clock,time0)/60) )); 72 | return; 73 | 74 | 75 | 76 | %------------------------------------------------------------------ 77 | % Re-weighted SV Thresholding 78 | % Sigma = argmin || Y-U*Sigma*V' ||^2 + tau * || Sigma ||_* 79 | %------------------------------------------------------------------ 80 | function [X W r] = Low_rank_SSC( Y, c1, nsig, m ) 81 | [U0,Sigma0,V0] = svd(full(Y),'econ'); 82 | Sigma0 = diag(Sigma0); 83 | S = max( Sigma0.^2/size(Y, 2) - nsig^2, 0 ); 84 | thr = c1*nsig^2./ ( sqrt(S) + eps ); 85 | S = soft(Sigma0, thr); 86 | r = sum( S>0 ); 87 | 88 | U = U0(:,1:r); 89 | V = V0(:,1:r); 90 | X = U*diag(S(1:r))*V'; 91 | 92 | if r==size(Y,1) 93 | wei = 1/size(Y,1); 94 | else 95 | wei = (size(Y,1)-r)/size(Y,1); 96 | end 97 | W = wei*ones( size(X) ); 98 | X = (X + m)*wei; 99 | return; 100 | 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Self-Convolution 2 | 3 | Description 4 | ----- 5 | 6 | Self-Convolution is a self-supervised and highly-efficient image operator that exploits non-local similarity. Self-Convolution can generalize many commonly-used non-local schemes, including block matching and non-local means. 7 | 8 | ![avatar](self-conv.png) 9 | 10 | This repo contains the Matlab code package of Self-Convolution which focuses on equivalent implementation of block matching, which includes 2D-patch and 3D-patch versions of Self-Convolution (dimension of the reference image patch). For each version, we provide a demo to show Self-Convolution can speed up the non-local denoising algorithm. To be specific, [SAIST](http://see.xidian.edu.cn/faculty/wsdong/Papers/Journal/TIP_LASSC.pdf) as an example method relying on 2D patches, and our proposed multi-modality image denoising method [Self-MM](https://arxiv.org/abs/2006.13714) as example of 3D patch. 11 | 12 | The Self-Convolution functions can be plugged in any block matching based image restoration method, just follow the similar usage steps. 13 | 14 | Usage 15 | ----- 16 | * 2D Patch 17 | 18 | Example method: [SAIST](http://see.xidian.edu.cn/faculty/wsdong/Papers/Journal/TIP_LASSC.pdf) 19 | 20 | Usage: 21 | 22 | 1. replace `Block_matching.m` function with our `self_convolution_2d.m` function (2d here refers to the two-dimensional search window) 23 | 24 | 2. run `Denoising_Main.m` (a gray-scale image denoising demo) 25 | 26 | * 3D Patch 27 | 28 | Example method: [Self-MM](https://arxiv.org/abs/2006.13714) 29 | 30 | Usage: run `demo_rgbnir_denoising.m` (a RGB-NIR image denoising demo) 31 | 32 | Experimental Results 33 | ----- 34 | Runtime (in seconds) comparisons of non-local algorithms using BM and Self-Convolution, for denoising 512 * 512 single-channel images (first 7 rows) and 256 * 256 * q multi-channel images (last 3 rows), where BMtime\% denotes the runtime portion of BM. 35 | 36 | | Method | Original Runtime | Self-Conv Runtime | BMtime\% |Original BM | Self-Conv | Speed-Ups| 37 | | ---- | ---- | ---- | ---- | ---- | ---- | ---- | 38 | | [SAIST](http://see.xidian.edu.cn/faculty/wsdong/Papers/Journal/TIP_LASSC.pdf) | 708.2 | 562.2 | 32.0\% |227.0 |78.6 |3X | 39 | | [WNNM](https://ieeexplore.ieee.org/document/6909762) | 63.2 | 43.8 | 36.9\% | 23.3 |7.8| 3X | 40 | | [STROLLR](http://ieeexplore.ieee.org/abstract/document/7952566/) | 87.7 | 68.9 | 36.7\% | 38.2 | 13.3 | 3X | 41 | | [GHP](https://www.cv-foundation.org/openaccess/content_cvpr_2013/papers/Zuo_Texture_Enhanced_Image_2013_CVPR_paper.pdf)| 412.6 | 218.3 |69.9\% |288.6 |94.2 |3X| 42 | | [NCSR](http://www4.comp.polyu.edu.hk/~cslzhang/paper/NCSR_TIP_final.pdf) | 134.7 | 82.4 | 57.1\% | 76.9 | 28.1 | 3X | 43 | | [PGPD](http://www4.comp.polyu.edu.hk/~cslzhang/paper/PGPD.pdf) | 305.2 | 89.6 | 85.3\%| 260.3 | 41.3 | 6X| 44 | | [RRC](https://arxiv.org/abs/1807.02504) | 601.2 | 505.6 | 26.9\%| 161.8 | 74.2 | 2X | 45 | | [MCWNNM](http://www4.comp.polyu.edu.hk/~csjunxu/paper/MCWNNM.pdf) | 2899.0 | 2371.3 | 15.8\%| 458.6 | 61.6 | 8X | 46 | |[SALT](http://transformlearning.csl.illinois.edu/assets/Bihan/ConferencePapers/BihanICCV2017salt.pdf) | 375.9 |113.8 |75.4\% |294.8 |33.2 |9X| 47 | | [Self-MM](https://arxiv.org/abs/2006.13714) |139.0| 44.3 |78.8\% |109.5 |16.3 |7X| 48 | 49 | All the experiments are carried out in the Matlab (R2019b) environmentrunning on a PC with Intel(R) Core(TM) i9-10920K CPU 3.50GHz. 50 | 51 | Citation 52 | ----- 53 | Paper available [here](https://ieeexplore.ieee.org/document/9414124). 54 | Long Journal Version [preprint](https://arxiv.org/pdf/2006.13714). 55 | 56 | In case of use, please cite our publication: 57 | 58 | L. Guo, Z. Zha, S. Ravishankar and B. Wen, "Self-Convolution: A Highly-Efficient Operator for Non-Local Image Restoration," ICASSP 2021. 59 | 60 | Bibtex: 61 | ``` 62 | @INPROCEEDINGS{9414124, 63 | author={Guo, Lanqing and Zha, Zhiyuan and Ravishankar, Saiprasad and Wen, Bihan}, 64 | booktitle={ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, 65 | title={Self-Convolution: A Highly-Efficient Operator for Non-Local Image Restoration}, 66 | year={2021}, 67 | volume={}, 68 | number={}, 69 | pages={1860-1864}, 70 | doi={10.1109/ICASSP39728.2021.9414124}} 71 | ``` 72 | ``` 73 | @article{guo2022exploiting, 74 | title={Exploiting Non-Local Priors via Self-Convolution For Highly-Efficient Image Restoration}, 75 | author={Guo, Lanqing and Zha, Zhiyuan and Ravishankar, Saiprasad and Wen, Bihan}, 76 | journal={IEEE Transactions on Image Processing}, 77 | year={2022}, 78 | publisher={IEEE} 79 | } 80 | ``` 81 | -------------------------------------------------------------------------------- /2DPatch/demo_SAIST/cal_ssim.m: -------------------------------------------------------------------------------- 1 | function ssim = cal_ssim( im1, im2, b_row, b_col ) 2 | 3 | [h w ch] = size( im1 ); 4 | ssim = 0; 5 | if ch==1 6 | ssim = ssim_index( im1(b_row+1:h-b_row, b_col+1:w-b_col), im2( b_row+1:h-b_row, b_col+1:w-b_col) ); 7 | else 8 | for i = 1:ch 9 | ssim = ssim + ssim_index( im1(b_row+1:h-b_row, b_col+1:w-b_col, i), im2( b_row+1:h-b_row, b_col+1:w-b_col, i) ); 10 | end 11 | ssim = ssim/3; 12 | end 13 | return; 14 | 15 | 16 | 17 | 18 | function [mssim, ssim_map] = ssim_index(img1, img2, K, window, L) 19 | 20 | %======================================================================== 21 | %SSIM Index, Version 1.0 22 | %Copyright(c) 2003 Zhou Wang 23 | %All Rights Reserved. 24 | % 25 | %The author was with Howard Hughes Medical Institute, and Laboratory 26 | %for Computational Vision at Center for Neural Science and Courant 27 | %Institute of Mathematical Sciences, New York University, USA. He is 28 | %currently with Department of Electrical and Computer Engineering, 29 | %University of Waterloo, Canada. 30 | % 31 | %---------------------------------------------------------------------- 32 | %Permission to use, copy, or modify this software and its documentation 33 | %for educational and research purposes only and without fee is hereby 34 | %granted, provided that this copyright notice and the original authors' 35 | %names appear on all copies and supporting documentation. This program 36 | %shall not be used, rewritten, or adapted as the basis of a commercial 37 | %software or hardware product without first obtaining permission of the 38 | %authors. The authors make no representations about the suitability of 39 | %this software for any purpose. It is provided "as is" without express 40 | %or implied warranty. 41 | %---------------------------------------------------------------------- 42 | % 43 | %This is an implementation of the algorithm for calculating the 44 | %Structural SIMilarity (SSIM) index between two images. Please refer 45 | %to the following paper: 46 | % 47 | %Z. Wang, A. C. Bovik, H. R. Sheikh, and E. P. Simoncelli, "Image 48 | %quality assessment: From error measurement to structural similarity" 49 | %IEEE Transactios on Image Processing, vol. 13, no. 4, Apr. 2004. 50 | % 51 | %Kindly report any suggestions or corrections to zhouwang@ieee.org 52 | % 53 | %---------------------------------------------------------------------- 54 | % 55 | %Input : (1) img1: the first image being compared 56 | % (2) img2: the second image being compared 57 | % (3) K: constants in the SSIM index formula (see the above 58 | % reference). defualt value: K = [0.01 0.03] 59 | % (4) window: local window for statistics (see the above 60 | % reference). default widnow is Gaussian given by 61 | % window = fspecial('gaussian', 11, 1.5); 62 | % (5) L: dynamic range of the images. default: L = 255 63 | % 64 | %Output: (1) mssim: the mean SSIM index value between 2 images. 65 | % If one of the images being compared is regarded as 66 | % perfect quality, then mssim can be considered as the 67 | % quality measure of the other image. 68 | % If img1 = img2, then mssim = 1. 69 | % (2) ssim_map: the SSIM index map of the test image. The map 70 | % has a smaller size than the input images. The actual size: 71 | % size(img1) - size(window) + 1. 72 | % 73 | %Default Usage: 74 | % Given 2 test images img1 and img2, whose dynamic range is 0-255 75 | % 76 | % [mssim ssim_map] = ssim_index(img1, img2); 77 | % 78 | %Advanced Usage: 79 | % User defined parameters. For example 80 | % 81 | % K = [0.05 0.05]; 82 | % window = ones(8); 83 | % L = 100; 84 | % [mssim ssim_map] = ssim_index(img1, img2, K, window, L); 85 | % 86 | %See the results: 87 | % 88 | % mssim %Gives the mssim value 89 | % imshow(max(0, ssim_map).^4) %Shows the SSIM index map 90 | % 91 | %======================================================================== 92 | 93 | 94 | if (nargin < 2 | nargin > 5) 95 | mssim = -Inf; 96 | ssim_map = -Inf; 97 | return; 98 | end 99 | 100 | if (size(img1) ~= size(img2)) 101 | mssim = -Inf; 102 | ssim_map = -Inf; 103 | return; 104 | end 105 | 106 | [M N] = size(img1); 107 | 108 | if (nargin == 2) 109 | if ((M < 11) | (N < 11)) 110 | mssim = -Inf; 111 | ssim_map = -Inf; 112 | return 113 | end 114 | window = fspecial('gaussian', 11, 1.5); % 115 | K(1) = 0.01; % default settings 116 | K(2) = 0.03; % 117 | L = 255; % 118 | end 119 | 120 | if (nargin == 3) 121 | if ((M < 11) | (N < 11)) 122 | mssim = -Inf; 123 | ssim_map = -Inf; 124 | return 125 | end 126 | window = fspecial('gaussian', 11, 1.5); 127 | L = 255; 128 | if (length(K) == 2) 129 | if (K(1) < 0 | K(2) < 0) 130 | mssim = -Inf; 131 | ssim_map = -Inf; 132 | return; 133 | end 134 | else 135 | mssim = -Inf; 136 | ssim_map = -Inf; 137 | return; 138 | end 139 | end 140 | 141 | if (nargin == 4) 142 | [H W] = size(window); 143 | if ((H*W) < 4 | (H > M) | (W > N)) 144 | mssim = -Inf; 145 | ssim_map = -Inf; 146 | return 147 | end 148 | L = 255; 149 | if (length(K) == 2) 150 | if (K(1) < 0 | K(2) < 0) 151 | mssim = -Inf; 152 | ssim_map = -Inf; 153 | return; 154 | end 155 | else 156 | mssim = -Inf; 157 | ssim_map = -Inf; 158 | return; 159 | end 160 | end 161 | 162 | if (nargin == 5) 163 | [H W] = size(window); 164 | if ((H*W) < 4 | (H > M) | (W > N)) 165 | mssim = -Inf; 166 | ssim_map = -Inf; 167 | return 168 | end 169 | if (length(K) == 2) 170 | if (K(1) < 0 | K(2) < 0) 171 | mssim = -Inf; 172 | ssim_map = -Inf; 173 | return; 174 | end 175 | else 176 | mssim = -Inf; 177 | ssim_map = -Inf; 178 | return; 179 | end 180 | end 181 | 182 | C1 = (K(1)*L)^2; 183 | C2 = (K(2)*L)^2; 184 | window = window/sum(sum(window)); 185 | img1 = double(img1); 186 | img2 = double(img2); 187 | 188 | mu1 = filter2(window, img1, 'valid'); 189 | mu2 = filter2(window, img2, 'valid'); 190 | mu1_sq = mu1.*mu1; 191 | mu2_sq = mu2.*mu2; 192 | mu1_mu2 = mu1.*mu2; 193 | sigma1_sq = filter2(window, img1.*img1, 'valid') - mu1_sq; 194 | sigma2_sq = filter2(window, img2.*img2, 'valid') - mu2_sq; 195 | sigma12 = filter2(window, img1.*img2, 'valid') - mu1_mu2; 196 | 197 | if (C1 > 0 & C2 > 0) 198 | ssim_map = ((2*mu1_mu2 + C1).*(2*sigma12 + C2))./((mu1_sq + mu2_sq + C1).*(sigma1_sq + sigma2_sq + C2)); 199 | else 200 | numerator1 = 2*mu1_mu2 + C1; 201 | numerator2 = 2*sigma12 + C2; 202 | denominator1 = mu1_sq + mu2_sq + C1; 203 | denominator2 = sigma1_sq + sigma2_sq + C2; 204 | ssim_map = ones(size(mu1)); 205 | index = (denominator1.*denominator2 > 0); 206 | ssim_map(index) = (numerator1(index).*numerator2(index))./(denominator1(index).*denominator2(index)); 207 | index = (denominator1 ~= 0) & (denominator2 == 0); 208 | ssim_map(index) = numerator1(index)./denominator1(index); 209 | end 210 | 211 | mssim = mean2(ssim_map); 212 | 213 | return --------------------------------------------------------------------------------