├── Methods-Codes ├── Affined_shapemask │ └── 00757_affinedshape.mat ├── Affined_shapemaskplus │ └── 00757_affinedshape.mat ├── All_GAT.m ├── All_GAT_plus.m ├── All_exp3.m ├── All_exp3_plus.m ├── Copy_of_test_GAT.m ├── GAT.m ├── LevelSet.m ├── Output_PortraitFCN │ └── 00757_output.mat ├── Output_PortraitFCNplus │ └── 00757_output.mat ├── RSF.m ├── TTTT.m ├── copy_test_imgs.m ├── demo_GAT.m ├── exp1.m ├── exp2_LevelSet.m ├── exp2_RSF.m ├── exp3_LevelSet_GAT.m ├── exp3_LevelSet_GAT_plus.m ├── exp3_RSF_plus.m ├── exp4_LevelSet_GAT.m ├── exp4_RSF.m ├── exp5_LevelSet_GAT_g.m ├── exp5_RSF.m ├── exp6_LevelSet_GAT.m ├── exp6_RSF.m ├── images_data_crop │ ├── 00174.jpg │ └── 00757.jpg ├── images_mask │ └── 00757_mask.mat ├── imgs_test │ └── 00757.jpg ├── meanmask.png ├── optimalAffine.m ├── portraitFCN.ipynb ├── temp.m ├── test_FCN8s.py ├── test_GAT.m ├── test_portraitFCN.py └── test_portraitFCNplus.py ├── README.md └── figs ├── 6.png ├── Experiment.png ├── Experiment1.png ├── Experiment2_1.png ├── Experiment2_2.png ├── Flow Diagram.png ├── GAT.png └── Paper.png /Methods-Codes/Affined_shapemask/00757_affinedshape.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/Affined_shapemask/00757_affinedshape.mat -------------------------------------------------------------------------------- /Methods-Codes/Affined_shapemaskplus/00757_affinedshape.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/Affined_shapemaskplus/00757_affinedshape.mat -------------------------------------------------------------------------------- /Methods-Codes/All_GAT.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/All_GAT.m -------------------------------------------------------------------------------- /Methods-Codes/All_GAT_plus.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/All_GAT_plus.m -------------------------------------------------------------------------------- /Methods-Codes/All_exp3.m: -------------------------------------------------------------------------------- 1 | clear;clc; 2 | All_IoU = []; 3 | All_IoU_Portrait = []; 4 | 5 | load('../../data/testlist.mat') 6 | for index=1:length(testlist) 7 | imgindex = testlist(index); 8 | if exist(['./Output_PortraitFCN/' sprintf('%05d',imgindex) '_output.mat'],'file') 9 | img = imread(['../../data/images_data_crop/' sprintf('%05d',imgindex) '.jpg']); 10 | load(['../../data/images_mask/' sprintf('%05d', imgindex) '_mask.mat']); % mask 11 | load(['./Output_PortraitFCN/' sprintf('%05d',imgindex) '_output.mat']); % res 12 | load(['./Affined_shapemask/' sprintf('%05d',imgindex) '_affinedshape.mat']); % affined_mask 13 | img = rgb2gray(img); 14 | img = double(img); 15 | %% 16 | T_res_1(:,:) = double(res(1,:,:)); 17 | T_res_2(:,:) = double(res(2,:,:)); 18 | T_res_c_1 = 1./(1+exp(T_res_2 - T_res_1)); 19 | T_res_c_2 = 1./(1+exp(T_res_1 - T_res_2)); 20 | diffs = exp(T_res_2 - T_res_1); 21 | finalres = diffs./(1+diffs); 22 | %% 23 | c0 = 200; 24 | initLSF = -(finalres*2*c0 - c0); 25 | T_initLSF = initLSF; 26 | smoothRatio = 1; 27 | initLSF = filter2(fspecial('average',smoothRatio), initLSF); % 平滑 28 | P1 = filter2(fspecial('average',smoothRatio), T_res_c_2); 29 | P2 = filter2(fspecial('average',smoothRatio), T_res_c_1); 30 | u = initLSF; 31 | %% 32 | iterNum = 200; 33 | timestep = .2; 34 | epsilon = 0.5; 35 | sigma = 4.0; 36 | lambda1 = 20.0; 37 | lambda2 = 20.0; 38 | rho1 = 0.05; 39 | rho2 = 0.05; 40 | pi1 = 2 * 500.0; 41 | pi2 = 2 * 500.0; 42 | nu = 0.005*255*255; % length 43 | mu = 1.0; % regularization 44 | %% 45 | K = fspecial('gaussian', round(2*sigma)*2+1, sigma); 46 | I = img; 47 | KI = conv2(img, K, 'same'); 48 | KONE = conv2(ones(size(img)), K, 'same'); 49 | %% 50 | for n = 1:iterNum 51 | u = exp2_RSF(u, I, K, KI, KONE, P1, P2, timestep, epsilon, lambda1, lambda2, rho1, rho2, pi1, pi2, affined_mask, 1-affined_mask, nu, mu, 1); 52 | end 53 | %% 54 | finalreg1 = double(u<0); 55 | IoU = sum(finalreg1(:).*mask(:))/sum(double((finalreg1(:)+mask(:))>0)); 56 | finalreg2 = double(finalres>0.5); 57 | originalIoU = sum(finalreg2(:).*mask(:))/sum(double((finalreg2(:)+mask(:))>0)); 58 | fprintf('%05d - Level set: %f\n', imgindex, IoU); 59 | fprintf('%05d - PortraitFCN: %f\n', imgindex, originalIoU); 60 | All_IoU = [All_IoU IoU]; 61 | All_IoU_Portrait = [All_IoU_Portrait originalIoU]; 62 | %% 63 | fprintf('%05d OK\n', imgindex); 64 | end 65 | end 66 | fprintf('Mean Level Set: %f\n', mean(All_IoU)); 67 | fprintf('Mean PortraitFCN: %f\n', mean(All_IoU_Portrait)); 68 | save('./IoU_PortraitFCN/IoU.mat', 'All_IoU', 'All_IoU_Portrait'); -------------------------------------------------------------------------------- /Methods-Codes/All_exp3_plus.m: -------------------------------------------------------------------------------- 1 | clear;clc; 2 | All_IoU = []; 3 | All_IoU_Portrait = []; 4 | 5 | load('../../data/testlist.mat') 6 | for index=1:length(testlist) 7 | imgindex = testlist(index); 8 | if exist(['./Output_PortraitFCNplus/' sprintf('%05d',imgindex) '_output.mat'],'file') 9 | img = imread(['../../data/images_data_crop/' sprintf('%05d',imgindex) '.jpg']); 10 | load(['../../data/images_mask/' sprintf('%05d', imgindex) '_mask.mat']); % mask 11 | load(['./Output_PortraitFCNplus/' sprintf('%05d',imgindex) '_output.mat']); % res 12 | load(['./Affined_shapemaskplus/' sprintf('%05d',imgindex) '_affinedshape.mat']); % affined_mask 13 | img = rgb2gray(img); 14 | img = double(img); 15 | %% 16 | T_res_1(:,:) = double(res(1,:,:)); 17 | T_res_2(:,:) = double(res(2,:,:)); 18 | T_res_c_1 = 1./(1+exp(T_res_2 - T_res_1)); 19 | T_res_c_2 = 1./(1+exp(T_res_1 - T_res_2)); 20 | diffs = exp(T_res_2 - T_res_1); 21 | finalres = diffs./(1+diffs); 22 | %% 23 | c0 = 200; 24 | initLSF = -(finalres*2*c0 - c0); 25 | T_initLSF = initLSF; 26 | smoothRatio = 1; 27 | initLSF = filter2(fspecial('average',smoothRatio), initLSF); % 平滑 28 | P1 = filter2(fspecial('average',smoothRatio), T_res_c_2); 29 | P2 = filter2(fspecial('average',smoothRatio), T_res_c_1); 30 | u = initLSF; 31 | %% 32 | iterNum = 200; 33 | timestep = 2; 34 | epsilon = 0.5; 35 | sigma = 4.0; 36 | lambda1 = 20.0; 37 | lambda2 = 20.0; 38 | rho1 = 0.05; 39 | rho2 = 0.05; 40 | pi1 = 2 * 500.0; 41 | pi2 = 2 * 500.0; 42 | nu = 0.005*255*255; % length 43 | mu = 0.2/timestep; % regularization 44 | %% 45 | K = fspecial('gaussian', round(2*sigma)*2+1, sigma); 46 | I = img; 47 | KI = conv2(img, K, 'same'); 48 | KONE = conv2(ones(size(img)), K, 'same'); 49 | %% 50 | for n = 1:iterNum 51 | u = exp3_RSF_plus(u, I, K, KI, KONE, P1, P2, timestep, epsilon, lambda1, lambda2, rho1, rho2, pi1, pi2, affined_mask, 1-affined_mask, nu, mu, 1); 52 | end 53 | %% 54 | finalreg1 = double(u<0); 55 | IoU = sum(finalreg1(:).*mask(:))/sum(double((finalreg1(:)+mask(:))>0)); 56 | finalreg2 = double(finalres>0.5); 57 | originalIoU = sum(finalreg2(:).*mask(:))/sum(double((finalreg2(:)+mask(:))>0)); 58 | fprintf('%05d - Level set: %f\n', imgindex, IoU); 59 | fprintf('%05d - PortraitFCN: %f\n', imgindex, originalIoU); 60 | All_IoU = [All_IoU IoU]; 61 | All_IoU_Portrait = [All_IoU_Portrait originalIoU]; 62 | %% 63 | fprintf('%05d OK\n', imgindex); 64 | end 65 | end 66 | fprintf('Mean Level Set: %f\n', mean(All_IoU)); 67 | fprintf('Mean PortraitFCN+: %f\n', mean(All_IoU_Portrait)); 68 | save('./IoU_PortraitFCNplus/IoU.mat', 'All_IoU', 'All_IoU_Portrait'); -------------------------------------------------------------------------------- /Methods-Codes/Copy_of_test_GAT.m: -------------------------------------------------------------------------------- 1 | clear;clc; 2 | imgindex = 715; % 83 193 429 3 | img = imread(['./images_data_crop/' sprintf('%05d',imgindex) '.jpg']); 4 | load(['./images_mask/' sprintf('%05d', imgindex) '_mask.mat']); 5 | load(['./Output_PortraitFCN/' sprintf('%05d',imgindex) '_output.mat']); 6 | load(['./Affined_shapemask/' sprintf('%05d',imgindex) '_affinedshape.mat']); 7 | shapemask = imread('meanmask.png'); 8 | shapemask = double(shapemask)./255; 9 | %% 10 | T_res_1(:,:) = double(res(1,:,:)); 11 | T_res_2(:,:) = double(res(2,:,:)); 12 | T_res_c_1 = 1./(1+exp(T_res_2 - T_res_1)); 13 | T_res_c_2 = 1./(1+exp(T_res_1 - T_res_2)); 14 | %% 15 | B = filter2(fspecial('average',150), double(affined_mask>0.6)); 16 | A = T_res_c_2.*B; 17 | %% 18 | subplot(1,5,1);imshow(img); 19 | subplot(1,5,2);imshow(mask); 20 | subplot(1,5,3);imshow(T_res_c_2); 21 | subplot(1,5,4);imshow(affined_mask); 22 | subplot(1,5,5);imshow(A); 23 | %% 24 | % 1061 25 | % 1056 26 | % 1048 27 | % 1026 28 | % 1009 29 | % 996 x 30 | % 994 31 | % 968 32 | % 931 33 | % 930 34 | % 919 35 | % 917 36 | % 876 37 | % 851 38 | % 839 39 | % 65 40 | % 136 41 | % 174 ** 42 | 43 | %% 44 | % finalreg = double(T_res_c_2>0.5); 45 | % B = labeloverlay(img, finalreg); 46 | % figure;imshow(B) -------------------------------------------------------------------------------- /Methods-Codes/GAT.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/GAT.m -------------------------------------------------------------------------------- /Methods-Codes/LevelSet.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/LevelSet.m -------------------------------------------------------------------------------- /Methods-Codes/Output_PortraitFCN/00757_output.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/Output_PortraitFCN/00757_output.mat -------------------------------------------------------------------------------- /Methods-Codes/Output_PortraitFCNplus/00757_output.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/Output_PortraitFCNplus/00757_output.mat -------------------------------------------------------------------------------- /Methods-Codes/RSF.m: -------------------------------------------------------------------------------- 1 | function u = RSF(u0, Img, Ksigma, KI, KONE, P1, P2, timestep, epsilon, lambda1, lambda2, rho1, rho2, pi1, pi2, nu, mu, numIter) 2 | u = u0; 3 | for j = 1:numIter 4 | u = NeumannBoundCond(u); 5 | K = curvature_central(u); 6 | 7 | DrcU = (epsilon/pi)./(epsilon^2. + u.^2); 8 | [f1, f2] = localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon); 9 | 10 | s1 = lambda1.*rho1.*P1 .* f1 .^2 - lambda2.*rho2.*P2 .* f2 .^ 2; 11 | s2 = lambda1.*rho1.*P1 .* f1 - lambda2.*rho2.*P2 .* f2; 12 | % dataForce = (lambda1 - lambda2) * KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 13 | dataForce = (lambda1.*rho1.*P1 - lambda2.*rho2.*P2) .* KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 14 | 15 | D = -DrcU .* (pi1 .* P1 - pi2 .* P2); 16 | 17 | A = -DrcU .* dataForce; 18 | P = mu * (4 * del2(u) - K); 19 | L = nu .* DrcU .* K; 20 | u = u + timestep * (L + P + A); 21 | 22 | end 23 | 24 | 25 | function [f1, f2]= localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon) 26 | % compute f1 and f2 27 | Hu=0.5*(1+(2/pi)*atan(u./epsilon)); % eq.(8) 28 | I=Img.*Hu; 29 | c1=conv2(Hu,Ksigma,'same'); 30 | c2=conv2(I,Ksigma,'same'); % the numerator of eq.(14) for i = 1 31 | f1=c2./(c1); % compute f1 according to eq.(14) for i = 1 32 | f2=(KI-c2)./(KONE-c1); 33 | 34 | function g = NeumannBoundCond(f) 35 | % Neumann boundary condition 36 | [nrow,ncol] = size(f); 37 | g = f; 38 | g([1 nrow],[1 ncol]) = g([3 nrow-2],[3 ncol-2]); 39 | g([1 nrow],2:end-1) = g([3 nrow-2],2:end-1); 40 | g(2:end-1,[1 ncol]) = g(2:end-1,[3 ncol-2]); 41 | 42 | function k = curvature_central(u) 43 | % compute curvature 44 | [ux,uy] = gradient(u); 45 | normDu = sqrt(ux.^2+uy.^2+1e-10); % the norm of the gradient plus a small possitive number 46 | % to avoid division by zero in the following computation. 47 | Nx = ux./normDu; 48 | Ny = uy./normDu; 49 | [nxx,junk] = gradient(Nx); 50 | [junk,nyy] = gradient(Ny); 51 | k = nxx+nyy; % compute divergence -------------------------------------------------------------------------------- /Methods-Codes/TTTT.m: -------------------------------------------------------------------------------- 1 | load('./Output_FCN8s/00688_output.mat') 2 | tsum = sum(exp(res), 1); 3 | T_res_1 = exp(res(21,:,:))./tsum; 4 | T_1(:,:) = T_res_1(1,:,:); 5 | TT = T_1./max(max(T_1)); 6 | imshow(TT) -------------------------------------------------------------------------------- /Methods-Codes/copy_test_imgs.m: -------------------------------------------------------------------------------- 1 | clear 2 | clc 3 | load('../../data/testlist.mat') 4 | for i=1:length(testlist) 5 | if exist(['../../data/portraitFCN_data/' sprintf('%05d',testlist(i)) '.mat'],'file') 6 | copyfile(['../../data/images_data_crop/' sprintf('%05d',testlist(i)) '.jpg'], './imgs_test/') 7 | end 8 | end -------------------------------------------------------------------------------- /Methods-Codes/demo_GAT.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/demo_GAT.m -------------------------------------------------------------------------------- /Methods-Codes/exp1.m: -------------------------------------------------------------------------------- 1 | clear;clc; 2 | imgindex = 312; 3 | img = imread(['./images_data_crop/' sprintf('%05d',imgindex) '.jpg']); 4 | load(['./Output_PortraitFCN/' sprintf('%05d',imgindex) '_output.mat']); 5 | 6 | T_res_1(:,:) = res(1,:,:); 7 | T_res_2(:,:) = res(2,:,:); 8 | res_d = res(2,:,:) - res(1,:,:); 9 | T_res_d(:,:) = res_d(1,:,:); 10 | 11 | img = rgb2gray(img); 12 | img = double(img)/255; 13 | imshow(img); 14 | S = exp(T_res_1)+exp(T_res_2); 15 | T_res_c_1 = exp(T_res_1)./S; 16 | T_res_c_2 = exp(T_res_2)./S; 17 | T_res_c_2(isnan(T_res_c_2)) = 1; 18 | figure; 19 | subplot(1,2,1);imagesc(T_res_c_2); 20 | subplot(1,2,2);imagesc(T_res_c_1); 21 | 22 | figure; 23 | subplot(1,2,1);imshow(img .* T_res_c_2); 24 | subplot(1,2,2);imshow(img .* T_res_c_1); -------------------------------------------------------------------------------- /Methods-Codes/exp2_LevelSet.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/exp2_LevelSet.m -------------------------------------------------------------------------------- /Methods-Codes/exp2_RSF.m: -------------------------------------------------------------------------------- 1 | function u = exp2_RSF(u0, Img, Ksigma, KI, KONE, P1, P2, timestep, epsilon, lambda1, lambda2, rho1, rho2, pi1, pi2, mask1, mask2, nu, mu, numIter) 2 | u = u0; 3 | for j = 1:numIter 4 | u = NeumannBoundCond(u); 5 | K = curvature_central(u); 6 | 7 | DrcU = (epsilon/pi)./(epsilon^2. + u.^2); 8 | [f1, f2] = localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon); 9 | 10 | s1 = lambda1.*rho1.*P1 .* f1 .^2 - lambda2.*rho2.*P2 .* f2 .^ 2; 11 | s2 = lambda1.*rho1.*P1 .* f1 - lambda2.*rho2.*P2 .* f2; 12 | % dataForce = (lambda1 - lambda2) * KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 13 | dataForce = (lambda1.*rho1.*P1 - lambda2.*rho2.*P2) .* KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 14 | 15 | D = -DrcU .* (pi1 .* mask1 - pi2 .*mask2); 16 | 17 | A = -DrcU .* dataForce; 18 | P = mu * (4 * del2(u) - K); 19 | L = nu .* DrcU .* K; 20 | u = u + timestep * (L + P + A + D); 21 | 22 | end 23 | 24 | 25 | function [f1, f2]= localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon) 26 | % compute f1 and f2 27 | Hu=0.5*(1+(2/pi)*atan(u./epsilon)); % eq.(8) 28 | I=Img.*Hu; 29 | c1=conv2(Hu,Ksigma,'same'); 30 | c2=conv2(I,Ksigma,'same'); % the numerator of eq.(14) for i = 1 31 | f1=c2./(c1); % compute f1 according to eq.(14) for i = 1 32 | f2=(KI-c2)./(KONE-c1); 33 | 34 | function g = NeumannBoundCond(f) 35 | % Neumann boundary condition 36 | [nrow,ncol] = size(f); 37 | g = f; 38 | g([1 nrow],[1 ncol]) = g([3 nrow-2],[3 ncol-2]); 39 | g([1 nrow],2:end-1) = g([3 nrow-2],2:end-1); 40 | g(2:end-1,[1 ncol]) = g(2:end-1,[3 ncol-2]); 41 | 42 | function k = curvature_central(u) 43 | % compute curvature 44 | [ux,uy] = gradient(u); 45 | normDu = sqrt(ux.^2+uy.^2+1e-10); % the norm of the gradient plus a small possitive number 46 | % to avoid division by zero in the following computation. 47 | Nx = ux./normDu; 48 | Ny = uy./normDu; 49 | [nxx,junk] = gradient(Nx); 50 | [junk,nyy] = gradient(Ny); 51 | k = nxx+nyy; % compute divergence -------------------------------------------------------------------------------- /Methods-Codes/exp3_LevelSet_GAT.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/exp3_LevelSet_GAT.m -------------------------------------------------------------------------------- /Methods-Codes/exp3_LevelSet_GAT_plus.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/exp3_LevelSet_GAT_plus.m -------------------------------------------------------------------------------- /Methods-Codes/exp3_RSF_plus.m: -------------------------------------------------------------------------------- 1 | function u = exp3_RSF_plus(u0, Img, Ksigma, KI, KONE, P1, P2, timestep, epsilon, lambda1, lambda2, rho1, rho2, pi1, pi2, mask1, mask2, nu, mu, numIter) 2 | u = u0; 3 | for j = 1:numIter 4 | u = NeumannBoundCond(u); 5 | K = curvature_central(u); 6 | 7 | DrcU = (epsilon/pi)./(epsilon^2. + u.^2); 8 | [f1, f2] = localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon); 9 | 10 | s1 = lambda1.*rho1.*P1 .* f1 .^2 - lambda2.*rho2.*P2 .* f2 .^ 2; 11 | s2 = lambda1.*rho1.*P1 .* f1 - lambda2.*rho2.*P2 .* f2; 12 | % dataForce = (lambda1 - lambda2) * KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 13 | dataForce = (lambda1.*rho1.*P1 - lambda2.*rho2.*P2) .* KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 14 | 15 | D = -DrcU .* (pi1 .* mask1.*P1 - pi2 .* mask2.*P2); 16 | 17 | A = -DrcU .* dataForce; 18 | P = mu * (4 * del2(u) - K); 19 | L = nu .* DrcU .* K; 20 | u = u + timestep * (L + P + A + D); 21 | 22 | end 23 | 24 | 25 | function [f1, f2]= localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon) 26 | % compute f1 and f2 27 | Hu=0.5*(1+(2/pi)*atan(u./epsilon)); % eq.(8) 28 | I=Img.*Hu; 29 | c1=conv2(Hu,Ksigma,'same'); 30 | c2=conv2(I,Ksigma,'same'); % the numerator of eq.(14) for i = 1 31 | f1=c2./(c1); % compute f1 according to eq.(14) for i = 1 32 | f2=(KI-c2)./(KONE-c1); 33 | 34 | function g = NeumannBoundCond(f) 35 | % Neumann boundary condition 36 | [nrow,ncol] = size(f); 37 | g = f; 38 | g([1 nrow],[1 ncol]) = g([3 nrow-2],[3 ncol-2]); 39 | g([1 nrow],2:end-1) = g([3 nrow-2],2:end-1); 40 | g(2:end-1,[1 ncol]) = g(2:end-1,[3 ncol-2]); 41 | 42 | function k = curvature_central(u) 43 | % compute curvature 44 | [ux,uy] = gradient(u); 45 | normDu = sqrt(ux.^2+uy.^2+1e-10); % the norm of the gradient plus a small possitive number 46 | % to avoid division by zero in the following computation. 47 | Nx = ux./normDu; 48 | Ny = uy./normDu; 49 | [nxx,junk] = gradient(Nx); 50 | [junk,nyy] = gradient(Ny); 51 | k = nxx+nyy; % compute divergence -------------------------------------------------------------------------------- /Methods-Codes/exp4_LevelSet_GAT.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/exp4_LevelSet_GAT.m -------------------------------------------------------------------------------- /Methods-Codes/exp4_RSF.m: -------------------------------------------------------------------------------- 1 | function u = exp4_RSF(u0, Img, Ksigma, KI, KONE, P1, P2, timestep, epsilon, lambda1, lambda2, rho1, rho2, pi1, pi2, mask1, mask2, nu, mu, numIter) 2 | u = u0; 3 | for j = 1:numIter 4 | u = NeumannBoundCond(u); 5 | K = curvature_central(u); 6 | 7 | DrcU = (epsilon/pi)./(epsilon^2. + u.^2); 8 | [f1, f2] = localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon); 9 | 10 | s1 = lambda1.*rho1.*(P1) .* f1 .^2 - lambda2.*rho2.*(P2) .* f2 .^ 2; 11 | s2 = lambda1.*rho1.*(P1) .* f1 - lambda2.*rho2.*(P2) .* f2; 12 | % dataForce = (lambda1 - lambda2) * KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 13 | dataForce = (lambda1.*rho1.*(P1) - lambda2.*rho2.*(P2)) .* KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 14 | 15 | D = -DrcU .* (pi1 .* (mask1) - pi2 .* (mask2)); 16 | 17 | A = -DrcU .* dataForce; 18 | P = mu * (4 * del2(u) - K); 19 | L = nu .* DrcU .* K; 20 | u = u + timestep * (L + P + A + D); 21 | 22 | end 23 | 24 | 25 | function [f1, f2]= localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon) 26 | % compute f1 and f2 27 | Hu=0.5*(1+(2/pi)*atan(u./epsilon)); % eq.(8) 28 | I=Img.*Hu; 29 | c1=conv2(Hu,Ksigma,'same'); 30 | c2=conv2(I,Ksigma,'same'); % the numerator of eq.(14) for i = 1 31 | f1=c2./(c1); % compute f1 according to eq.(14) for i = 1 32 | f2=(KI-c2)./(KONE-c1); 33 | 34 | function g = NeumannBoundCond(f) 35 | % Neumann boundary condition 36 | [nrow,ncol] = size(f); 37 | g = f; 38 | g([1 nrow],[1 ncol]) = g([3 nrow-2],[3 ncol-2]); 39 | g([1 nrow],2:end-1) = g([3 nrow-2],2:end-1); 40 | g(2:end-1,[1 ncol]) = g(2:end-1,[3 ncol-2]); 41 | 42 | function k = curvature_central(u) 43 | % compute curvature 44 | [ux,uy] = gradient(u); 45 | normDu = sqrt(ux.^2+uy.^2+1e-10); % the norm of the gradient plus a small possitive number 46 | % to avoid division by zero in the following computation. 47 | Nx = ux./normDu; 48 | Ny = uy./normDu; 49 | [nxx,junk] = gradient(Nx); 50 | [junk,nyy] = gradient(Ny); 51 | k = nxx+nyy; % compute divergence -------------------------------------------------------------------------------- /Methods-Codes/exp5_LevelSet_GAT_g.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/exp5_LevelSet_GAT_g.m -------------------------------------------------------------------------------- /Methods-Codes/exp5_RSF.m: -------------------------------------------------------------------------------- 1 | function phi = exp5_RSF(phi_0, g, P1, P2, mask1, mask2, mu, timestep, epsilon, lambda, alfa, pi1, pi2, numIter, potentialFunction) 2 | phi=phi_0; 3 | [vx, vy] = gradient(g); 4 | for j = 1:numIter 5 | phi=NeumannBoundCond(phi); 6 | [phi_x,phi_y]=gradient(phi); 7 | s=sqrt(phi_x.^2 + phi_y.^2); 8 | smallNumber=1e-10; 9 | Nx=phi_x./(s+smallNumber); % add a small positive number to avoid division by zero 10 | Ny=phi_y./(s+smallNumber); 11 | curvature=div(Nx,Ny); 12 | if strcmp(potentialFunction,'single-well') 13 | distRegTerm = 4*del2(phi)-curvature; % compute distance regularization term in equation (13) with the single-well potential p1. 14 | elseif strcmp(potentialFunction,'double-well') 15 | distRegTerm=distReg_p2(phi); % compute the distance regularization term in eqaution (13) with the double-well potential p2. 16 | else 17 | disp('Error: Wrong choice of potential function. Please input the string "single-well" or "double-well" in the drlse_edge function.'); 18 | end 19 | diracPhi=Dirac(phi,epsilon); 20 | areaTerm=diracPhi.*g; % balloon/pressure force 21 | edgeTerm=diracPhi.*(vx.*Nx+vy.*Ny) + diracPhi.*g.*curvature; 22 | D = -diracPhi.*(pi1.*P1.*mask1 - pi2.*P2.*mask2); 23 | phi=phi + timestep*(mu*distRegTerm + lambda*edgeTerm + alfa*areaTerm); 24 | end 25 | 26 | 27 | function f = distReg_p2(phi) 28 | % compute the distance regularization term with the double-well potential p2 in eqaution (16) 29 | [phi_x,phi_y]=gradient(phi); 30 | s=sqrt(phi_x.^2 + phi_y.^2); 31 | a=(s>=0) & (s<=1); 32 | b=(s>1); 33 | ps=a.*sin(2*pi*s)/(2*pi)+b.*(s-1); % compute first order derivative of the double-well potential p2 in eqaution (16) 34 | dps=((ps~=0).*ps+(ps==0))./((s~=0).*s+(s==0)); % compute d_p(s)=p'(s)/s in equation (10). As s-->0, we have d_p(s)-->1 according to equation (18) 35 | f = div(dps.*phi_x - phi_x, dps.*phi_y - phi_y) + 4*del2(phi); 36 | 37 | function f = div(nx,ny) 38 | [nxx,junk]=gradient(nx); 39 | [junk,nyy]=gradient(ny); 40 | f=nxx+nyy; 41 | 42 | function f = Dirac(x, sigma) 43 | f=(1/2/sigma)*(1+cos(pi*x/sigma)); 44 | b = (x<=sigma) & (x>=-sigma); 45 | f = f.*b; 46 | 47 | function g = NeumannBoundCond(f) 48 | % Make a function satisfy Neumann boundary condition 49 | [nrow,ncol] = size(f); 50 | g = f; 51 | g([1 nrow],[1 ncol]) = g([3 nrow-2],[3 ncol-2]); 52 | g([1 nrow],2:end-1) = g([3 nrow-2],2:end-1); 53 | g(2:end-1,[1 ncol]) = g(2:end-1,[3 ncol-2]); % compute divergence -------------------------------------------------------------------------------- /Methods-Codes/exp6_LevelSet_GAT.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/exp6_LevelSet_GAT.m -------------------------------------------------------------------------------- /Methods-Codes/exp6_RSF.m: -------------------------------------------------------------------------------- 1 | function u = exp6_RSF(u0, Img, Ksigma, KI, KONE,timestep, epsilon, lambda1, lambda2, pi1, pi2, mask1, mask2, nu, mu, numIter) 2 | u = u0; 3 | for j = 1:numIter 4 | u = NeumannBoundCond(u); 5 | K = curvature_central(u); 6 | 7 | DrcU = (epsilon/pi)./(epsilon^2. + u.^2); 8 | [f1, f2] = localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon); 9 | 10 | s1 = lambda1.* f1 .^2 - lambda2.* f2 .^ 2; 11 | s2 = lambda1 .* f1 - lambda2.* f2; 12 | % dataForce = (lambda1 - lambda2) * KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 13 | dataForce = (lambda1 - lambda2) .* KONE .* Img .* Img + conv2(s1, Ksigma, 'same') - 2.* Img .* conv2(s2, Ksigma, 'same'); 14 | 15 | D = -DrcU .* (pi1 .* (mask1) - pi2 .* (mask2)); 16 | 17 | A = -DrcU .* dataForce; 18 | P = mu * (4 * del2(u) - K); 19 | L = nu .* DrcU .* K; 20 | u = u + timestep * (L + P + A + D); 21 | 22 | end 23 | 24 | 25 | function [f1, f2]= localBinaryFit(Img, u, KI, KONE, Ksigma, epsilon) 26 | % compute f1 and f2 27 | Hu=0.5*(1+(2/pi)*atan(u./epsilon)); % eq.(8) 28 | I=Img.*Hu; 29 | c1=conv2(Hu,Ksigma,'same'); 30 | c2=conv2(I,Ksigma,'same'); % the numerator of eq.(14) for i = 1 31 | f1=c2./(c1); % compute f1 according to eq.(14) for i = 1 32 | f2=(KI-c2)./(KONE-c1); 33 | 34 | function g = NeumannBoundCond(f) 35 | % Neumann boundary condition 36 | [nrow,ncol] = size(f); 37 | g = f; 38 | g([1 nrow],[1 ncol]) = g([3 nrow-2],[3 ncol-2]); 39 | g([1 nrow],2:end-1) = g([3 nrow-2],2:end-1); 40 | g(2:end-1,[1 ncol]) = g(2:end-1,[3 ncol-2]); 41 | 42 | function k = curvature_central(u) 43 | % compute curvature 44 | [ux,uy] = gradient(u); 45 | normDu = sqrt(ux.^2+uy.^2+1e-10); % the norm of the gradient plus a small possitive number 46 | % to avoid division by zero in the following computation. 47 | Nx = ux./normDu; 48 | Ny = uy./normDu; 49 | [nxx,junk] = gradient(Nx); 50 | [junk,nyy] = gradient(Ny); 51 | k = nxx+nyy; % compute divergence -------------------------------------------------------------------------------- /Methods-Codes/images_data_crop/00174.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/images_data_crop/00174.jpg -------------------------------------------------------------------------------- /Methods-Codes/images_data_crop/00757.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/images_data_crop/00757.jpg -------------------------------------------------------------------------------- /Methods-Codes/images_mask/00757_mask.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/images_mask/00757_mask.mat -------------------------------------------------------------------------------- /Methods-Codes/imgs_test/00757.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/imgs_test/00757.jpg -------------------------------------------------------------------------------- /Methods-Codes/meanmask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/Methods-Codes/meanmask.png -------------------------------------------------------------------------------- /Methods-Codes/optimalAffine.m: -------------------------------------------------------------------------------- 1 | function [affined_mask] = optimalAffine(imask, iprobmap, mask) 2 | fun = @(x)objectFun(x, imask, iprobmap); 3 | options = optimoptions('particleswarm','SwarmSize',50, 'MaxIterations', 50, 'MinNeighborsFraction', 1); 4 | lb = [0 0 0 0 -300 -300]; 5 | ub = [300 300 300 300 300]; 6 | [x, fval, exitflag] = particleswarm(fun, 6, lb, ub, options); 7 | T = [x(1) x(2) 0; x(3) x(4) 0; x(5) x(6) 1]; 8 | tform = maketform('affine' ,T); 9 | affined_mask = imtransform(tform, mask); 10 | end 11 | 12 | function O = objectFun(x, mask, probmap) 13 | A = [x(1) x(2); x(3) x(4)]; 14 | b = [x(5); x(6)]; 15 | affined = A * mask + b; 16 | [m1, n1] = size(affined); 17 | [m2, n2] = size(probmap); 18 | D = zeros(n1, n2); 19 | for i = 1:n1 20 | for j = 1:n2 21 | D(i,j) = norm(affined(:,i) - probmap(:,j)); 22 | end 23 | end 24 | mina = min(D,[],2); 25 | minp = min(D); 26 | O = mean(mina) + mean(minp); 27 | end -------------------------------------------------------------------------------- /Methods-Codes/portraitFCN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "caffe_root = '../caffe-portraitseg/'\n", 11 | "import sys\n", 12 | "sys.path.insert(0, caffe_root + 'python')" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "import numpy as np\n", 22 | "from matplotlib import pyplot as plt\n", 23 | "import caffe\n", 24 | "caffe.set_mode_cpu()" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "MODEL_FILE = './our_models/deploy_3channels.prototxt'\n", 34 | "PRETRAINED = './our_models/bgr.caffemodel'" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "net = caffe.Net(MODEL_FILE, PRETRAINED, caffe.TEST)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "net.blobs['data'].reshape(1, 3, 800, 600)" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "# create transformer for the input called 'data'\n", 62 | "transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})\n", 63 | "\n", 64 | "transformer.set_transpose('data', (2,0,1)) # move image channels to outermost dimension\n", 65 | "transformer.set_mean('data', np.asarray([122.675, 116.669, 114.008])) # subtract the dataset-mean value in each channel\n", 66 | "transformer.set_raw_scale('data', 255) # rescale from [0, 1] to [0, 255]\n", 67 | "transformer.set_channel_swap('data', (2,1,0)) # swap channels from RGB to BGR" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "image = caffe.io.load_image('00321.jpg')" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "transformed_image = transformer.preprocess('data', image)\n", 86 | "transformed_image /= 255" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "net.blobs['data'].data[...] = transformed_image" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "output = net.forward() " 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "res = output['upscore']" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "res = res.reshape(res.shape[1:])" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [ 131 | "diffs = np.exp(res[1,:,:] - res[0,:,:])" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "finalres = diffs/(1+diffs)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "markdown", 145 | "metadata": {}, 146 | "source": [ 147 | "---" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 1, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "import scipy.io as scio" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 2, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "testlistPath = '../data/testlist.mat'\n", 166 | "outputPath = './Output_PortraitFCN/'" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 3, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "testlist = scio.loadmat(testlistPath)['testlist']" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "for i in range(len(testlist)):\n", 185 | " image = scio.loadmat('../data/portraitFCN_data/%05d.mat' % testlist[0][i])['img']\n", 186 | " image = image.transpose((2,0,1))\n", 187 | " net.blobs['data'].data[...] = image\n", 188 | " output = net.forward() \n", 189 | " res = output['upscore']\n", 190 | " res = res.reshape(res.shape[1:])\n", 191 | " scio.savemat(outputPath + '%05d_output.mat'% testlist[0][i], {'res':res})" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 12, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "image = scio.loadmat('../data/portraitFCN_data/%05d.mat' % testlist[0][0])['img']" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 13, 206 | "metadata": {}, 207 | "outputs": [ 208 | { 209 | "data": { 210 | "text/plain": [ 211 | "(800, 600, 3)" 212 | ] 213 | }, 214 | "execution_count": 13, 215 | "metadata": {}, 216 | "output_type": "execute_result" 217 | } 218 | ], 219 | "source": [ 220 | "image.shape" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": 15, 226 | "metadata": {}, 227 | "outputs": [ 228 | { 229 | "data": { 230 | "text/plain": [ 231 | "array([[[-0.1961098 , -0.20003137, -0.19218824, ..., -0.11767843,\n", 232 | " -0.11375686, -0.11375686],\n", 233 | " [-0.1961098 , -0.1961098 , -0.19218824, ..., -0.11767843,\n", 234 | " -0.11375686, -0.11375686],\n", 235 | " [-0.18826667, -0.19218824, -0.19218824, ..., -0.11375686,\n", 236 | " -0.11375686, -0.11375686],\n", 237 | " ..., \n", 238 | " [ 0.2038902 , 0.2038902 , 0.19996863, ..., 0.23134118,\n", 239 | " 0.23134118, 0.23134118],\n", 240 | " [ 0.2038902 , 0.19996863, 0.19996863, ..., 0.23134118,\n", 241 | " 0.23134118, 0.23134118],\n", 242 | " [ 0.19996863, 0.19996863, 0.19996863, ..., 0.23134118,\n", 243 | " 0.23134118, 0.23134118]],\n", 244 | "\n", 245 | " [[-0.21046667, -0.21438824, -0.21438824, ..., -0.18301569,\n", 246 | " -0.17517255, -0.17517255],\n", 247 | " [-0.21046667, -0.21046667, -0.21438824, ..., -0.18301569,\n", 248 | " -0.17517255, -0.17517255],\n", 249 | " [-0.20262353, -0.2065451 , -0.2065451 , ..., -0.17909412,\n", 250 | " -0.17517255, -0.17517255],\n", 251 | " ..., \n", 252 | " [ 0.13855294, 0.13855294, 0.13463137, ..., 0.16600392,\n", 253 | " 0.16600392, 0.16600392],\n", 254 | " [ 0.13855294, 0.13463137, 0.13463137, ..., 0.16600392,\n", 255 | " 0.16600392, 0.16600392],\n", 256 | " [ 0.13463137, 0.13463137, 0.13463137, ..., 0.16600392,\n", 257 | " 0.16600392, 0.16600392]],\n", 258 | "\n", 259 | " [[-0.19480392, -0.19872549, -0.20264706, ..., -0.18696078,\n", 260 | " -0.19088235, -0.19088235],\n", 261 | " [-0.19480392, -0.19480392, -0.20264706, ..., -0.18696078,\n", 262 | " -0.19088235, -0.19088235],\n", 263 | " [-0.18696078, -0.19088235, -0.19088235, ..., -0.18303922,\n", 264 | " -0.19088235, -0.19088235],\n", 265 | " ..., \n", 266 | " [ 0.15813725, 0.15813725, 0.15421569, ..., 0.16205882,\n", 267 | " 0.16205882, 0.16205882],\n", 268 | " [ 0.15813725, 0.15421569, 0.15421569, ..., 0.16205882,\n", 269 | " 0.16205882, 0.16205882],\n", 270 | " [ 0.15421569, 0.15421569, 0.15421569, ..., 0.16205882,\n", 271 | " 0.16205882, 0.16205882]]])" 272 | ] 273 | }, 274 | "execution_count": 15, 275 | "metadata": {}, 276 | "output_type": "execute_result" 277 | } 278 | ], 279 | "source": [ 280 | "image.transpose((2,0,1))" 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": null, 286 | "metadata": {}, 287 | "outputs": [], 288 | "source": [] 289 | } 290 | ], 291 | "metadata": { 292 | "kernelspec": { 293 | "display_name": "Python 2", 294 | "language": "python", 295 | "name": "python2" 296 | }, 297 | "language_info": { 298 | "codemirror_mode": { 299 | "name": "ipython", 300 | "version": 3 301 | }, 302 | "file_extension": ".py", 303 | "mimetype": "text/x-python", 304 | "name": "python", 305 | "nbconvert_exporter": "python", 306 | "pygments_lexer": "ipython3", 307 | "version": "3.6.5" 308 | } 309 | }, 310 | "nbformat": 4, 311 | "nbformat_minor": 2 312 | } 313 | -------------------------------------------------------------------------------- /Methods-Codes/temp.m: -------------------------------------------------------------------------------- 1 | clear;clc; 2 | imgindex = 492; 3 | load(['./images_mask/' sprintf('%05d', imgindex) '_mask.mat']); 4 | load(['./Output_PortraitFCNplus/' sprintf('%05d',imgindex) '_output.mat']); 5 | shapemask = imread('meanmask.png'); 6 | shapemask = double(shapemask)./255; 7 | %% 8 | T_res_1(:,:) = double(res(1,:,:)); 9 | T_res_2(:,:) = double(res(2,:,:)); 10 | T_res_c_1 = 1./(1+exp(T_res_2 - T_res_1)); 11 | T_res_c_2 = 1./(1+exp(T_res_1 - T_res_2)); 12 | %% 13 | TEMP = ones(size(mask)); 14 | figure;imshow(T_res_c_2); 15 | figure; 16 | imshow(TEMP);hold on;axis off,axis equal; 17 | contour(T_res_c_2,[0.5 0.5],'b');hold off; 18 | figure; 19 | imshow(TEMP);hold on;axis off,axis equal; 20 | contour(shapemask,[0.6 0.6],'b');hold off; 21 | -------------------------------------------------------------------------------- /Methods-Codes/test_FCN8s.py: -------------------------------------------------------------------------------- 1 | import os 2 | caffe_root = '../../caffe-portraitseg/' 3 | import sys 4 | sys.path.insert(0, caffe_root + 'python') 5 | 6 | import numpy as np 7 | from matplotlib import pyplot as plt 8 | import caffe 9 | caffe.set_mode_cpu() 10 | 11 | MODEL_FILE = '../../training/FCN8s_models/fcn-8s-pascal-deploy.prototxt' 12 | PRETRAINED = '../../training/FCN8s_models/fcn-8s-pascal.caffemodel' 13 | net = caffe.Net(MODEL_FILE, PRETRAINED, caffe.TEST) 14 | 15 | net.blobs['data'].reshape(1, 3, 800, 600) 16 | 17 | 18 | import scipy.io as scio 19 | 20 | testlistPath = '../../data/testlist.mat' 21 | outputPath = './Output_FCN8s/' 22 | 23 | testlist = scio.loadmat(testlistPath)['testlist'][0] 24 | 25 | for i in range(len(testlist)): 26 | if os.path.exists('../../data/portraitFCN_data/%05d.mat' % testlist[i]) is True: 27 | image = scio.loadmat('../../data/portraitFCN_data/%05d.mat' % testlist[i])['img'] 28 | image = image.transpose((2,0,1)) 29 | net.blobs['data'].data[...] = image 30 | output = net.forward() 31 | res = output['upscore'] 32 | res = res.reshape(res.shape[1:]) 33 | scio.savemat(outputPath + '%05d_output.mat'% testlist[i], {'res':res}) 34 | print('%d OK' % testlist[i]) 35 | -------------------------------------------------------------------------------- /Methods-Codes/test_GAT.m: -------------------------------------------------------------------------------- 1 | clear;clc; 2 | imgindex = 174; 3 | img = imread(['./images_data_crop/' sprintf('%05d',imgindex) '.jpg']); 4 | load(['./data/images_mask/' sprintf('%05d', imgindex) '_mask.mat']); 5 | load(['./Output_PortraitFCN/' sprintf('%05d',imgindex) '_output.mat']); 6 | load(['./Affined_shapemask/' sprintf('%05d',imgindex) '_affinedshape.mat']); 7 | shapemask = imread('meanmask.png'); 8 | shapemask = double(shapemask)./255; 9 | %% 10 | T_res_1(:,:) = double(res(1,:,:)); 11 | T_res_2(:,:) = double(res(2,:,:)); 12 | T_res_c_1 = 1./(1+exp(T_res_2 - T_res_1)); 13 | T_res_c_2 = 1./(1+exp(T_res_1 - T_res_2)); 14 | %% 15 | subplot(1,5,1);imshow(img); 16 | subplot(1,5,2);imshow(mask); 17 | subplot(1,5,3);imshow(T_res_c_2); 18 | subplot(1,5,4);imshow(affined_mask); 19 | subplot(1,5,5);imshow(T_res_c_2.*affined_mask); -------------------------------------------------------------------------------- /Methods-Codes/test_portraitFCN.py: -------------------------------------------------------------------------------- 1 | import os 2 | caffe_root = '../../caffe-portraitseg/' 3 | import sys 4 | sys.path.insert(0, caffe_root + 'python') 5 | 6 | import numpy as np 7 | from matplotlib import pyplot as plt 8 | import caffe 9 | caffe.set_mode_cpu() 10 | 11 | MODEL_FILE = '../our_models/deploy_3channels.prototxt' 12 | PRETRAINED = '../our_models/bgr.caffemodel' 13 | net = caffe.Net(MODEL_FILE, PRETRAINED, caffe.TEST) 14 | 15 | net.blobs['data'].reshape(1, 3, 800, 600) 16 | 17 | 18 | import scipy.io as scio 19 | 20 | testlistPath = '../../data/testlist.mat' 21 | outputPath = './Output_PortraitFCN/' 22 | 23 | testlist = scio.loadmat(testlistPath)['testlist'][0] 24 | 25 | for i in range(len(testlist)): 26 | if os.path.exists('../../data/portraitFCN_data/%05d.mat' % testlist[i]) is True: 27 | image = scio.loadmat('../../data/portraitFCN_data/%05d.mat' % testlist[i])['img'] 28 | image = image.transpose((2,0,1)) 29 | net.blobs['data'].data[...] = image 30 | output = net.forward() 31 | res = output['upscore'] 32 | res = res.reshape(res.shape[1:]) 33 | scio.savemat(outputPath + '%05d_output.mat'% testlist[i], {'res':res}) 34 | print('%d OK' % testlist[i]) 35 | 36 | 37 | 38 | 39 | ''' 40 | image = scio.loadmat('../../data/portraitFCN_data/%05d.mat' % testlist[0][0])['img'] 41 | image = image.transpose((2,0,1)) 42 | net.blobs['data'].data[...] = image 43 | output = net.forward() 44 | res = output['upscore'] 45 | res = res.reshape(res.shape[1:]) 46 | diffs = np.exp(res[1,:,:] - res[0,:,:]) 47 | finalres = diffs/(1+diffs) 48 | plt.imshow(finalres) 49 | plt.show() 50 | scio.savemat(outputPath + '%05d_output.mat'% testlist[0][0], {'res':res}) 51 | ''' 52 | -------------------------------------------------------------------------------- /Methods-Codes/test_portraitFCNplus.py: -------------------------------------------------------------------------------- 1 | import os 2 | caffe_root = '../../caffe-portraitseg/' 3 | import sys 4 | sys.path.insert(0, caffe_root + 'python') 5 | 6 | import numpy as np 7 | from matplotlib import pyplot as plt 8 | import caffe 9 | caffe.set_mode_cpu() 10 | 11 | MODEL_FILE = '../our_models/deploy_6channels.prototxt' 12 | PRETRAINED = '../our_models/bgr_mmask_xy.caffemodel' 13 | net = caffe.Net(MODEL_FILE, PRETRAINED, caffe.TEST) 14 | 15 | net.blobs['data'].reshape(1, 6, 800, 600) 16 | 17 | 18 | import scipy.io as scio 19 | 20 | testlistPath = '../../data/testlist.mat' 21 | outputPath = './Output_PortraitFCNplus/' 22 | 23 | testlist = scio.loadmat(testlistPath)['testlist'][0] 24 | 25 | for i in range(len(testlist)): 26 | if os.path.exists('../../data/portraitFCN+_data/%05d.mat' % testlist[i]) is True: 27 | image = scio.loadmat('../../data/portraitFCN+_data/%05d.mat' % testlist[i])['img'] 28 | image = image.transpose((2,0,1)) 29 | net.blobs['data'].data[...] = image 30 | output = net.forward() 31 | res = output['upscore'] 32 | res = res.reshape(res.shape[1:]) 33 | scio.savemat(outputPath + '%05d_output.mat'% testlist[i], {'res':res}) 34 | print('%d OK' % testlist[i]) 35 | 36 | 37 | 38 | 39 | ''' 40 | image = scio.loadmat('../../data/portraitFCN_data/%05d.mat' % testlist[0][0])['img'] 41 | image = image.transpose((2,0,1)) 42 | net.blobs['data'].data[...] = image 43 | output = net.forward() 44 | res = output['upscore'] 45 | res = res.reshape(res.shape[1:]) 46 | diffs = np.exp(res[1,:,:] - res[0,:,:]) 47 | finalres = diffs/(1+diffs) 48 | plt.imshow(finalres) 49 | plt.show() 50 | scio.savemat(outputPath + '%05d_output.mat'% testlist[0][0], {'res':res}) 51 | ''' 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Level Set based Shape Prior and Deep Learning for Image Segmentation 2 | Open codes for paper "Level Set based Shape Prior and Deep Learning for Image Segmentation" 3 | 4 | ![Paper](https://github.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/blob/master/figs/Paper.png?raw=true) 5 | ## Abstract 6 | Deep convolutional neural network (DCNN) can effectively extract the hidden patterns in images and learn realistic image priors from the training set. And fully convolutional networks (FCNs) have achieved state-of-the-art performance in the task of the image segmentation. However, these methods have the disadvantages of the noise, boundary roughness and no prior shape. Therefore, this paper proposes a level set with the deep prior method for the image segmentation based on the priors learned by FCNs from the training set. The output of the FCNs is treated as a probability map and the global affine transformation (GAT) is used to obtain the optimal affine transformation of the intrinsic prior shape at a specific image. And then, the level set method is used to integrate the information of the original image, the probability map and the corrected prior shape to achieve the image segmentation. Compared with the traditional level set method for images of simple scenes, the proposed method combines the traditional level set method with FCNs and corrected prior shape to solve the disadvantage of FCNs, making the level set method applicable to the image segmentation of complex scenes. Finally, a series of experiments with Portrait data set are used to verify the effectiveness of the proposed method. The experimental results show that the proposed method can obtain more accurate segmentation results than the traditional FCNs. 7 | 8 | 9 | ## The Level Set with The Deep Prior Method 10 | In order to improve the performance of FCNs and complete objective segmentation of the complex sense by using the level set method, the level set with the deep prior method is proposed as shown in Fig. \ref{fig: Flow diagram}. 11 | 12 | ![The Flow Diagram](https://github.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/blob/master/figs/Flow%20Diagram.png?raw=true) 13 | 14 | The output of FCNs is taken as a probability map that each pixel belongs to a different category. The segmentation shape represented by the probability map is noisy, but it still retains a large part of the correct segmentation. Therefore, an optimal affine transformation of the standard shape mask (the shape prior) of the image can be obtained based on the "probability" shape with the GAT method. Finally, the image, the probability map and the affine mask are used as the input of the level set method to implement the image segmentation. 15 | 16 | After training on the training set, FCNs can extract features of the target and learn patterns of the target, and these capabilities are represented in the probability map. Therefore, the probability map preserves the information of the segmentation target based on the pixels in the receptive field. For example, in the Portrait data set, the probability map can represent the probability that each pixel belongs to a person. Although there is a certain probability which is incorrect, it still keeps most of the correct predictions, even the correct patterns information. Based on these properties of the probability map, it is possible to use the method of combining the probability map with the shape priors and the level set method for the semantic segmentation. And then how to get the optimal affine transformation of the standard shape prior using the GAT. 17 | 18 | ![GAT](https://github.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/blob/master/figs/GAT.png?raw=true) 19 | 20 | ## Results 21 | ![Results](https://github.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/blob/master/figs/6.png?raw=true) 22 | 23 | ## Experiments 24 | ![Experiments](https://github.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/blob/master/figs/Experiment.png?raw=true) 25 | ### Experiment 1 26 | ![Experiment 1](https://github.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/blob/master/figs/Experiment1.png?raw=true) 27 | 28 | ### Experiment 2 29 | ![Experiment 2 1](https://github.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/blob/master/figs/Experiment2_1.png?raw=true) 30 | 31 | ![Experiment 2 2](https://github.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/blob/master/figs/Experiment2_2.png?raw=true) 32 | 33 | ## Reference 34 | 35 | - [PortraitFCN](http://xiaoyongshen.me/webpage_portrait/index.html) 36 | - [Global Affine Transformation](https://ieeexplore.ieee.org/abstract/document/735806/) 37 | - [Level Set](http://www.engr.uconn.edu/~cmli/) 38 | 39 | -------------------------------------------------------------------------------- /figs/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/figs/6.png -------------------------------------------------------------------------------- /figs/Experiment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/figs/Experiment.png -------------------------------------------------------------------------------- /figs/Experiment1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/figs/Experiment1.png -------------------------------------------------------------------------------- /figs/Experiment2_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/figs/Experiment2_1.png -------------------------------------------------------------------------------- /figs/Experiment2_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/figs/Experiment2_2.png -------------------------------------------------------------------------------- /figs/Flow Diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/figs/Flow Diagram.png -------------------------------------------------------------------------------- /figs/GAT.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/figs/GAT.png -------------------------------------------------------------------------------- /figs/Paper.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zsh965866221/LevelSet-ShapePrior-DeepLearning/2236dadf14cae5351c40ed53d175753b611190b5/figs/Paper.png --------------------------------------------------------------------------------