├── Code ├── CalMAE.m ├── Enhancedmeasure.m ├── Fmeasure_calu.m ├── Plot_curve.m ├── ReadMe.txt ├── S_object.m ├── S_region.m ├── StructureMeasure.m ├── WFb.m ├── WFb.m~ ├── WeiFm_Sm.m ├── calImageMean.m ├── gather_the_results.m ├── linspecer.m ├── main.m ├── main_LF.m └── plot_WeiFm_Sm.m ├── Dataset ├── EORSSD │ └── GT │ │ └── 0004.png └── ORSSD │ └── GT │ └── 0005.png ├── README.md └── SalMap └── MCCNet ├── EORSSD └── 0004.png └── ORSSD └── 0005.png /Code/CalMAE.m: -------------------------------------------------------------------------------- 1 | function mae = CalMAE(smap, gtImg) 2 | % Code Author: Wangjiang Zhu 3 | % Email: wangjiang88119@gmail.com 4 | % Date: 3/24/2014 5 | if size(smap, 1) ~= size(gtImg, 1) || size(smap, 2) ~= size(gtImg, 2) 6 | error('Saliency map and gt Image have different sizes!\n'); 7 | end 8 | 9 | if ~islogical(gtImg) 10 | gtImg = gtImg(:,:,1) > 128; 11 | end 12 | 13 | smap = im2double(smap(:,:,1)); 14 | fgPixels = smap(gtImg); 15 | fgErrSum = length(fgPixels) - sum(fgPixels); 16 | bgErrSum = sum(smap(~gtImg)); 17 | mae = (fgErrSum + bgErrSum) / numel(gtImg); -------------------------------------------------------------------------------- /Code/Enhancedmeasure.m: -------------------------------------------------------------------------------- 1 | function [score]= Emeasure(FM,GT) 2 | % Emeasure Compute the Enhanced Alignment measure (as proposed in "Enhanced-alignment 3 | % Measure for Binary Foreground Map Evaluation" [Deng-Ping Fan et. al - IJCAI'18 oral paper]) 4 | % Usage: 5 | % score = Emeasure(FM,GT) 6 | % Input: 7 | % FM - Binary foreground map. Type: double. 8 | % GT - Binary ground truth. Type: double. 9 | % Output: 10 | % score - The Enhanced alignment score 11 | 12 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%Important Note:%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 13 | %The code is for academic purposes only. Please cite this paper if you make use of it: 14 | 15 | %@conference{Fan2018Enhanced, title={Enhanced-alignment Measure for Binary Foreground Map Evaluation}, 16 | % author={Fan, Deng-Ping and Gong, Cheng and Cao, Yang and Ren, Bo and Cheng, Ming-Ming and Borji, Ali}, 17 | % year = {2018}, 18 | % booktitle = {IJCAI} 19 | % } 20 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 21 | 22 | FM = logical(FM); 23 | GT = logical(GT); 24 | 25 | %Use double for computations. 26 | dFM = double(FM); 27 | dGT = double(GT); 28 | 29 | %Special case: 30 | if (sum(dGT(:))==0)% if the GT is completely black 31 | enhanced_matrix = 1.0 - dFM; %only calculate the black area of intersection 32 | elseif(sum(~dGT(:))==0)%if the GT is completely white 33 | enhanced_matrix = dFM; %only calcualte the white area of intersection 34 | else 35 | %Normal case: 36 | 37 | %1.compute alignment matrix 38 | align_matrix = AlignmentTerm(dFM,dGT); 39 | %2.compute enhanced alignment matrix 40 | enhanced_matrix = EnhancedAlignmentTerm(align_matrix); 41 | end 42 | 43 | %3.Emeasure score 44 | [w,h] = size(GT); 45 | score = sum(enhanced_matrix(:))./(w*h - 1 + eps); 46 | end 47 | 48 | % Alignment Term 49 | function [align_Matrix] = AlignmentTerm(dFM,dGT) 50 | 51 | %compute global mean 52 | mu_FM = mean2(dFM); 53 | mu_GT = mean2(dGT); 54 | 55 | %compute the bias matrix 56 | align_FM = dFM - mu_FM; 57 | align_GT = dGT - mu_GT; 58 | 59 | %compute alignment matrix 60 | align_Matrix = 2.*(align_GT.*align_FM)./(align_GT.*align_GT + align_FM.*align_FM + eps); 61 | 62 | end 63 | 64 | % Enhanced Alignment Term function. f(x) = 1/4*(1 + x)^2) 65 | function enhanced = EnhancedAlignmentTerm(align_Matrix) 66 | enhanced = ((align_Matrix + 1).^2)/4; 67 | end 68 | 69 | 70 | -------------------------------------------------------------------------------- /Code/Fmeasure_calu.m: -------------------------------------------------------------------------------- 1 | %% 2 | function [PreFtem, RecallFtem, FmeasureF] = Fmeasure_calu(sMap,gtMap,gtsize, threshold) 3 | %threshold = 2* mean(sMap(:)) ; 4 | if ( threshold > 1 ) 5 | threshold = 1; 6 | end 7 | 8 | Label3 = zeros( gtsize ); 9 | Label3( sMap>=threshold ) = 1; 10 | 11 | NumRec = length( find( Label3==1 ) ); 12 | LabelAnd = Label3 & gtMap; 13 | NumAnd = length( find ( LabelAnd==1 ) ); 14 | num_obj = sum(sum(gtMap)); 15 | 16 | if NumAnd == 0 17 | PreFtem = 0; 18 | RecallFtem = 0; 19 | FmeasureF = 0; 20 | else 21 | PreFtem = NumAnd/NumRec; 22 | RecallFtem = NumAnd/num_obj; 23 | FmeasureF = ( ( 1.3* PreFtem * RecallFtem ) / ( .3 * PreFtem + RecallFtem ) ); 24 | end 25 | 26 | %Fmeasure = [PreFtem, RecallFtem, FmeasureF]; 27 | 28 | -------------------------------------------------------------------------------- /Code/Plot_curve.m: -------------------------------------------------------------------------------- 1 | clear all; close all; clc; 2 | 3 | %SaliencyMap Path 4 | SalMapPath = '../SalMap/'; 5 | %Models = {'LHM','DESM','CDB','ACSD','GP','LBE','CDCP','SE','MDSF','DF','CTMF','PDNet','PCF'};%'TPF' 6 | %Models = {'PCF','PDNet','CTMF','DF','MDSF','SE','CDCP','LBE','GP','ACSD','CDB','DESM','LHM'}; 7 | %Models = {'SSAV'}; 8 | Models = {'CPFP','MMCI','TANet','LGY'}; 9 | %Deep Model 10 | %DeepModels = {'DF','PDNet','CTMF','PCF'};%TPF-missing, MDSF-svm, 11 | 12 | modelNum = length(Models); 13 | groupNum = floor(modelNum/3)+1; 14 | 15 | %Datasets 16 | DataPath = '../Dataset/'; 17 | %Datasets = {'GIT'}; 18 | %Datasets = {'STERE','DES','NLPR','SSD','LFSD','GIT'};%'NJU2K', 19 | %Datasets = {'GIT','SSD','DES'}; 20 | %Datasets = {'SSD','DES','LFSD','STERE','NJU2K','GIT','NLPR','SIP'}; 21 | % Datasets = {'SSD','DES','LFSD','STERE'}; 22 | Datasets = {'NJU2K'}; 23 | 24 | %Results 25 | ResDir = '/home/lgy/flownet2-master/RGBD_MODEL/test/RGBDBenchmark-EvaluationTools/Result_overall/'; 26 | 27 | method_colors = linspecer(groupNum); 28 | 29 | % colors 30 | %str=['r','r','r','g','g','g','b','b','b','c','c','c','m','m','m','y','y','y','k','k','k','g','g','b','b','m','m','k','k','r','r','b','b','c','c','m','m']; 31 | 32 | str=['r','r','g','g','b','b','y','y','c','c','k','k','m','m']; 33 | %str=['r','g','b','y','c','m','k','r','g','b','y','c','m','k']; 34 | 35 | 36 | datasetNum = length(Datasets); 37 | for d = 1:datasetNum 38 | close all; 39 | dataset = Datasets{d}; 40 | fprintf('Processing %d/%d: %s Dataset\n',d,datasetNum,dataset); 41 | 42 | matPath = [ResDir dataset '-mat/']; 43 | plotMetrics = gather_the_results(modelNum, matPath, Models); 44 | 45 | %% plot the PR curves 46 | figure(1); 47 | hold on; 48 | grid on; 49 | axis([0 1 0 1]); 50 | title(dataset); 51 | xlabel('Recall'); 52 | ylabel('Precision'); 53 | 54 | for i = 1 : length(plotMetrics.Alg_names) 55 | if mod(i,2)==0 56 | plot(plotMetrics.Recall(:,i), plotMetrics.Pre(:,i), '--', 'Color', str(i), 'LineWidth', 2); 57 | elseif mod(i,2)==1 58 | plot(plotMetrics.Recall(:,i), plotMetrics.Pre(:,i), 'Color', str(i), 'LineWidth', 2); 59 | end 60 | [~,max_idx] = max(plotMetrics.Fmeasure_Curve(:,i)); 61 | h1 = plot(plotMetrics.Recall(max_idx,i), plotMetrics.Pre(max_idx,i), '.', 'Color', str(i),'markersize',20); 62 | set(get(get(h1,'Annotation'),'LegendInformation'),'IconDisplayStyle','off'); 63 | end 64 | legend(plotMetrics.Alg_names,'Location','SouthWest'); 65 | set(gcf,'position',[0 600 560 420]); 66 | 67 | figPath = [ResDir 'Curve/']; 68 | if ~exist(figPath,'dir') 69 | mkdir(figPath); 70 | end 71 | saveas(gcf, [figPath dataset '_PRCurve.fig'] ); 72 | saveas(gcf, [figPath dataset '_PRCurve.pdf'] ); 73 | 74 | %% plot the F-measure curves 75 | figure(2); 76 | hold on; 77 | grid on; 78 | axis([0 255 0 1]); 79 | title(dataset); 80 | xlabel('Threshold'); 81 | ylabel('F-measure'); 82 | x = [255:-1:0]'; 83 | for i = 1 : length(plotMetrics.Alg_names) 84 | if mod(i,2)==0 85 | plot(x, plotMetrics.Fmeasure_Curve(:,i), '--', 'Color', str(i), 'LineWidth', 2); 86 | elseif mod(i,2)==1 87 | plot(x, plotMetrics.Fmeasure_Curve(:,i), 'Color', str(i), 'LineWidth', 2); 88 | end 89 | 90 | [maxF,max_idx] = max(plotMetrics.Fmeasure_Curve(:,i)); 91 | h2 = plot(255-max_idx, maxF, '.', 'Color', str(i),'markersize',20); 92 | set(get(get(h2,'Annotation'),'LegendInformation'),'IconDisplayStyle','off'); 93 | end 94 | legend(plotMetrics.Alg_names,'Location','SouthWest'); 95 | %legend(plotMetrics.Alg_names); 96 | set(gcf,'position',[1160 600 560 420]); 97 | saveas(gcf, [figPath dataset '_FmCurve.fig']); 98 | saveas(gcf, [figPath dataset '_FmCurve.pdf'] ); 99 | end 100 | 101 | -------------------------------------------------------------------------------- /Code/ReadMe.txt: -------------------------------------------------------------------------------- 1 | % Author: Deng-Ping Fan 2 | % Date: 2018/9/8 3 | % E-mail: dengpingfan@mail.nankai.edu.cn 4 | % HomePage: http://dpfan.net/ 5 | 6 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 7 | Just run the main.m to get the S-measure, E-measure, F-measure, and MAE score. 8 | Then you can run the Plot_curve.m code to generate the PR Curve. You can also download our 9 | pre-runing mat file from our website: http://dpfan.net/d3netbenchmark/. 10 | 11 | If the you use our code please cite the related works. 12 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 13 | 14 | @inproceedings{fan2017structure, 15 | title={{Structure-measure: A New Way to Evaluate Foreground Maps}}, 16 | author={Fan, Deng-Ping and Cheng, Ming-Ming and Liu, Yun and Li, Tao and Borji, Ali}, 17 | booktitle={IEEE International Conference on Computer Vision (ICCV)}, 18 | pages = {4548-4557}, 19 | year={2017}, 20 | note={\url{http://dpfan.net/smeasure/}}, 21 | organization={IEEE} 22 | } 23 | 24 | @inproceedings{Fan2018Enhanced, 25 | author={Fan, Deng-Ping and Gong, Cheng and Cao, Yang and Ren, Bo and Cheng, Ming-Ming and Borji, Ali}, 26 | title={{Enhanced-alignment Measure for Binary Foreground Map Evaluation}}, 27 | booktitle={International Joint Conference on Artificial Intelligence (IJCAI)}, 28 | pages={698--704}, 29 | note={\url{http://dpfan.net/e-measure/}}, 30 | year={2018} 31 | } 32 | -------------------------------------------------------------------------------- /Code/S_object.m: -------------------------------------------------------------------------------- 1 | function Q = S_object(prediction,GT) 2 | % S_object Computes the object similarity between foreground maps and ground 3 | % truth(as proposed in "Structure-measure:A new way to evaluate foreground 4 | % maps" [Deng-Ping Fan et. al - ICCV 2017]) 5 | % Usage: 6 | % Q = S_object(prediction,GT) 7 | % Input: 8 | % prediction - Binary/Non binary foreground map with values in the range 9 | % [0 1]. Type: double. 10 | % GT - Binary ground truth. Type: logical. 11 | % Output: 12 | % Q - The object similarity score 13 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 14 | 15 | % compute the similarity of the foreground in the object level 16 | prediction_fg = prediction; 17 | prediction_fg(~GT)=0; 18 | O_FG = Object(prediction_fg,GT); 19 | 20 | % compute the similarity of the background 21 | prediction_bg = 1.0 - prediction; 22 | prediction_bg(GT) = 0; 23 | O_BG = Object(prediction_bg,~GT); 24 | 25 | % combine the foreground measure and background measure together 26 | u = mean2(GT); 27 | Q = u * O_FG + (1 - u) * O_BG; 28 | 29 | end 30 | 31 | function score = Object(prediction,GT) 32 | 33 | % check the input 34 | if isempty(prediction) 35 | score = 0; 36 | return; 37 | end 38 | if isinteger(prediction) 39 | prediction = double(prediction); 40 | end 41 | if (~isa( prediction, 'double' )) 42 | error('prediction should be of type: double'); 43 | end 44 | if ((max(prediction(:))>1) || min(prediction(:))<0) 45 | error('prediction should be in the range of [0 1]'); 46 | end 47 | if(~islogical(GT)) 48 | error('GT should be of type: logical'); 49 | end 50 | 51 | % compute the mean of the foreground or background in prediction 52 | x = mean2(prediction(GT)); 53 | 54 | % compute the standard deviations of the foreground or background in prediction 55 | sigma_x = std(prediction(GT)); 56 | 57 | score = 2.0 * x./(x^2 + 1.0 + sigma_x + eps); 58 | end -------------------------------------------------------------------------------- /Code/S_region.m: -------------------------------------------------------------------------------- 1 | function Q = S_region(prediction,GT) 2 | % S_region computes the region similarity between the foreground map and 3 | % ground truth(as proposed in "Structure-measure:A new way to evaluate 4 | % foreground maps" [Deng-Ping Fan et. al - ICCV 2017]) 5 | % Usage: 6 | % Q = S_region(prediction,GT) 7 | % Input: 8 | % prediction - Binary/Non binary foreground map with values in the range 9 | % [0 1]. Type: double. 10 | % GT - Binary ground truth. Type: logical. 11 | % Output: 12 | % Q - The region similarity score 13 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 14 | 15 | % find the centroid of the GT 16 | [X,Y] = centroid(GT); 17 | 18 | % divide GT into 4 regions 19 | [GT_1,GT_2,GT_3,GT_4,w1,w2,w3,w4] = divideGT(GT,X,Y); 20 | 21 | %Divede prediction into 4 regions 22 | [prediction_1,prediction_2,prediction_3,prediction_4] = Divideprediction(prediction,X,Y); 23 | 24 | %Compute the ssim score for each regions 25 | Q1 = ssim(prediction_1,GT_1); 26 | Q2 = ssim(prediction_2,GT_2); 27 | Q3 = ssim(prediction_3,GT_3); 28 | Q4 = ssim(prediction_4,GT_4); 29 | 30 | %Sum the 4 scores 31 | Q = w1 * Q1 + w2 * Q2 + w3 * Q3 + w4 * Q4; 32 | 33 | end 34 | 35 | function [X,Y] = centroid(GT) 36 | % Centroid Compute the centroid of the GT 37 | % Usage: 38 | % [X,Y] = Centroid(GT) 39 | % Input: 40 | % GT - Binary ground truth. Type: logical. 41 | % Output: 42 | % [X,Y] - The coordinates of centroid. 43 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 44 | [rows,cols] = size(GT); 45 | 46 | if(sum(GT(:))==0) 47 | X = round(cols/2); 48 | Y = round(rows/2); 49 | else 50 | total=sum(GT(:)); 51 | i=1:cols; 52 | j=(1:rows)'; 53 | X=round(sum(sum(GT,1).*i)/total); 54 | Y=round(sum(sum(GT,2).*j)/total); 55 | 56 | %dGT = double(GT); 57 | %x = ones(rows,1)*(1:cols); 58 | %y = (1:rows)'*ones(1,cols); 59 | %area = sum(dGT(:)); 60 | %X = round(sum(sum(dGT.*x))/area); 61 | %Y = round(sum(sum(dGT.*y))/area); 62 | end 63 | 64 | end 65 | 66 | % divide the GT into 4 regions according to the centroid of the GT and return the weights 67 | function [LT,RT,LB,RB,w1,w2,w3,w4] = divideGT(GT,X,Y) 68 | % LT - left top; 69 | % RT - right top; 70 | % LB - left bottom; 71 | % RB - right bottom; 72 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 73 | 74 | %width and height of the GT 75 | [hei,wid] = size(GT); 76 | area = wid * hei; 77 | 78 | %copy the 4 regions 79 | LT = GT(1:Y,1:X); 80 | RT = GT(1:Y,X+1:wid); 81 | LB = GT(Y+1:hei,1:X); 82 | RB = GT(Y+1:hei,X+1:wid); 83 | 84 | %The different weight (each block proportional to the GT foreground region). 85 | w1 = (X*Y)./area; 86 | w2 = ((wid-X)*Y)./area; 87 | w3 = (X*(hei-Y))./area; 88 | w4 = 1.0 - w1 - w2 - w3; 89 | end 90 | 91 | %Divide the prediction into 4 regions according to the centroid of the GT 92 | function [LT,RT,LB,RB] = Divideprediction(prediction,X,Y) 93 | 94 | %width and height of the prediction 95 | [hei,wid] = size(prediction); 96 | 97 | %copy the 4 regions 98 | LT = prediction(1:Y,1:X); 99 | RT = prediction(1:Y,X+1:wid); 100 | LB = prediction(Y+1:hei,1:X); 101 | RB = prediction(Y+1:hei,X+1:wid); 102 | 103 | end 104 | 105 | function Q = ssim(prediction,GT) 106 | % ssim computes the region similarity between foreground maps and ground 107 | % truth(as proposed in "Structure-measure: A new way to evaluate foreground 108 | % maps" [Deng-Ping Fan et. al - ICCV 2017]) 109 | % Usage: 110 | % Q = ssim(prediction,GT) 111 | % Input: 112 | % prediction - Binary/Non binary foreground map with values in the range 113 | % [0 1]. Type: double. 114 | % GT - Binary ground truth. Type: logical. 115 | % Output: 116 | % Q - The region similarity score 117 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 118 | 119 | dGT = double(GT); 120 | 121 | [hei,wid] = size(prediction); 122 | N = wid*hei; 123 | 124 | %Compute the mean of SM,GT 125 | x = mean2(prediction); 126 | y = mean2(dGT); 127 | 128 | %Compute the variance of SM,GT 129 | sigma_x2 = sum(sum((prediction - x).^2))./(N - 1 + eps);%sigma_x2 = var(prediction(:)) 130 | sigma_y2 = sum(sum((dGT - y).^2))./(N - 1 + eps); %sigma_y2 = var(dGT(:)); 131 | 132 | %Compute the covariance between SM and GT 133 | sigma_xy = sum(sum((prediction - x).*(dGT - y)))./(N - 1 + eps); 134 | 135 | alpha = 4 * x * y * sigma_xy; 136 | beta = (x.^2 + y.^2).*(sigma_x2 + sigma_y2); 137 | 138 | if(alpha ~= 0) 139 | Q = alpha./(beta + eps); 140 | elseif(alpha == 0 && beta == 0) 141 | Q = 1.0; 142 | else 143 | Q = 0; 144 | end 145 | 146 | end 147 | -------------------------------------------------------------------------------- /Code/StructureMeasure.m: -------------------------------------------------------------------------------- 1 | function Q = StructureMeasure(prediction,GT) 2 | % StructureMeasure computes the similarity between the foreground map and 3 | % ground truth(as proposed in "Structure-measure: A new way to evaluate 4 | % foreground maps" [Deng-Ping Fan et. al - ICCV 2017]) 5 | % Usage: 6 | % Q = StructureMeasure(prediction,GT) 7 | % Input: 8 | % prediction - Binary/Non binary foreground map with values in the range 9 | % [0 1]. Type: double. 10 | % GT - Binary ground truth. Type: logical. 11 | % Output: 12 | % Q - The computed similarity score 13 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 14 | 15 | % Check input 16 | if (~isa(prediction,'double')) 17 | error('The prediction should be double type...'); 18 | end 19 | if ((max(prediction(:))>1) || min(prediction(:))<0) 20 | error('The prediction should be in the range of [0 1]...'); 21 | end 22 | if (~islogical(GT)) 23 | error('GT should be logical type...'); 24 | end 25 | 26 | y = mean2(GT); 27 | 28 | if (y==0)% if the GT is completely black 29 | x = mean2(prediction); 30 | Q = 1.0 - x; %only calculate the area of intersection 31 | elseif(y==1)%if the GT is completely white 32 | x = mean2(prediction); 33 | Q = x; %only calcualte the area of intersection 34 | else 35 | alpha = 0.5; 36 | Q = alpha*S_object(prediction,GT)+(1-alpha)*S_region(prediction,GT); 37 | if (Q<0) 38 | Q=0; 39 | end 40 | end 41 | 42 | end 43 | -------------------------------------------------------------------------------- /Code/WFb.m: -------------------------------------------------------------------------------- 1 | function [Q]= WFb(FG,GT) 2 | % WFb Compute the Weighted F-beta measure (as proposed in "How to Evaluate 3 | % Foreground Maps?" [Margolin et. al - CVPR'14]) 4 | % Usage: 5 | % Q = FbW(FG,GT) 6 | % Input: 7 | % FG - Binary/Non binary foreground map with values in the range [0 1]. Type: double. 8 | % GT - Binary ground truth. Type: logical. 9 | % Output: 10 | % Q - The Weighted F-beta score 11 | 12 | %Check input 13 | if (~isa( FG, 'double' )) 14 | error('FG should be of type: double'); 15 | end 16 | if ((max(FG(:))>1) || min(FG(:))<0) 17 | error('FG should be in the range of [0 1]'); 18 | end 19 | if (~islogical(GT)) 20 | error('GT should be of type: logical'); 21 | end 22 | 23 | dGT = double(GT); %Use double for computations. 24 | 25 | 26 | E = abs(FG-dGT); 27 | % [Ef, Et, Er] = deal(abs(FG-GT)); 28 | 29 | [Dst,IDXT] = bwdist(dGT); 30 | %Pixel dependency 31 | K = fspecial('gaussian',7,5); 32 | Et = E; 33 | Et(~GT)=Et(IDXT(~GT)); %To deal correctly with the edges of the foreground region 34 | EA = imfilter(Et,K); 35 | MIN_E_EA = E; 36 | MIN_E_EA(GT & EA1) || min(FG(:))<0) 17 | error('FG should be in the range of [0 1]'); 18 | end 19 | if (~islogical(GT)) 20 | error('GT should be of type: logical'); 21 | end 22 | 23 | dGT = double(GT); %Use double for computations. 24 | 25 | 26 | E = abs(FG-dGT); 27 | % [Ef, Et, Er] = deal(abs(FG-GT)); 28 | 29 | [Dst,IDXT] = bwdist(dGT); 30 | %Pixel dependency 31 | K = fspecial('gaussian',7,5); 32 | Et = E; 33 | Et(~GT)=Et(IDXT(~GT)); %To deal correctly with the edges of the foreground region 34 | EA = imfilter(Et,K); 35 | MIN_E_EA = E; 36 | MIN_E_EA(GT & EA2) 67 | gt = rgb2gray(gt); 68 | end 69 | 70 | if ~islogical(gt) 71 | gt = gt(:,:,1) > 128; 72 | end 73 | 74 | %load salency 75 | 76 | sal = imread([salPath name]); 77 | % imwrite(sal,[salPath '/a/' name]); 78 | 79 | %check size 80 | if size(sal, 1) ~= size(gt, 1) || size(sal, 2) ~= size(gt, 2) 81 | sal = imresize(sal,size(gt)); 82 | imwrite(sal,[salPath name]); 83 | fprintf('Error occurs in the path: %s!!!\n', [salPath name]); %check whether the size of the salmap is equal the gt map. 84 | end 85 | 86 | sal = im2double(sal(:,:,1)); 87 | 88 | %normalize sal to [0, 1] 89 | sal = reshape(mapminmax(sal(:)',0,1),size(sal)); 90 | Sscore = StructureMeasure(sal,logical(gt)); 91 | Smeasure(i) = Sscore; 92 | 93 | [weiFm(i)]= WFb(sal,gt); 94 | 95 | end 96 | 97 | %Smeasure score 98 | Smeasure = mean2(Smeasure); 99 | 100 | %weighted F-m score 101 | WeiFm = mean2(weiFm); 102 | 103 | %Save the mat file so that you can reload the mat file and plot the PR Curve 104 | save([ResPath model],'Smeasure','WeiFm'); 105 | 106 | fprintf(fileID, '(Dataset:%s; Model:%s) Smeasure:%.3f; WeiFm:%.3f.\n',dataset,model,Smeasure,WeiFm); 107 | end 108 | toc; 109 | 110 | end 111 | 112 | 113 | -------------------------------------------------------------------------------- /Code/calImageMean.m: -------------------------------------------------------------------------------- 1 | clear; 2 | clc; 3 | close all; 4 | 5 | imPath = '/home/lgy/flownet2-master/RGBT_MODEL/dataset/RGBT_VT821/RGB/'; 6 | TPath = '/home/lgy/flownet2-master/RGBT_MODEL/dataset/RGBT_VT821/T/'; 7 | files = dir([imPath '*.jpg']); 8 | num1 = length(files); 9 | 10 | IMAGE_DIM_H = 288; 11 | IMAGE_DIM_W = 288; 12 | 13 | im_R = zeros(IMAGE_DIM_H,IMAGE_DIM_W); 14 | im_G = zeros(IMAGE_DIM_H,IMAGE_DIM_W); 15 | im_B = zeros(IMAGE_DIM_H,IMAGE_DIM_W); 16 | 17 | imT_R = zeros(IMAGE_DIM_H,IMAGE_DIM_W); 18 | imT_G = zeros(IMAGE_DIM_H,IMAGE_DIM_W); 19 | imT_B = zeros(IMAGE_DIM_H,IMAGE_DIM_W); 20 | 21 | for i = 1 : num1 22 | im = imread([imPath files(i).name]); 23 | im = single(im); 24 | im = imresize(im, [IMAGE_DIM_H IMAGE_DIM_W], 'bilinear'); 25 | im_R = im(:,:,1) + im_R; 26 | im_G = im(:,:,2) + im_G; 27 | im_B = im(:,:,3) + im_B; 28 | 29 | imT = (imread([TPath files(i).name(1:end-4) '.jpg' ])); 30 | imT = single(imT); 31 | imT = imresize(imT, [IMAGE_DIM_H IMAGE_DIM_W], 'bilinear'); 32 | imT_R = imT(:,:,1) + imT_R; 33 | imT_G = imT(:,:,2) + imT_G; 34 | imT_B = imT(:,:,3) + imT_B; 35 | 36 | end 37 | 38 | % num_all = num1; 39 | % im_R_mean = sum(sum(im_R/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 40 | % im_G_mean = sum(sum(im_G/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 41 | % im_B_mean = sum(sum(im_B/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 42 | % 43 | % imT_R_mean = sum(sum(imT_R/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 44 | % imT_G_mean = sum(sum(imT_G/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 45 | % imT_B_mean = sum(sum(imT_B/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 46 | % 47 | % display(['im_R_mean: ' num2str(im_R_mean) ', im_G_mean: ' num2str(im_G_mean) ', im_B_mean: ' num2str(im_B_mean) ]); 48 | % display(['T_R_mean: ' num2str(imT_R_mean) ', T_G_mean: ' num2str(imT_G_mean) ', T_B_mean: ' num2str(imT_B_mean) ]); 49 | 50 | 51 | %% 52 | imPath = '/home/lgy/flownet2-master/RGBT_MODEL/dataset/RGBT_VT1000/RGB/'; 53 | TPath = '/home/lgy/flownet2-master/RGBT_MODEL/dataset/RGBT_VT1000/T/'; 54 | files2 = dir([imPath '*.png']); 55 | num2 = length(files2); 56 | 57 | for j = 1 : num2 58 | im = imread([imPath files2(j).name]); 59 | im = single(im); 60 | im = imresize(im, [IMAGE_DIM_H IMAGE_DIM_W], 'bilinear'); 61 | im_R = im(:,:,1) + im_R; 62 | im_G = im(:,:,2) + im_G; 63 | im_B = im(:,:,3) + im_B; 64 | 65 | imT = (imread([TPath files2(j).name(1:end-4) '.png' ])); 66 | imT = single(imT); 67 | imT = imresize(imT, [IMAGE_DIM_H IMAGE_DIM_W], 'bilinear'); 68 | imT_R = imT(:,:,1) + imT_R; 69 | imT_G = imT(:,:,2) + imT_G; 70 | imT_B = imT(:,:,3) + imT_B; 71 | 72 | end 73 | 74 | num_all = num1 + num2; 75 | im_R_mean = sum(sum(im_R/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 76 | im_G_mean = sum(sum(im_G/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 77 | im_B_mean = sum(sum(im_B/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 78 | 79 | imT_R_mean = sum(sum(imT_R/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 80 | imT_G_mean = sum(sum(imT_G/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 81 | imT_B_mean = sum(sum(imT_B/num_all))/(IMAGE_DIM_H*IMAGE_DIM_W); 82 | 83 | display(['im_R_mean: ' num2str(im_R_mean) ', im_G_mean: ' num2str(im_G_mean) ', im_B_mean: ' num2str(im_B_mean) ]); 84 | display(['T_R_mean: ' num2str(imT_R_mean) ', T_G_mean: ' num2str(imT_G_mean) ', T_B_mean: ' num2str(imT_B_mean) ]); 85 | 86 | % im_R_mean: 129.4971, im_G_mean: 132.4638, im_B_mean: 127.1397 87 | % T_R_mean: 174.2724, T_G_mean: 77.3299, T_B_mean: 86.1234 88 | 89 | 90 | -------------------------------------------------------------------------------- /Code/gather_the_results.m: -------------------------------------------------------------------------------- 1 | function plotMetrics = gather_the_results(modelNum, matPath, alg_params_exist) 2 | 3 | alg_names = cell(modelNum,1); 4 | thrNum = 256; 5 | plotMetrics.Pre = zeros(thrNum,modelNum); 6 | plotMetrics.Recall = zeros(thrNum,modelNum); 7 | %plotMetrics.Fmeasure = zeros(thrNum,modelNum); 8 | %plotMetrics.MAE = zeros(1,modelNum); 9 | 10 | 11 | % gather the existing results 12 | for i = 1 : modelNum 13 | alg_names{i} = alg_params_exist{1,i}; 14 | Metrics = load([matPath,alg_names{i},'.mat']); 15 | plotMetrics.Pre(:,i) = Metrics.column_Pr; 16 | plotMetrics.Recall(:,i) = Metrics.column_Rec; 17 | %plotMetrics.Fmeasure(:,i) = Metrics.column_F; 18 | %plotMetrics.MAE(:,i) = Metrics.MAE; 19 | end 20 | plotMetrics.Fmeasure_Curve = (1+0.3).*plotMetrics.Pre.*plotMetrics.Recall./... 21 | (0.3*plotMetrics.Pre+plotMetrics.Recall); 22 | plotMetrics.Alg_names = alg_names; 23 | 24 | end -------------------------------------------------------------------------------- /Code/linspecer.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MathLee/MatlabEvaluationTools/63f9b309f52717ec9db7dfb5ebdb7392bcdf264f/Code/linspecer.m -------------------------------------------------------------------------------- /Code/main.m: -------------------------------------------------------------------------------- 1 | % % % % % % % % % % clear all; close all; clc; 2 | 3 | %SaliencyMap Path 4 | % SalMapPath = '../SalMap/Image_SOD/'; %The saliency map results can be downloaded from our webpage: http://dpfan.net/d3netbenchmark/ 5 | SalMapPath = '../SalMap/'; 6 | 7 | %Evaluated Models 8 | Models = {'MCCNet'}; 9 | 10 | %Datasets 11 | DataPath = '../Dataset/'; 12 | Datasets = {'ORSSD', 'EORSSD'}; 13 | 14 | %% 15 | 16 | %Evaluated Score Results 17 | ResDir = '../Result_overall/'; 18 | 19 | %Initial paramters setting 20 | Thresholds = 1:-1/255:0; 21 | datasetNum = length(Datasets); 22 | modelNum = length(Models); 23 | 24 | for d = 1:datasetNum 25 | 26 | tic; 27 | dataset = Datasets{d}; 28 | fprintf('Processing %d/%d: %s Dataset\n',d,datasetNum,dataset); 29 | 30 | ResPath = [ResDir dataset '-mat/']; 31 | if ~exist(ResPath,'dir') 32 | mkdir(ResPath); 33 | end 34 | resTxt = [ResDir dataset '_result-overall.txt']; 35 | fileID = fopen(resTxt,'w'); 36 | 37 | for m = 1:modelNum 38 | model = Models{m}; 39 | 40 | gtPath = [DataPath dataset '/GT/']; 41 | 42 | salPath = [SalMapPath model '/' dataset '/']; 43 | 44 | imgFiles = dir([salPath '*.png']); 45 | imgNUM = length(imgFiles); 46 | 47 | [threshold_Fmeasure, threshold_Emeasure] = deal(zeros(imgNUM,length(Thresholds))); 48 | 49 | [threshold_Precion, threshold_Recall] = deal(zeros(imgNUM,length(Thresholds))); 50 | 51 | [Smeasure, adpFmeasure, adpEmeasure, MAE, weiFm] =deal(zeros(1,imgNUM)); 52 | 53 | parfor i = 1:imgNUM %parfor i = 1:imgNUM You may also need the parallel strategy. 54 | 55 | fprintf('Evaluating(%s Dataset,%s Model): %d/%d\n',dataset, model, i,imgNUM); 56 | name = imgFiles(i).name; 57 | 58 | %load gt 59 | gt = imread([gtPath name]); 60 | 61 | if (ndims(gt)>2) 62 | gt = rgb2gray(gt); 63 | end 64 | 65 | if ~islogical(gt) 66 | gt = gt(:,:,1) > 128; 67 | end 68 | 69 | %load salency 70 | sal = imread([salPath name]); 71 | 72 | %check size 73 | if size(sal, 1) ~= size(gt, 1) || size(sal, 2) ~= size(gt, 2) 74 | sal = imresize(sal,size(gt)); 75 | imwrite(sal,[salPath name]); 76 | fprintf('Error occurs in the path: %s!!!\n', [salPath name]); %check whether the size of the salmap is equal the gt map. 77 | end 78 | 79 | sal = im2double(sal(:,:,1)); 80 | 81 | %normalize sal to [0, 1] 82 | sal = reshape(mapminmax(sal(:)',0,1),size(sal)); 83 | Sscore = StructureMeasure(sal,logical(gt)); 84 | Smeasure(i) = Sscore; 85 | 86 | % [weiFm(i)]= WFb(sal,gt); 87 | [weiFm(i)]= 0; 88 | 89 | % Using the 2 times of average of sal map as the adaptive threshold. 90 | threshold = 2* mean(sal(:)) ; 91 | [~,~,adpFmeasure(i)] = Fmeasure_calu(sal,double(gt),size(gt),threshold); 92 | 93 | 94 | Bi_sal = zeros(size(sal)); 95 | Bi_sal(sal>threshold)=1; 96 | adpEmeasure(i) = Enhancedmeasure(Bi_sal,gt); 97 | 98 | [threshold_F, threshold_E] = deal(zeros(1,length(Thresholds))); 99 | [threshold_Pr, threshold_Rec] = deal(zeros(1,length(Thresholds))); 100 | 101 | for t = 1:length(Thresholds) 102 | threshold = Thresholds(t); 103 | [threshold_Pr(t), threshold_Rec(t), threshold_F(t)] = Fmeasure_calu(sal,double(gt),size(gt),threshold); 104 | 105 | Bi_sal = zeros(size(sal)); 106 | Bi_sal(sal>threshold)=1; 107 | threshold_E(t) = Enhancedmeasure(Bi_sal,gt); 108 | end 109 | 110 | threshold_Fmeasure(i,:) = threshold_F; 111 | threshold_Emeasure(i,:) = threshold_E; 112 | threshold_Precion(i,:) = threshold_Pr; 113 | threshold_Recall(i,:) = threshold_Rec; 114 | 115 | MAE(i) = mean2(abs(double(logical(gt)) - sal)); 116 | 117 | end 118 | 119 | %Precision and Recall 120 | column_Pr = mean(threshold_Precion,1); 121 | column_Rec = mean(threshold_Recall,1); 122 | 123 | %Mean, Max F-measure score 124 | column_F = mean(threshold_Fmeasure,1); 125 | meanFm = mean(column_F); 126 | maxFm = max(column_F); 127 | 128 | %Mean, Max E-measure score 129 | column_E = mean(threshold_Emeasure,1); 130 | meanEm = mean(column_E); 131 | maxEm = max(column_E); 132 | 133 | %Adaptive threshold for F-measure and E-measure score 134 | adpFm = mean2(adpFmeasure); 135 | adpEm = mean2(adpEmeasure); 136 | 137 | %Smeasure score 138 | Smeasure = mean2(Smeasure); 139 | 140 | %MAE score 141 | mae = mean2(MAE); 142 | 143 | %WeiFm score 144 | WeiFm = mean2(weiFm); 145 | 146 | %Save the mat file so that you can reload the mat file and plot the PR Curve 147 | save([ResPath model],'Smeasure', 'mae', 'column_Pr', 'column_Rec', 'column_F', 'adpFm', 'meanFm', 'maxFm', 'column_E', 'adpEm', 'meanEm', 'maxEm'); 148 | 149 | fprintf(fileID, '(Dataset:%s; Model:%s) Smeasure:%.4f; MAE:%.4f; adpEm:%.4f; meanEm:%.4f; maxEm:%.4f; adpFm:%.4f; meanFm:%.4f; maxFm:%.4f.\n',dataset,model,Smeasure, mae, adpEm, meanEm, maxEm, adpFm, meanFm, maxFm); 150 | end 151 | toc; 152 | 153 | end 154 | 155 | 156 | -------------------------------------------------------------------------------- /Code/main_LF.m: -------------------------------------------------------------------------------- 1 | clear all; close all; clc; 2 | 3 | %SaliencyMap Path 4 | % SalMapPath = '../SalMap/Image_SOD/'; %The saliency map results can be downloaded from our webpage: http://dpfan.net/d3netbenchmark/ 5 | SalMapPath = '../LF_SalMap/'; 6 | 7 | %Evaluated Models 8 | % Models = {'CMC', 'SMFF', 'VOS', 'LVNet', 'DSG', 'HDCT', 'RCRR', 'RRWR', 'SMD', 'DSS', 'EGNet', 'PFAN', 'PoolNet', 'R3Net', 'RADF', 'DAFNet-VGG', 'DAFNet-Res'}; 9 | % Models = { '34', '39', '49', '59', '64'}; 10 | % Models = { '16-39', '17-39'}; 11 | % Models = {'4D-Ours','2D-BASNet(CVPR2019)', '2D-CAGNet(PR2020)', '2D-CPD(CVPR2019)', '2D-ITSD(CVPR2020)', '2D-PFAN(CVPR2019)', '3D-BBSNet(ECCV2020)', '3D-CMWNet(ECCV2020)', '3D-ICNet(TIP2020)', '3D-S2MA(CVPR2020)', '3D-SSF(CVPR2020)', '4D-DLFS(IJCAI2019)', '4D-LFDCN(TIP2020)', '4D-LFNet(TIP2020)', '4D-MoLF(NeurIPS2019)', '4D-MTCNet(TCSVT2020)'}; 12 | % Models = {'4D-Ours', '3D-S2MA(CVPR2020)', '3D-SSF(CVPR2020)', '4D-DLFS(IJCAI2019)', '4D-LFDCN(TIP2020)', '4D-LFNet(TIP2020)', '4D-MoLF(NeurIPS2019)', '4D-MTCNet(TCSVT2020)'}; 13 | Models = { 'HAINet'}; 14 | 15 | 16 | %Datasets 17 | DataPath = '../Dataset/'; 18 | % Datasets = {'ORSSD', 'EORSSD'}; 19 | Datasets = {'HFUT-Lytro Illum(TIP2020)', 'HFUT-Lytro(TOMCCAP2017)'}; 20 | % Datasets = {'HFUT-Lytro(TOMCCAP2017)'}; 21 | % Datasets = {'ORSSD'}; 22 | 23 | %% 24 | 25 | %Evaluated Score Results 26 | % ResDir = '../Result_image_sod/'; 27 | ResDir = '../Result_overall/'; 28 | 29 | %Initial paramters setting 30 | Thresholds = 1:-1/255:0; 31 | datasetNum = length(Datasets); 32 | modelNum = length(Models); 33 | 34 | for d = 1:datasetNum 35 | 36 | tic; 37 | dataset = Datasets{d}; 38 | fprintf('Processing %d/%d: %s Dataset\n',d,datasetNum,dataset); 39 | 40 | ResPath = [ResDir dataset '-mat/']; 41 | if ~exist(ResPath,'dir') 42 | mkdir(ResPath); 43 | end 44 | resTxt = [ResDir dataset '_result-overall.txt']; 45 | fileID = fopen(resTxt,'w'); 46 | 47 | for m = 1:modelNum 48 | model = Models{m}; 49 | 50 | gtPath = [DataPath dataset '/GT/']; 51 | 52 | salPath = [SalMapPath model '/' dataset '/']; 53 | 54 | imgFiles = dir([salPath '*.png']); 55 | imgNUM = length(imgFiles); 56 | 57 | [threshold_Fmeasure, threshold_Emeasure] = deal(zeros(imgNUM,length(Thresholds))); 58 | 59 | [threshold_Precion, threshold_Recall] = deal(zeros(imgNUM,length(Thresholds))); 60 | 61 | [Smeasure, adpFmeasure, adpEmeasure, MAE, weiFm] =deal(zeros(1,imgNUM)); 62 | 63 | parfor i = 1:imgNUM %parfor i = 1:imgNUM You may also need the parallel strategy. 64 | 65 | fprintf('Evaluating(%s Dataset,%s Model): %d/%d\n',dataset, model, i,imgNUM); 66 | name = imgFiles(i).name; 67 | 68 | %load gt 69 | gt = imread([gtPath name]); 70 | 71 | if (ndims(gt)>2) 72 | gt = rgb2gray(gt); 73 | end 74 | 75 | if ~islogical(gt) 76 | gt = gt(:,:,1) > 128; 77 | end 78 | 79 | %load salency 80 | sal = imread([salPath name]); 81 | 82 | %check size 83 | if size(sal, 1) ~= size(gt, 1) || size(sal, 2) ~= size(gt, 2) 84 | sal = imresize(sal,size(gt)); 85 | imwrite(sal,[salPath name]); 86 | fprintf('Error occurs in the path: %s!!!\n', [salPath name]); %check whether the size of the salmap is equal the gt map. 87 | end 88 | 89 | sal = im2double(sal(:,:,1)); 90 | 91 | %normalize sal to [0, 1] 92 | sal = reshape(mapminmax(sal(:)',0,1),size(sal)); 93 | Sscore = StructureMeasure(sal,logical(gt)); 94 | Smeasure(i) = Sscore; 95 | 96 | [weiFm(i)]= WFb(sal,gt); 97 | % [weiFm(i)]= 0; 98 | 99 | % Using the 2 times of average of sal map as the adaptive threshold. 100 | threshold = 2* mean(sal(:)) ; 101 | [~,~,adpFmeasure(i)] = Fmeasure_calu(sal,double(gt),size(gt),threshold); 102 | 103 | 104 | Bi_sal = zeros(size(sal)); 105 | Bi_sal(sal>threshold)=1; 106 | adpEmeasure(i) = Enhancedmeasure(Bi_sal,gt); 107 | 108 | [threshold_F, threshold_E] = deal(zeros(1,length(Thresholds))); 109 | [threshold_Pr, threshold_Rec] = deal(zeros(1,length(Thresholds))); 110 | 111 | for t = 1:length(Thresholds) 112 | threshold = Thresholds(t); 113 | [threshold_Pr(t), threshold_Rec(t), threshold_F(t)] = Fmeasure_calu(sal,double(gt),size(gt),threshold); 114 | 115 | Bi_sal = zeros(size(sal)); 116 | Bi_sal(sal>threshold)=1; 117 | threshold_E(t) = Enhancedmeasure(Bi_sal,gt); 118 | end 119 | 120 | threshold_Fmeasure(i,:) = threshold_F; 121 | threshold_Emeasure(i,:) = threshold_E; 122 | threshold_Precion(i,:) = threshold_Pr; 123 | threshold_Recall(i,:) = threshold_Rec; 124 | 125 | MAE(i) = mean2(abs(double(logical(gt)) - sal)); 126 | 127 | end 128 | 129 | %Precision and Recall 130 | column_Pr = mean(threshold_Precion,1); 131 | column_Rec = mean(threshold_Recall,1); 132 | 133 | %Mean, Max F-measure score 134 | column_F = mean(threshold_Fmeasure,1); 135 | meanFm = mean(column_F); 136 | maxFm = max(column_F); 137 | 138 | %Mean, Max E-measure score 139 | column_E = mean(threshold_Emeasure,1); 140 | meanEm = mean(column_E); 141 | maxEm = max(column_E); 142 | 143 | %Adaptive threshold for F-measure and E-measure score 144 | adpFm = mean2(adpFmeasure); 145 | adpEm = mean2(adpEmeasure); 146 | 147 | %Smeasure score 148 | Smeasure = mean2(Smeasure); 149 | 150 | %MAE score 151 | mae = mean2(MAE); 152 | 153 | %WeiFm score 154 | WeiFm = mean2(weiFm); 155 | 156 | %Save the mat file so that you can reload the mat file and plot the PR Curve 157 | % save([ResPath model],'Smeasure', 'mae', 'column_Pr', 'column_Rec', 'column_F', 'adpFm', 'meanFm', 'maxFm', 'column_E', 'adpEm', 'meanEm', 'maxEm'); 158 | % fprintf(fileID, '(Dataset:%s; Model:%s) Smeasure:%.4f; MAE:%.4f; adpEm:%.4f; meanEm:%.4f; maxEm:%.4f; adpFm:%.4f; meanFm:%.4f; maxFm:%.4f.\n',dataset,model,Smeasure, mae, adpEm, meanEm, maxEm, adpFm, meanFm, maxFm); 159 | save([ResPath model],'Smeasure', 'mae', 'WeiFm', 'column_Pr', 'column_Rec', 'column_F', 'adpFm', 'meanFm', 'maxFm', 'column_E', 'adpEm', 'meanEm', 'maxEm'); 160 | fprintf(fileID, '(Dataset:%s; Model:%s) Smeasure:%.4f; MAE:%.4f; WeiFm:%.4f; adpEm:%.4f; meanEm:%.4f; maxEm:%.4f; adpFm:%.4f; meanFm:%.4f; maxFm:%.4f.\n',dataset,model,Smeasure, mae, WeiFm, adpEm, meanEm, maxEm, adpFm, meanFm, maxFm); 161 | 162 | end 163 | toc; 164 | 165 | end 166 | 167 | 168 | -------------------------------------------------------------------------------- /Code/plot_WeiFm_Sm.m: -------------------------------------------------------------------------------- 1 | clear all; close all; clc; 2 | 3 | %SaliencyMap Path 4 | SalMapPath = '../SalMap/'; 5 | %Models = {'LHM','DESM','CDB','ACSD','GP','LBE','CDCP','SE','MDSF','DF','CTMF','PDNet','PCF'};%'TPF' 6 | %Models = {'PCF','PDNet','CTMF','DF','MDSF','SE','CDCP','LBE','GP','ACSD','CDB','DESM','LHM'}; 7 | % Datasets = {'DES'}; 8 | % Models = {'DF','MDSF','AFNet','PCF','MMCI','TANet','CTMF','CPFP','D3Net','DMRA','Ours'};%DES 9 | 10 | % Datasets = {'NLPR'}; 11 | % Models = {'DF','MDSF','CTMF','AFNet','PCF','MMCI','TANet','CPFP','DMRA','D3Net','Ours'};%'NLPR' 12 | 13 | Datasets = {'LFSD'}; 14 | Models = {'MDSF','CTMF','AFNet','PCF','MMCI','TANet','DF','D3Net','CPFP','DMRA','Ours'};%LFSD 15 | 16 | % Datasets = {'NJU2K'}; 17 | % Models = {'DF','MDSF','CTMF','AFNet','PCF','MMCI','TANet','CPFP','DMRA','D3Net','Ours'};%'NJU2K' 18 | 19 | % Datasets = {'STERE'}; 20 | % Models = {'DF','MDSF','CTMF','AFNet','PCF','TANet','MMCI','CPFP','D3Net','Ours'};%'STERE' 21 | %Deep Model 22 | %DeepModels = {'DF','PDNet','CTMF','PCF'};%TPF-missing, MDSF-svm, 23 | 24 | modelNum = length(Models); 25 | groupNum = floor(modelNum/3)+1; 26 | 27 | %Datasets 28 | DataPath = '../Dataset/'; 29 | % Datasets = {'SSD'}; 30 | % Datasets = {'STERE'};%'STERE' 'DES','NLPR','NJU2K','LFSD' 31 | %Datasets = {'GIT','SSD','DES'}; 32 | % Datasets = {'SSD','DES','LFSD','STERE','NJU2K','GIT','NLPR','SIP'}; 33 | %Datasets = {'SSD','DES','LFSD','STERE','NJU2K','NLPR','SIP'}; 34 | % 35 | %Results 36 | ResDir = '../Result_overall/WeiFm_Sm/'; 37 | 38 | method_colors = linspecer(groupNum); 39 | 40 | % colors 41 | %str=['r','r','r','g','g','g','b','b','b','c','c','c','m','m','m','y','y','y','k','k','k','g','g','b','b','m','m','k','k','r','r','b','b','c','c','m','m']; 42 | 43 | % str=['r','r','g','g','b','b','y','y','c','c','k','k','m','m']; 44 | str=['y','g','b','y','c','k','m','y','g','b','r','c','m','k']; 45 | 46 | % point label 47 | Pstr = ['p','>','<','^','v','d','s','*','x','o','.','h']; 48 | 49 | 50 | datasetNum = length(Datasets); 51 | for d = 1:datasetNum 52 | close all; 53 | dataset = Datasets{d}; 54 | fprintf('Processing %d/%d: %s Dataset\n',d,datasetNum,dataset); 55 | 56 | matPath = [ResDir dataset '-mat/']; 57 | plotMetrics = gather_the_results(modelNum, matPath, Models); 58 | 59 | %% plot the WeiFm_Sm curves 60 | figure(1); 61 | hold on; 62 | grid on; 63 | axis([0 1 0 1]); 64 | set(gca,'XTick',[0.72:0.04:0.92]); 65 | % title(dataset); 66 | xlabel('Structure Similarity'); 67 | ylabel('Weighted F-measure'); 68 | set(gca,'linewidth',1.5) 69 | set(get(gca,'XLabel'),'FontSize',21) 70 | set(get(gca,'YLabel'),'FontSize',21) 71 | 72 | for i = 1 : length(plotMetrics.Alg_names)%[200 200 200]/255 73 | % if mod(i,2)==0 74 | % plot(plotMetrics.Smeasure(1,i), plotMetrics.WeiFm(1,i), '.', 'Color', str(i), 'markersize', 20); 75 | % elseif mod(i,2)==1 76 | % plot(plotMetrics.Smeasure(1,i), plotMetrics.WeiFm(1,i), '.', 'Color', str(i), 'markersize', 20); 77 | % end 78 | % [~,max_idx] = max(plotMetrics.Fmeasure_Curve(:,i)); 79 | % h1 = plot(plotMetrics.Recall(max_idx,i), plotMetrics.Pre(max_idx,i), '.', 'Color', str(i),'markersize',20); 80 | text(plotMetrics.Smeasure(1,i),plotMetrics.WeiFm(1,i),Models{i},Pstr(i), 'Color', str(i), 'markersize', 20); 81 | set(get(get(h1,'Annotation'),'LegendInformation'),'IconDisplayStyle','off'); 82 | end 83 | % legend(plotMetrics.Alg_names(7:11),'Location','SouthWest','FontSize',17); 84 | set(gcf,'position',[0 600 560 420]); 85 | box on 86 | 87 | figPath = [ResDir 'Curve/']; 88 | if ~exist(figPath,'dir') 89 | mkdir(figPath); 90 | end 91 | % saveas(gcf, [figPath dataset '_PRCurve.fig'] ); 92 | saveas(gcf, [figPath dataset '_WeiFm_Sm.pdf'] ); 93 | 94 | end 95 | 96 | 97 | 98 | % plotMetrics = gather_the_results(modelNum, matPath, Models); 99 | 100 | function plotMetrics = gather_the_results(modelNum, matPath, alg_params_exist) 101 | 102 | alg_names = cell(modelNum,1); 103 | thrNum = 256; 104 | % plotMetrics.Pre = zeros(thrNum,modelNum); 105 | % plotMetrics.Recall = zeros(thrNum,modelNum); 106 | %plotMetrics.Fmeasure = zeros(thrNum,modelNum); 107 | % plotMetrics.MAE = zeros(1,modelNum); 108 | plotMetrics.Smeasure = zeros(thrNum,modelNum); 109 | plotMetrics.WeiFm = zeros(1,modelNum); 110 | 111 | 112 | % gather the existing results 113 | for i = 1 : modelNum 114 | alg_names{i} = alg_params_exist{1,i}; 115 | Metrics = load([matPath,alg_names{i},'.mat']); 116 | % plotMetrics.Pre(:,i) = Metrics.column_Pr; 117 | % plotMetrics.Recall(:,i) = Metrics.column_Rec; 118 | %plotMetrics.Fmeasure(:,i) = Metrics.column_F; 119 | % plotMetrics.MAE(:,i) = Metrics.MAE; 120 | plotMetrics.Smeasure(:,i) = Metrics.Smeasure; 121 | plotMetrics.WeiFm(:,i) = Metrics.WeiFm; 122 | end 123 | % plotMetrics.Fmeasure_Curve = (1+0.3).*plotMetrics.Pre.*plotMetrics.Recall./... 124 | % (0.3*plotMetrics.Pre+plotMetrics.Recall); 125 | plotMetrics.Alg_names = alg_names; 126 | 127 | end 128 | 129 | -------------------------------------------------------------------------------- /Dataset/EORSSD/GT/0004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MathLee/MatlabEvaluationTools/63f9b309f52717ec9db7dfb5ebdb7392bcdf264f/Dataset/EORSSD/GT/0004.png -------------------------------------------------------------------------------- /Dataset/ORSSD/GT/0005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MathLee/MatlabEvaluationTools/63f9b309f52717ec9db7dfb5ebdb7392bcdf264f/Dataset/ORSSD/GT/0005.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MatlabEvaluationTools 2 | The evaluation tool (Matlab version) for saliency maps. 3 | 4 | 1. Put GTs of your dataset in ./Dataset. 5 | 2. Put your saliency maps in ./SalMap. 6 | 3. Modify the 'Models' and 'Datasets' in ./Code/main.m, and then run it. 7 | -------------------------------------------------------------------------------- /SalMap/MCCNet/EORSSD/0004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MathLee/MatlabEvaluationTools/63f9b309f52717ec9db7dfb5ebdb7392bcdf264f/SalMap/MCCNet/EORSSD/0004.png -------------------------------------------------------------------------------- /SalMap/MCCNet/ORSSD/0005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MathLee/MatlabEvaluationTools/63f9b309f52717ec9db7dfb5ebdb7392bcdf264f/SalMap/MCCNet/ORSSD/0005.png --------------------------------------------------------------------------------