├── Code ├── ADF │ ├── APMFnet.py │ ├── LaplaceEdge.py │ ├── config.py │ ├── dataset.py │ ├── get_PRF_mat.py │ ├── run.py │ ├── solver.py │ └── vgg.py ├── M3S-NIR │ ├── AdjcProcloop.m │ ├── ComputeAffinityMatrix.m │ ├── DEMO11.m │ ├── Plot_PreRecallThousand.m │ ├── ReadDAT.m │ ├── SLICSuperpixelSegmentation.exe │ ├── _SLIC.bmp │ ├── adjacency.m │ ├── colorspace.m │ ├── compare_PRF.m │ ├── compare_PRF1k.m │ ├── descendPropagation.m │ ├── form_codebook.m │ ├── globalW.m │ ├── labelCluster.m │ ├── lastCMR.m │ ├── main_new.m │ ├── makeweights.m │ ├── normalize.m │ ├── readme.txt │ ├── removeframe.m │ ├── softthreshold.m │ ├── twoCMR.m │ ├── vgg_argparse.m │ ├── vgg_kmeans.m │ ├── vgg_kmiter.cxx │ ├── vgg_kmiter.dll │ ├── vgg_kmiter.mexglx │ ├── vgg_kmiter.mexw64 │ └── weightc.m ├── MIED │ ├── README.md │ ├── eval.py │ ├── lib │ │ ├── data_prefetcher.py │ │ ├── dataset.py │ │ └── transform.py │ ├── net.py │ ├── test.py │ ├── train.py │ └── vgg.py ├── MTMR │ ├── AdjcProcloop.m │ ├── CMR.m │ ├── DEMO11.m │ ├── OursPRF.m │ ├── Plot_PreRecallThousand_contrast.m │ ├── ReadDAT.m │ ├── SLICSuperpixelSegmentation.exe │ ├── adjacency.m │ ├── colorspace.m │ ├── compare_PRF.m │ ├── main_new - 备份.m │ ├── main_new.m │ ├── makeweights.m │ ├── normalize.m │ ├── readme.txt │ └── removeframe.m └── SGDL │ ├── AdjcProcloop.m │ ├── ComputeAffinityMatrix.m │ ├── ExtractFCNfeature.m │ ├── FCN.zip │ ├── Find_Edge_Superpixels.m │ ├── GetEdges.m │ ├── GetMeanColor.m │ ├── Learn.m │ ├── ReadDAT.m │ ├── SLICSuperpixelSegmentation.exe │ ├── adjacency.m │ ├── colorspace.m │ ├── demo.m │ ├── find_connect_superpixel_DoubleIn_Opposite.m │ ├── makeweights.m │ ├── normalize.m │ └── readme.txt └── Readme.md /Code/ADF/APMFnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/ADF/APMFnet.py -------------------------------------------------------------------------------- /Code/ADF/LaplaceEdge.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | import numpy as np 4 | #laplace operator layer for edge extraction 5 | 6 | class Laplace_edge_layer(nn.Module): 7 | def __init__(self): 8 | super(Laplace_edge_layer,self).__init__() 9 | self.laplace=nn.Conv2d(in_channels=1,out_channels=1,kernel_size=3,stride=1,padding=1,bias=False) 10 | self.weight_init() 11 | def weight_init(self): 12 | self.laplace.weight.data=torch.tensor(np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]])[np.newaxis,np.newaxis,...]).float() 13 | def forward(self, x): 14 | return torch.abs(self.laplace(x)/9) 15 | 16 | # class Laplace_edge_layer(nn.Module): 17 | # def __init__(self): 18 | # super(Laplace_edge_layer,self).__init__() 19 | # self.laplace=nn.Conv2d(in_channels=1,out_channels=1,kernel_size=3,stride=1,padding=1,bias=False) 20 | # self.act=nn.Tanh() 21 | # self.weight_init() 22 | # def weight_init(self): 23 | # self.laplace.weight.data=torch.tensor(np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]])[np.newaxis,np.newaxis,...]).float() 24 | # def forward(self, x): 25 | # return self.act(torch.abs(self.laplace(x))) 26 | # 27 | # def iou_edge_loss(edge,label): 28 | # edge, label = torch.squeeze(edge,1), torch.squeeze(label,1) 29 | # inter = torch.sum(torch.mean(torch.mul(edge,label),0)) 30 | # union = torch.add(torch.sum(torch.mean(torch.mul(edge,edge),0)),torch.sum(torch.mean(torch.mul(label,label),0))) 31 | # loss = torch.add(1, -(2 * inter) / union) 32 | # return loss 33 | 34 | 35 | -------------------------------------------------------------------------------- /Code/ADF/config.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | #for net 3 | n_color=3 4 | cuda=True 5 | GPUid=2 6 | mode="test" 7 | useMSA=True 8 | useCA=True 9 | pretrained_model='vgg16.pth' 10 | save_folder="./model" 11 | edge_loss=True 12 | config_vgg = {'deep_pool': [[512, 512, 256, 128], [512, 256, 128, 128], [True, True, True, False], [True, True, True, False]], 'score': 128} 13 | 14 | 15 | #for train 16 | lr=1e-4 17 | wd=0.0005 18 | epoch=25 19 | lr_decay_epoch=[] 20 | batch_size=1 21 | num_thread=1 22 | epoch_save=5 23 | iter_size=10 24 | show_every=100 25 | 26 | train_root="/" 27 | 28 | #for test 29 | model='model/final.pth' 30 | test_root='/' 31 | test_fold="./our/" 32 | -------------------------------------------------------------------------------- /Code/ADF/dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | from PIL import Image 3 | import cv2 4 | import torch 5 | from torch.utils import data 6 | import numpy as np 7 | import random 8 | 9 | 10 | dataset_mean_rgb=np.array([140.72236226, 157.80380783, 135.71176741]) #mean value of trainset rgb 11 | dataset_mean_t=np.array([87.13143683, 92.03255377, 192.20629105]) #mean value of trainset t 12 | 13 | class ImageDataTrain(data.Dataset): 14 | def __init__(self, train_root): 15 | self.data_rgb_root = os.path.join(train_root,"RGB") 16 | self.data_t_root = os.path.join(train_root,"T") 17 | self.gt_root=os.path.join(train_root,"GT") 18 | self.imlist=os.listdir(self.data_rgb_root) 19 | self.im_num = len(self.imlist) 20 | def __getitem__(self, item): 21 | im_name=self.imlist[item % self.im_num] 22 | rgb_img = load_image(os.path.join(self.data_rgb_root, im_name),mode='rgb') 23 | t_img = load_image(os.path.join(self.data_t_root, im_name),mode='t') 24 | label=load_sal_label(os.path.join(self.gt_root, im_name[:-4]+'.png')) 25 | rgb_img,t_img,label = cv_random_flip(rgb_img,t_img,label) 26 | rgb_img = torch.Tensor(rgb_img) 27 | t_img = torch.Tensor(t_img) 28 | label = torch.Tensor(label) 29 | sample = {'rgb_img': rgb_img, 't_img': t_img,'label':label} 30 | return sample 31 | 32 | def __len__(self): 33 | return self.im_num 34 | 35 | 36 | class ImageDataTest(data.Dataset): 37 | def __init__(self,test_root): 38 | self.test_rgb_root = os.path.join(test_root,"RGB") 39 | self.test_t_root = os.path.join(test_root,"T") 40 | self.image_list = os.listdir(self.test_rgb_root) 41 | self.image_num = len(self.image_list) 42 | 43 | def __getitem__(self, item): 44 | rgb_path=os.path.join(self.test_rgb_root, self.image_list[item]) 45 | t_path=os.path.join(self.test_t_root, self.image_list[item]) 46 | rgb,t,im_size = load_image_test(rgb_path,t_path) 47 | rgb = torch.Tensor(rgb) 48 | t = torch.Tensor(t) 49 | return {'rgb': rgb,'t':t, 'name': self.image_list[item % self.image_num], 'size': im_size} 50 | def __len__(self): 51 | return self.image_num 52 | 53 | 54 | def get_loader(config, mode='train', pin=False): 55 | shuffle = False 56 | if mode == 'train': 57 | shuffle = True 58 | dataset = ImageDataTrain(config.train_root) 59 | data_loader = data.DataLoader(dataset=dataset, batch_size=config.batch_size, shuffle=shuffle, num_workers=config.num_thread, pin_memory=pin) 60 | else: 61 | dataset = ImageDataTest(config.test_root) 62 | data_loader = data.DataLoader(dataset=dataset, batch_size=1, shuffle=shuffle, num_workers=1, pin_memory=pin) 63 | return data_loader 64 | 65 | def load_image(path,mode='rgb'): 66 | if not os.path.exists(path): 67 | print('File {} not exists'.format(path)) 68 | im = cv2.imread(path) 69 | in_ = np.array(im, dtype=np.float32) 70 | if mode=='rgb': 71 | in_ -= dataset_mean_rgb 72 | elif mode=='t': 73 | in_ -= dataset_mean_t 74 | in_ = in_.transpose((2,0,1)) 75 | return in_ 76 | 77 | def load_image_test(rgb_path,t_path): 78 | if not os.path.exists(rgb_path): 79 | print('File {} not exists'.format(rgb_path)) 80 | if not os.path.exists(t_path): 81 | print('File {} not exists'.format(t_path)) 82 | rgb,t = cv2.imread(rgb_path),cv2.imread(t_path) 83 | in_rgb,in_t = np.array(rgb, dtype=np.float32),np.array(t, dtype=np.float32) 84 | im_size = tuple(in_rgb.shape[:2]) 85 | in_rgb -= dataset_mean_rgb 86 | in_t -= dataset_mean_t 87 | in_rgb,in_t = in_rgb.transpose((2,0,1)),in_t.transpose((2,0,1)) 88 | return in_rgb,in_t,im_size 89 | 90 | def load_sal_label(path): 91 | im = Image.open(path) 92 | label = np.array(im, dtype=np.float32) 93 | if len(label.shape) == 3: 94 | label = label[:,:,0] 95 | label = label / 255. 96 | label = label[np.newaxis, ...] 97 | return label 98 | 99 | def cv_random_flip(rgb,t,label): 100 | flip_flag = random.randint(0, 1) 101 | if flip_flag == 1: 102 | rgb = rgb[:,:,::-1].copy() 103 | t = t[:, :, ::-1].copy() 104 | label = label[:,:,::-1].copy() 105 | return rgb,t,label 106 | -------------------------------------------------------------------------------- /Code/ADF/get_PRF_mat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import glob2 4 | import os 5 | import scipy.io as sio # mat 6 | import matplotlib.pyplot as plt 7 | eps = 2.2204e-16 8 | 9 | 10 | def parameter(): 11 | p = {} 12 | p['gtThreshold'] = 0.5 13 | p['beta'] = np.sqrt(0.3) 14 | p['thNum'] = 100 15 | p['thList'] = np.linspace(0, 1, p['thNum']) 16 | 17 | return p 18 | 19 | 20 | def im2double(im): 21 | return cv2.normalize(im.astype('float'), 22 | None, 23 | 0.0, 1.0, 24 | cv2.NORM_MINMAX) 25 | 26 | 27 | def prCount(gtMask, curSMap, p): 28 | gtH, gtW = gtMask.shape[0:2] 29 | algH, algW = curSMap.shape[0:2] 30 | 31 | if gtH != algH or gtW != algW: 32 | curSMap = cv2.resize(curSMap, (gtW, gtH)) 33 | 34 | gtMask = (gtMask >= p['gtThreshold']).astype(np.float32) 35 | gtInd = np.where(gtMask > 0) 36 | gtCnt = np.sum(gtMask) 37 | 38 | if gtCnt == 0: 39 | prec = [] 40 | recall = [] 41 | else: 42 | hitCnt = np.zeros((p['thNum'], 1), np.float32) 43 | algCnt = np.zeros((p['thNum'], 1), np.float32) 44 | 45 | for k, curTh in enumerate(p['thList']): 46 | thSMap = (curSMap >= curTh).astype(np.float32) 47 | hitCnt[k] = np.sum(thSMap[gtInd]) 48 | algCnt[k] = np.sum(thSMap) 49 | 50 | prec = hitCnt / (algCnt+eps) 51 | recall = hitCnt / gtCnt 52 | 53 | return prec, recall 54 | 55 | 56 | def PR_Curve(resDir, gtDir): 57 | p = parameter() 58 | beta = p['beta'] 59 | gtImgs = glob2.iglob(gtDir + '/*.png') ######## 60 | 61 | prec = [] 62 | recall = [] 63 | 64 | for gtName in gtImgs: 65 | dir, name = os.path.split(gtName) 66 | mapName = os.path.join(resDir,name[:-4]+'.png') 67 | print(mapName) 68 | curMap = im2double(cv2.imread(mapName, cv2.IMREAD_GRAYSCALE)) 69 | 70 | curGT = im2double(cv2.imread(gtName, cv2.IMREAD_GRAYSCALE)) 71 | 72 | if curMap.shape[0] != curGT.shape[0]: 73 | curMap = cv2.resize(curMap, (curGT.shape[1], curGT.shape[2])) 74 | 75 | curPrec, curRecall = prCount(curGT, curMap, p) 76 | 77 | prec.append(curPrec) 78 | recall.append(curRecall) 79 | 80 | 81 | prec = np.hstack(prec[:]) 82 | recall = np.hstack(recall[:]) 83 | 84 | prec = np.mean(prec, 1) 85 | recall = np.mean(recall, 1) 86 | 87 | # compute the max F-Score 88 | score = (1+beta**2)*prec*recall / (beta**2*prec + recall) 89 | curTh = np.argmax(score) 90 | curScore = np.max(score) 91 | res = {} 92 | res['prec'] = prec 93 | res['recall'] = recall 94 | res['curScore'] = curScore 95 | res['curTh'] = curTh 96 | res['fscore']=score 97 | 98 | 99 | return res 100 | 101 | 102 | def MAE_Value(resDir, gtDir): 103 | p = parameter() 104 | gtThreshold = p['gtThreshold'] 105 | 106 | gtImgs = glob2.iglob(gtDir + '/*.png') 107 | 108 | MAE = [] 109 | 110 | 111 | for gtName in gtImgs: 112 | dir, name = os.path.split(gtName) 113 | mapName= os.path.join(resDir,name[:-4]+'.png') ###### 114 | 115 | #print(mapName) 116 | if os.path.exists(mapName) is False: 117 | mapName = mapName.replace('.png', '.jpg') 118 | if os.path.exists(mapName) is False: 119 | mapName = mapName.replace('.jpg','.bmp') 120 | 121 | curMap = im2double(cv2.imread(mapName, cv2.IMREAD_GRAYSCALE)) 122 | 123 | curGT = im2double(cv2.imread(gtName, cv2.IMREAD_GRAYSCALE)) 124 | curGT = (curGT >= gtThreshold).astype(np.float32) 125 | 126 | if curMap.shape[0] != curGT.shape[0]: 127 | curMap = cv2.resize(curMap, (curGT.shape[1], curGT.shape[2])) 128 | 129 | diff = np.abs(curMap - curGT) 130 | 131 | MAE.append(np.mean(diff)) 132 | 133 | return np.mean(MAE) 134 | 135 | 136 | if __name__ == "__main__": 137 | 138 | method = 'our_rgbt_5000_noat' 139 | resDir = 'our_rgbt_5000_noat' 140 | gtDir = '/home/lizhun/data/VT5000/Test/GT' 141 | mae = MAE_Value(resDir, gtDir) 142 | pr = PR_Curve(resDir, gtDir) 143 | 144 | FMeasureF = pr['curScore'] 145 | print('max F:', pr['curScore']) 146 | print('MAE:', mae) 147 | mat_path = './' + method + '.mat' 148 | print(mat_path) 149 | sio.savemat(mat_path, {'Pre': pr['prec'], 'Recall': pr['recall'],'Fscore':pr['fscore'],'maxF_index':pr['curTh'],'FMeasureF': FMeasureF,'MAE':mae}) 150 | -------------------------------------------------------------------------------- /Code/ADF/run.py: -------------------------------------------------------------------------------- 1 | coding='utf-8' 2 | import config 3 | import os 4 | import solver 5 | from dataset import get_loader 6 | import torch 7 | import random 8 | import numpy as np 9 | 10 | if __name__ == '__main__': 11 | 12 | random.seed(1185) 13 | np.random.seed(1185) 14 | torch.manual_seed(1185) 15 | if config.cuda: 16 | torch.cuda.set_device(config.GPUid) 17 | torch.backends.cudnn.benchmark = False 18 | torch.backends.cudnn.deterministic = True 19 | torch.cuda.manual_seed(1185) 20 | torch.cuda.manual_seed_all(1185) 21 | 22 | if config.mode == 'train': 23 | train_loader = get_loader(config) 24 | if not os.path.exists(config.save_folder): os.mkdir(config.save_folder) 25 | train = solver.train(train_loader,config) 26 | elif config.mode == 'test': 27 | test_loader = get_loader(config, mode='test') 28 | if not os.path.exists(config.test_fold): os.mkdir(config.test_fold) 29 | test = solver.test(test_loader, config) 30 | else: 31 | print("ERROR MODE!") -------------------------------------------------------------------------------- /Code/ADF/solver.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from collections import OrderedDict 3 | from torch.nn import functional as F 4 | from torch.optim import Adam 5 | from APMFnet import build_model 6 | from LaplaceEdge import Laplace_edge_layer 7 | import numpy as np 8 | import os 9 | import cv2 10 | import time 11 | 12 | def train(train_loader,config): 13 | 14 | net = build_model(config) 15 | lap = Laplace_edge_layer() 16 | if config.cuda: 17 | net = net.cuda() 18 | lap = lap.cuda() 19 | 20 | print('loading pretrained model...') 21 | if config.cuda: 22 | stat = torch.load(config.pretrained_model) 23 | else: 24 | stat = torch.load(config.pretrained_model, map_location='cpu') 25 | net.rgb_net.load_pretrained_model(stat) 26 | net.t_net.load_pretrained_model(stat) 27 | newstat = OrderedDict() 28 | for i in stat: 29 | [idx, wtype] = i.split('.') 30 | if int(idx) >= 5: 31 | newstat[str(int(idx) - 4) + '.' + wtype] = stat[i] 32 | net.fusnet.base.load_state_dict(newstat) 33 | 34 | optimizer = Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=config.lr,weight_decay=config.wd) 35 | iter_num = len(train_loader.dataset) // config.batch_size 36 | aveGrad = 0 37 | net.train() 38 | 39 | for epoch in range(1, config.epoch + 1): 40 | r_sal_loss = 0 41 | net.zero_grad() 42 | for i, data_batch in enumerate(train_loader): 43 | rgb, t, label = data_batch['rgb_img'], data_batch['t_img'], data_batch['label'] 44 | if (rgb.size(2) != label.size(2)) or (t.size(2) != label.size(2)) or (rgb.size(3) != label.size(3)) or ( 45 | t.size(3) != label.size(3)): 46 | print('IMAGE ERROR, PASSING```') 47 | continue 48 | if config.cuda: 49 | rgb, t, label = rgb.cuda(), t.cuda(), label.cuda() 50 | sal_pred = net(rgb, t) 51 | sal_loss_fuse = F.binary_cross_entropy_with_logits(sal_pred, label, reduction='sum') 52 | ##edge loss 53 | if config.edge_loss: 54 | edge_pred = lap(torch.sigmoid(sal_pred)) 55 | edge_label = lap(label).detach() 56 | edge_loss = F.binary_cross_entropy(edge_pred, edge_label, reduction='sum') 57 | sal_loss = (0.7 * sal_loss_fuse + 0.3 * edge_loss) / (config.iter_size * config.batch_size) 58 | else: 59 | sal_loss = sal_loss_fuse / (config.iter_size * config.batch_size) 60 | 61 | r_sal_loss += sal_loss.data 62 | sal_loss.backward() 63 | aveGrad += 1 64 | if aveGrad % config.iter_size == 0: 65 | optimizer.step() 66 | optimizer.zero_grad() 67 | aveGrad = 0 68 | 69 | if i % (config.show_every // config.batch_size) == 0: 70 | print('epoch: [%2d/%2d], iter: [%5d/%5d] || loss : %10.4f' % ( 71 | epoch, config.epoch, i, iter_num, r_sal_loss )) 72 | r_sal_loss = 0 73 | if epoch % config.epoch_save == 0: 74 | torch.save(net.state_dict(), '%s/epoch_%d.pth' % (config.save_folder,epoch)) 75 | 76 | if epoch in config.lr_decay_epoch: 77 | optimizer = Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=config.lr* 0.1, 78 | weight_decay=config.wd) 79 | torch.save(net.state_dict(), '%s/final.pth' % config.save_folder) 80 | 81 | def test(test_loader,config): 82 | net = build_model(config) 83 | print('loading model from %s...' % config.model) 84 | if config.cuda: 85 | net = net.cuda() 86 | net.load_state_dict(torch.load(config.model)) 87 | else: 88 | net.load_state_dict(torch.load(config.model, map_location='cpu')) 89 | net.eval() 90 | time_s = time.time() 91 | img_num = len(test_loader) 92 | for i, data_batch in enumerate(test_loader): 93 | rgb, t, name, im_size = data_batch['rgb'], data_batch['t'], data_batch['name'][0], np.asarray( 94 | data_batch['size']) 95 | with torch.no_grad(): 96 | if config.cuda: 97 | rgb, t = rgb.cuda(), t.cuda() 98 | preds = net(rgb, t) 99 | pred = np.squeeze(torch.sigmoid(preds).cpu().data.numpy()) 100 | multi_fuse = 255 * pred 101 | cv2.imwrite(os.path.join(config.test_fold, name[:-4] + '.png'), multi_fuse) 102 | time_e = time.time() 103 | print('speed: %f FPS' % (img_num / (time_e - time_s))) 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /Code/ADF/vgg.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class MSA(nn.Module): 5 | def __init__(self,in_ch,useCA=True): 6 | super(MSA,self).__init__() 7 | self.useCA=useCA 8 | #for spatial mask in: max_map avg_map 9 | self.s_mask=nn.Sequential( 10 | nn.Conv2d(2, 1, 3, 1,1), 11 | nn.ReLU(), 12 | nn.Conv2d(1,1,3,1,1), 13 | nn.Sigmoid() 14 | ) 15 | #for channel weight 16 | if self.useCA: 17 | self.avg_weight=nn.AdaptiveAvgPool2d(1) 18 | self.max_weight=nn.AdaptiveMaxPool2d(1) 19 | self.fus = nn.Sequential( 20 | nn.Conv2d(in_ch,in_ch//2,1,1,0), 21 | nn.ReLU(), 22 | nn.Conv2d(in_ch//2, in_ch, 1, 1, 0), 23 | ) 24 | self.c_mask=nn.Sigmoid() 25 | 26 | def forward(self,x): 27 | if self.useCA: 28 | avg_map_c = self.avg_weight(x) 29 | max_map_c = self.max_weight(x) 30 | c_mask = self.c_mask(torch.add(self.fus(avg_map_c),self.fus(max_map_c))) 31 | x = torch.mul(x,c_mask) 32 | max_map_s = torch.unsqueeze(torch.max(x,1)[0],1) 33 | avg_map_s = torch.unsqueeze(torch.mean(x,1),1) 34 | map_s = torch.cat((max_map_s,avg_map_s),1) 35 | s_mask = self.s_mask(map_s) 36 | x = torch.mul(x,s_mask) 37 | return x 38 | 39 | # vgg16 40 | def vgg(cfg, i, batch_norm=False): 41 | layers = [] 42 | in_channels = i 43 | stage = 1 44 | for v in cfg: 45 | if v == 'M': 46 | stage += 1 47 | if stage == 6: 48 | layers += [nn.MaxPool2d(kernel_size=3, stride=1, padding=1)] 49 | else: 50 | layers += [nn.MaxPool2d(kernel_size=3, stride=2, padding=1)] 51 | else: 52 | if stage == 6: 53 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) 54 | else: 55 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) 56 | if batch_norm: 57 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 58 | else: 59 | layers += [conv2d, nn.ReLU(inplace=True)] 60 | in_channels = v 61 | return layers 62 | 63 | class a_vgg16(nn.Module): 64 | def __init__(self,config): 65 | super(a_vgg16, self).__init__() 66 | self.cfg =[64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'] 67 | self.extract = [3, 8, 15, 22, 29] # [ 8, 15, 22, 29] 68 | # 64:1 -->128:1/2 -->256:1/4 -->512 :1/8 --> 512:1/16 -->M-> 512,1/16 69 | self.base = nn.ModuleList(vgg(self.cfg, 3)) 70 | 71 | #MSA 72 | self.useMSA=config.useMSA 73 | if self.useMSA: 74 | self.feature_channels=[64,128,256,512,512] 75 | msas=[] 76 | for fc in self.feature_channels: 77 | msas+=[MSA(fc,config.useCA)] 78 | self.msas=nn.ModuleList(msas) 79 | else: 80 | print('no MSA used!') 81 | 82 | for m in self.modules(): 83 | if isinstance(m, nn.Conv2d): 84 | m.weight.data.normal_(0, 0.01) 85 | elif isinstance(m, nn.BatchNorm2d): 86 | m.weight.data.fill_(1) 87 | m.bias.data.zero_() 88 | 89 | def load_pretrained_model(self, model): 90 | self.base.load_state_dict(model, strict=False) 91 | 92 | def forward(self, x): 93 | tmp_x = [] 94 | index=0 95 | for k in range(len(self.base)): 96 | x = self.base[k](x) 97 | if k in self.extract: 98 | if self.useMSA: 99 | x=self.msas[index](x) 100 | index+=1 101 | tmp_x.append(x) #collect feature maps 1(64) 1/2(128) 1/4(256) 1/8(512) 1/16(512) 102 | return tmp_x 103 | 104 | -------------------------------------------------------------------------------- /Code/M3S-NIR/AdjcProcloop.m: -------------------------------------------------------------------------------- 1 | function adjcMerge = AdjcProcloop(M,N) 2 | % $Description: 3 | % -compute the adjacent matrix 4 | % $Agruments 5 | % Input; 6 | % -M: superpixel label matrix 7 | % -N: superpixel number 8 | % Output: 9 | % -adjcMerge: adjacent matrix 10 | 11 | adjcMerge = zeros(N,N); 12 | [m n] = size(M); 13 | 14 | for i = 1:m-1 15 | for j = 1:n-1 16 | if(M(i,j)~=M(i,j+1)) 17 | adjcMerge(M(i,j),M(i,j+1)) = 1; 18 | adjcMerge(M(i,j+1),M(i,j)) = 1; 19 | end; 20 | if(M(i,j)~=M(i+1,j)) 21 | adjcMerge(M(i,j),M(i+1,j)) = 1; 22 | adjcMerge(M(i+1,j),M(i,j)) = 1; 23 | end; 24 | if(M(i,j)~=M(i+1,j+1)) 25 | adjcMerge(M(i,j),M(i+1,j+1)) = 1; 26 | adjcMerge(M(i+1,j+1),M(i,j)) = 1; 27 | end; 28 | if(M(i+1,j)~=M(i,j+1)) 29 | adjcMerge(M(i+1,j),M(i,j+1)) = 1; 30 | adjcMerge(M(i,j+1),M(i+1,j)) = 1; 31 | end; 32 | end; 33 | end; 34 | bd=unique([M(1,:),M(m,:),M(:,1)',M(:,n)']); 35 | for i=1:length(bd) 36 | for j=i+1:length(bd) 37 | adjcMerge(bd(i),bd(j))=1; 38 | adjcMerge(bd(j),bd(i))=1; 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /Code/M3S-NIR/ComputeAffinityMatrix.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/ComputeAffinityMatrix.m -------------------------------------------------------------------------------- /Code/M3S-NIR/DEMO11.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/DEMO11.m -------------------------------------------------------------------------------- /Code/M3S-NIR/Plot_PreRecallThousand.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/Plot_PreRecallThousand.m -------------------------------------------------------------------------------- /Code/M3S-NIR/ReadDAT.m: -------------------------------------------------------------------------------- 1 | function B = ReadDAT(image_size,data_path) 2 | % $Description: 3 | % -read the superpixel labels from .dat file 4 | % $Agruments 5 | % Input; 6 | % -image_size: [width height] 7 | % -data_path: the path of the .dat file 8 | % Output: 9 | % -label matrix width*height 10 | 11 | row = image_size(1); 12 | colomn = image_size(2); 13 | fid = fopen(data_path,'r'); 14 | A = fread(fid, row * colomn, 'uint32')'; 15 | A = A + 1; 16 | B = reshape(A,[colomn, row]); 17 | B = B'; 18 | fclose(fid); -------------------------------------------------------------------------------- /Code/M3S-NIR/SLICSuperpixelSegmentation.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/SLICSuperpixelSegmentation.exe -------------------------------------------------------------------------------- /Code/M3S-NIR/_SLIC.bmp: -------------------------------------------------------------------------------- 1 | BM66( -------------------------------------------------------------------------------- /Code/M3S-NIR/adjacency.m: -------------------------------------------------------------------------------- 1 | function W=adjacency(edges,weights,N) 2 | %Function W=adjacency(edges,weights,N) computes the weighted 3 | % adjacency matrix of a point and edge set. For an unweighted 4 | % matrix, set weights equal to ones(1,M) where M is the number 5 | % of edges in the graph. 6 | % 7 | %Inputs: edges - A Mx2 list of M edges indexing into points 8 | % weights - The weights used to determine matrix values. 9 | % If not specified, uses vector of all ones 10 | % N - Optional number of nodes in the graph. Used if 11 | % N > max(edges) (i.e., isolated nodes are present) 12 | % 13 | %Outputs: W - Adjacency matrix 14 | % 15 | % 16 | %Note: Adjacency matrix is not of the Seidel type 17 | % (i.e. all values are positive) 18 | %5/19/03 - Leo Grady 19 | 20 | % Copyright (C) 2002, 2003 Leo Grady 21 | % Computer Vision and Computational Neuroscience Lab 22 | % Department of Cognitive and Neural Systems 23 | % Boston University 24 | % Boston, MA 02215 25 | % 26 | % This program is free software; you can redistribute it and/or 27 | % modify it under the terms of the GNU General Public License 28 | % as published by the Free Software Foundation; either version 2 29 | % of the License, or (at your option) any later version. 30 | % 31 | % This program is distributed in the hope that it will be useful, 32 | % but WITHOUT ANY WARRANTY; without even the implied warranty of 33 | % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 34 | % GNU General Public License for more details. 35 | % 36 | % You should have received a copy of the GNU General Public License 37 | % along with this program; if not, write to the Free Software 38 | % Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 39 | % 40 | % Date - $Id: adjacency.m,v 1.2 2003/08/21 17:29:29 lgrady Exp $ 41 | %========================================================================% 42 | 43 | %If weights are not specified, use unity weighting 44 | if nargin == 1 45 | weights=ones(size(edges,1),1); 46 | end 47 | 48 | %If N is not specified, use maximum values of edges 49 | if nargin < 3 50 | N=max(max(edges)); 51 | end 52 | 53 | %Build sparse adjacency matrix 54 | W=sparse([edges(:,1);edges(:,2)],[edges(:,2);edges(:,1)], ... 55 | [weights;weights],N,N); 56 | -------------------------------------------------------------------------------- /Code/M3S-NIR/colorspace.m: -------------------------------------------------------------------------------- 1 | function varargout = colorspace(Conversion,varargin) 2 | %COLORSPACE Convert a color image between color representations. 3 | % B = COLORSPACE(S,A) converts the color representation of image A 4 | % where S is a string specifying the conversion. S tells the 5 | % source and destination color spaces, S = 'dest<-src', or 6 | % alternatively, S = 'src->dest'. Supported color spaces are 7 | % 8 | % 'RGB' R'G'B' Red Green Blue (ITU-R BT.709 gamma-corrected) 9 | % 'YPbPr' Luma (ITU-R BT.601) + Chroma 10 | % 'YCbCr'/'YCC' Luma + Chroma ("digitized" version of Y'PbPr) 11 | % 'YUV' NTSC PAL Y'UV Luma + Chroma 12 | % 'YIQ' NTSC Y'IQ Luma + Chroma 13 | % 'YDbDr' SECAM Y'DbDr Luma + Chroma 14 | % 'JPEGYCbCr' JPEG-Y'CbCr Luma + Chroma 15 | % 'HSV'/'HSB' Hue Saturation Value/Brightness 16 | % 'HSL'/'HLS'/'HSI' Hue Saturation Luminance/Intensity 17 | % 'XYZ' CIE XYZ 18 | % 'Lab' CIE L*a*b* (CIELAB) 19 | % 'Luv' CIE L*u*v* (CIELUV) 20 | % 'Lch' CIE L*ch (CIELCH) 21 | % 22 | % All conversions assume 2 degree observer and D65 illuminant. Color 23 | % space names are case insensitive. When R'G'B' is the source or 24 | % destination, it can be omitted. For example 'yuv<-' is short for 25 | % 'yuv<-rgb'. 26 | % 27 | % MATLAB uses two standard data formats for R'G'B': double data with 28 | % intensities in the range 0 to 1, and uint8 data with integer-valued 29 | % intensities from 0 to 255. As MATLAB's native datatype, double data is 30 | % the natural choice, and the R'G'B' format used by colorspace. However, 31 | % for memory and computational performance, some functions also operate 32 | % with uint8 R'G'B'. Given uint8 R'G'B' color data, colorspace will 33 | % first cast it to double R'G'B' before processing. 34 | % 35 | % If A is an Mx3 array, like a colormap, B will also have size Mx3. 36 | % 37 | % [B1,B2,B3] = COLORSPACE(S,A) specifies separate output channels. 38 | % COLORSPACE(S,A1,A2,A3) specifies separate input channels. 39 | 40 | % Pascal Getreuer 2005-2006 41 | 42 | %%% Input parsing %%% 43 | if nargin < 2, error('Not enough input arguments.'); end 44 | [SrcSpace,DestSpace] = parse(Conversion); 45 | 46 | if nargin == 2 47 | Image = varargin{1}; 48 | elseif nargin >= 3 49 | Image = cat(3,varargin{:}); 50 | else 51 | error('Invalid number of input arguments.'); 52 | end 53 | 54 | FlipDims = (size(Image,3) == 1); 55 | 56 | if FlipDims, Image = permute(Image,[1,3,2]); end 57 | if ~isa(Image,'double'), Image = double(Image)/255; end 58 | if size(Image,3) ~= 3, error('Invalid input size.'); end 59 | 60 | SrcT = gettransform(SrcSpace); 61 | DestT = gettransform(DestSpace); 62 | 63 | if ~ischar(SrcT) & ~ischar(DestT) 64 | % Both source and destination transforms are affine, so they 65 | % can be composed into one affine operation 66 | T = [DestT(:,1:3)*SrcT(:,1:3),DestT(:,1:3)*SrcT(:,4)+DestT(:,4)]; 67 | Temp = zeros(size(Image)); 68 | Temp(:,:,1) = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3) + T(10); 69 | Temp(:,:,2) = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3) + T(11); 70 | Temp(:,:,3) = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3) + T(12); 71 | Image = Temp; 72 | elseif ~ischar(DestT) 73 | Image = rgb(Image,SrcSpace); 74 | Temp = zeros(size(Image)); 75 | Temp(:,:,1) = DestT(1)*Image(:,:,1) + DestT(4)*Image(:,:,2) + DestT(7)*Image(:,:,3) + DestT(10); 76 | Temp(:,:,2) = DestT(2)*Image(:,:,1) + DestT(5)*Image(:,:,2) + DestT(8)*Image(:,:,3) + DestT(11); 77 | Temp(:,:,3) = DestT(3)*Image(:,:,1) + DestT(6)*Image(:,:,2) + DestT(9)*Image(:,:,3) + DestT(12); 78 | Image = Temp; 79 | else 80 | Image = feval(DestT,Image,SrcSpace); 81 | end 82 | 83 | %%% Output format %%% 84 | if nargout > 1 85 | varargout = {Image(:,:,1),Image(:,:,2),Image(:,:,3)}; 86 | else 87 | if FlipDims, Image = permute(Image,[1,3,2]); end 88 | varargout = {Image}; 89 | end 90 | 91 | return; 92 | 93 | 94 | function [SrcSpace,DestSpace] = parse(Str) 95 | % Parse conversion argument 96 | 97 | if isstr(Str) 98 | Str = lower(strrep(strrep(Str,'-',''),' ','')); 99 | k = find(Str == '>'); 100 | 101 | if length(k) == 1 % Interpret the form 'src->dest' 102 | SrcSpace = Str(1:k-1); 103 | DestSpace = Str(k+1:end); 104 | else 105 | k = find(Str == '<'); 106 | 107 | if length(k) == 1 % Interpret the form 'dest<-src' 108 | DestSpace = Str(1:k-1); 109 | SrcSpace = Str(k+1:end); 110 | else 111 | error(['Invalid conversion, ''',Str,'''.']); 112 | end 113 | end 114 | 115 | SrcSpace = alias(SrcSpace); 116 | DestSpace = alias(DestSpace); 117 | else 118 | SrcSpace = 1; % No source pre-transform 119 | DestSpace = Conversion; 120 | if any(size(Conversion) ~= 3), error('Transformation matrix must be 3x3.'); end 121 | end 122 | return; 123 | 124 | 125 | function Space = alias(Space) 126 | Space = strrep(Space,'cie',''); 127 | 128 | if isempty(Space) 129 | Space = 'rgb'; 130 | end 131 | 132 | switch Space 133 | case {'ycbcr','ycc'} 134 | Space = 'ycbcr'; 135 | case {'hsv','hsb'} 136 | Space = 'hsv'; 137 | case {'hsl','hsi','hls'} 138 | Space = 'hsl'; 139 | case {'rgb','yuv','yiq','ydbdr','ycbcr','jpegycbcr','xyz','lab','luv','lch'} 140 | return; 141 | end 142 | return; 143 | 144 | 145 | function T = gettransform(Space) 146 | % Get a colorspace transform: either a matrix describing an affine transform, 147 | % or a string referring to a conversion subroutine 148 | switch Space 149 | case 'ypbpr' 150 | T = [0.299,0.587,0.114,0;-0.1687367,-0.331264,0.5,0;0.5,-0.418688,-0.081312,0]; 151 | case 'yuv' 152 | % R'G'B' to NTSC/PAL YUV 153 | % Wikipedia: http://en.wikipedia.org/wiki/YUV 154 | T = [0.299,0.587,0.114,0;-0.147,-0.289,0.436,0;0.615,-0.515,-0.100,0]; 155 | case 'ydbdr' 156 | % R'G'B' to SECAM YDbDr 157 | % Wikipedia: http://en.wikipedia.org/wiki/YDbDr 158 | T = [0.299,0.587,0.114,0;-0.450,-0.883,1.333,0;-1.333,1.116,0.217,0]; 159 | case 'yiq' 160 | % R'G'B' in [0,1] to NTSC YIQ in [0,1];[-0.595716,0.595716];[-0.522591,0.522591]; 161 | % Wikipedia: http://en.wikipedia.org/wiki/YIQ 162 | T = [0.299,0.587,0.114,0;0.595716,-0.274453,-0.321263,0;0.211456,-0.522591,0.311135,0]; 163 | case 'ycbcr' 164 | % R'G'B' (range [0,1]) to ITU-R BRT.601 (CCIR 601) Y'CbCr 165 | % Wikipedia: http://en.wikipedia.org/wiki/YCbCr 166 | % Poynton, Equation 3, scaling of R'G'B to Y'PbPr conversion 167 | T = [65.481,128.553,24.966,16;-37.797,-74.203,112.0,128;112.0,-93.786,-18.214,128]; 168 | case 'jpegycbcr' 169 | % Wikipedia: http://en.wikipedia.org/wiki/YCbCr 170 | T = [0.299,0.587,0.114,0;-0.168736,-0.331264,0.5,0.5;0.5,-0.418688,-0.081312,0.5]*255; 171 | case {'rgb','xyz','hsv','hsl','lab','luv','lch'} 172 | T = Space; 173 | otherwise 174 | error(['Unknown color space, ''',Space,'''.']); 175 | end 176 | return; 177 | 178 | 179 | function Image = rgb(Image,SrcSpace) 180 | % Convert to Rec. 709 R'G'B' from 'SrcSpace' 181 | switch SrcSpace 182 | case 'rgb' 183 | return; 184 | case 'hsv' 185 | % Convert HSV to R'G'B' 186 | Image = huetorgb((1 - Image(:,:,2)).*Image(:,:,3),Image(:,:,3),Image(:,:,1)); 187 | case 'hsl' 188 | % Convert HSL to R'G'B' 189 | L = Image(:,:,3); 190 | Delta = Image(:,:,2).*min(L,1-L); 191 | Image = huetorgb(L-Delta,L+Delta,Image(:,:,1)); 192 | case {'xyz','lab','luv','lch'} 193 | % Convert to CIE XYZ 194 | Image = xyz(Image,SrcSpace); 195 | % Convert XYZ to RGB 196 | T = [3.240479,-1.53715,-0.498535;-0.969256,1.875992,0.041556;0.055648,-0.204043,1.057311]; 197 | R = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3); % R 198 | G = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3); % G 199 | B = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3); % B 200 | % Desaturate and rescale to constrain resulting RGB values to [0,1] 201 | AddWhite = -min(min(min(R,G),B),0); 202 | Scale = max(max(max(R,G),B)+AddWhite,1); 203 | R = (R + AddWhite)./Scale; 204 | G = (G + AddWhite)./Scale; 205 | B = (B + AddWhite)./Scale; 206 | % Apply gamma correction to convert RGB to Rec. 709 R'G'B' 207 | Image(:,:,1) = gammacorrection(R); % R' 208 | Image(:,:,2) = gammacorrection(G); % G' 209 | Image(:,:,3) = gammacorrection(B); % B' 210 | otherwise % Conversion is through an affine transform 211 | T = gettransform(SrcSpace); 212 | temp = inv(T(:,1:3)); 213 | T = [temp,-temp*T(:,4)]; 214 | R = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3) + T(10); 215 | G = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3) + T(11); 216 | B = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3) + T(12); 217 | AddWhite = -min(min(min(R,G),B),0); 218 | Scale = max(max(max(R,G),B)+AddWhite,1); 219 | R = (R + AddWhite)./Scale; 220 | G = (G + AddWhite)./Scale; 221 | B = (B + AddWhite)./Scale; 222 | Image(:,:,1) = R; 223 | Image(:,:,2) = G; 224 | Image(:,:,3) = B; 225 | end 226 | 227 | % Clip to [0,1] 228 | Image = min(max(Image,0),1); 229 | return; 230 | 231 | 232 | function Image = xyz(Image,SrcSpace) 233 | % Convert to CIE XYZ from 'SrcSpace' 234 | WhitePoint = [0.950456,1,1.088754]; 235 | 236 | switch SrcSpace 237 | case 'xyz' 238 | return; 239 | case 'luv' 240 | % Convert CIE L*uv to XYZ 241 | WhitePointU = (4*WhitePoint(1))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 242 | WhitePointV = (9*WhitePoint(2))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 243 | L = Image(:,:,1); 244 | Y = (L + 16)/116; 245 | Y = invf(Y)*WhitePoint(2); 246 | U = Image(:,:,2)./(13*L + 1e-6*(L==0)) + WhitePointU; 247 | V = Image(:,:,3)./(13*L + 1e-6*(L==0)) + WhitePointV; 248 | Image(:,:,1) = -(9*Y.*U)./((U-4).*V - U.*V); % X 249 | Image(:,:,2) = Y; % Y 250 | Image(:,:,3) = (9*Y - (15*V.*Y) - (V.*Image(:,:,1)))./(3*V); % Z 251 | case {'lab','lch'} 252 | Image = lab(Image,SrcSpace); 253 | % Convert CIE L*ab to XYZ 254 | fY = (Image(:,:,1) + 16)/116; 255 | fX = fY + Image(:,:,2)/500; 256 | fZ = fY - Image(:,:,3)/200; 257 | Image(:,:,1) = WhitePoint(1)*invf(fX); % X 258 | Image(:,:,2) = WhitePoint(2)*invf(fY); % Y 259 | Image(:,:,3) = WhitePoint(3)*invf(fZ); % Z 260 | otherwise % Convert from some gamma-corrected space 261 | % Convert to Rec. 701 R'G'B' 262 | Image = rgb(Image,SrcSpace); 263 | % Undo gamma correction 264 | R = invgammacorrection(Image(:,:,1)); 265 | G = invgammacorrection(Image(:,:,2)); 266 | B = invgammacorrection(Image(:,:,3)); 267 | % Convert RGB to XYZ 268 | T = inv([3.240479,-1.53715,-0.498535;-0.969256,1.875992,0.041556;0.055648,-0.204043,1.057311]); 269 | Image(:,:,1) = T(1)*R + T(4)*G + T(7)*B; % X 270 | Image(:,:,2) = T(2)*R + T(5)*G + T(8)*B; % Y 271 | Image(:,:,3) = T(3)*R + T(6)*G + T(9)*B; % Z 272 | end 273 | return; 274 | 275 | 276 | function Image = hsv(Image,SrcSpace) 277 | % Convert to HSV 278 | Image = rgb(Image,SrcSpace); 279 | V = max(Image,[],3); 280 | S = (V - min(Image,[],3))./(V + (V == 0)); 281 | Image(:,:,1) = rgbtohue(Image); 282 | Image(:,:,2) = S; 283 | Image(:,:,3) = V; 284 | return; 285 | 286 | 287 | function Image = hsl(Image,SrcSpace) 288 | % Convert to HSL 289 | switch SrcSpace 290 | case 'hsv' 291 | % Convert HSV to HSL 292 | MaxVal = Image(:,:,3); 293 | MinVal = (1 - Image(:,:,2)).*MaxVal; 294 | L = 0.5*(MaxVal + MinVal); 295 | temp = min(L,1-L); 296 | Image(:,:,2) = 0.5*(MaxVal - MinVal)./(temp + (temp == 0)); 297 | Image(:,:,3) = L; 298 | otherwise 299 | Image = rgb(Image,SrcSpace); % Convert to Rec. 701 R'G'B' 300 | % Convert R'G'B' to HSL 301 | MinVal = min(Image,[],3); 302 | MaxVal = max(Image,[],3); 303 | L = 0.5*(MaxVal + MinVal); 304 | temp = min(L,1-L); 305 | S = 0.5*(MaxVal - MinVal)./(temp + (temp == 0)); 306 | Image(:,:,1) = rgbtohue(Image); 307 | Image(:,:,2) = S; 308 | Image(:,:,3) = L; 309 | end 310 | return; 311 | 312 | 313 | function Image = lab(Image,SrcSpace) 314 | % Convert to CIE L*a*b* (CIELAB) 315 | WhitePoint = [0.950456,1,1.088754]; 316 | 317 | switch SrcSpace 318 | case 'lab' 319 | return; 320 | case 'lch' 321 | % Convert CIE L*CH to CIE L*ab 322 | C = Image(:,:,2); 323 | Image(:,:,2) = cos(Image(:,:,3)*pi/180).*C; % a* 324 | Image(:,:,3) = sin(Image(:,:,3)*pi/180).*C; % b* 325 | otherwise 326 | Image = xyz(Image,SrcSpace); % Convert to XYZ 327 | % Convert XYZ to CIE L*a*b* 328 | X = Image(:,:,1)/WhitePoint(1); 329 | Y = Image(:,:,2)/WhitePoint(2); 330 | Z = Image(:,:,3)/WhitePoint(3); 331 | fX = f(X); 332 | fY = f(Y); 333 | fZ = f(Z); 334 | Image(:,:,1) = 116*fY - 16; % L* 335 | Image(:,:,2) = 500*(fX - fY); % a* 336 | Image(:,:,3) = 200*(fY - fZ); % b* 337 | end 338 | return; 339 | 340 | 341 | function Image = luv(Image,SrcSpace) 342 | % Convert to CIE L*u*v* (CIELUV) 343 | WhitePoint = [0.950456,1,1.088754]; 344 | WhitePointU = (4*WhitePoint(1))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 345 | WhitePointV = (9*WhitePoint(2))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 346 | 347 | Image = xyz(Image,SrcSpace); % Convert to XYZ 348 | U = (4*Image(:,:,1))./(Image(:,:,1) + 15*Image(:,:,2) + 3*Image(:,:,3)); 349 | V = (9*Image(:,:,2))./(Image(:,:,1) + 15*Image(:,:,2) + 3*Image(:,:,3)); 350 | Y = Image(:,:,2)/WhitePoint(2); 351 | L = 116*f(Y) - 16; 352 | Image(:,:,1) = L; % L* 353 | Image(:,:,2) = 13*L.*(U - WhitePointU); % u* 354 | Image(:,:,3) = 13*L.*(V - WhitePointV); % v* 355 | return; 356 | 357 | 358 | function Image = lch(Image,SrcSpace) 359 | % Convert to CIE L*ch 360 | Image = lab(Image,SrcSpace); % Convert to CIE L*ab 361 | H = atan2(Image(:,:,3),Image(:,:,2)); 362 | H = H*180/pi + 360*(H < 0); 363 | Image(:,:,2) = sqrt(Image(:,:,2).^2 + Image(:,:,3).^2); % C 364 | Image(:,:,3) = H; % H 365 | return; 366 | 367 | 368 | function Image = huetorgb(m0,m2,H) 369 | % Convert HSV or HSL hue to RGB 370 | N = size(H); 371 | H = min(max(H(:),0),360)/60; 372 | m0 = m0(:); 373 | m2 = m2(:); 374 | F = H - round(H/2)*2; 375 | M = [m0, m0 + (m2-m0).*abs(F), m2]; 376 | Num = length(m0); 377 | j = [2 1 0;1 2 0;0 2 1;0 1 2;1 0 2;2 0 1;2 1 0]*Num; 378 | k = floor(H) + 1; 379 | Image = reshape([M(j(k,1)+(1:Num).'),M(j(k,2)+(1:Num).'),M(j(k,3)+(1:Num).')],[N,3]); 380 | return; 381 | 382 | 383 | function H = rgbtohue(Image) 384 | % Convert RGB to HSV or HSL hue 385 | [M,i] = sort(Image,3); 386 | i = i(:,:,3); 387 | Delta = M(:,:,3) - M(:,:,1); 388 | Delta = Delta + (Delta == 0); 389 | R = Image(:,:,1); 390 | G = Image(:,:,2); 391 | B = Image(:,:,3); 392 | H = zeros(size(R)); 393 | k = (i == 1); 394 | H(k) = (G(k) - B(k))./Delta(k); 395 | k = (i == 2); 396 | H(k) = 2 + (B(k) - R(k))./Delta(k); 397 | k = (i == 3); 398 | H(k) = 4 + (R(k) - G(k))./Delta(k); 399 | H = 60*H + 360*(H < 0); 400 | H(Delta == 0) = nan; 401 | return; 402 | 403 | 404 | function Rp = gammacorrection(R) 405 | Rp = real(1.099*R.^0.45 - 0.099); 406 | i = (R < 0.018); 407 | Rp(i) = 4.5138*R(i); 408 | return; 409 | 410 | 411 | function R = invgammacorrection(Rp) 412 | R = real(((Rp + 0.099)/1.099).^(1/0.45)); 413 | i = (R < 0.018); 414 | R(i) = Rp(i)/4.5138; 415 | return; 416 | 417 | 418 | function fY = f(Y) 419 | fY = real(Y.^(1/3)); 420 | i = (Y < 0.008856); 421 | fY(i) = Y(i)*(841/108) + (4/29); 422 | return; 423 | 424 | 425 | function Y = invf(fY) 426 | Y = fY.^3; 427 | i = (Y < 0.008856); 428 | Y(i) = (fY(i) - 4/29)*(108/841); 429 | return; 430 | -------------------------------------------------------------------------------- /Code/M3S-NIR/compare_PRF.m: -------------------------------------------------------------------------------- 1 | clc 2 | clear 3 | 4 | MR=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\MR\out.mat') 5 | plot(MR.Recall ,MR.Pre, 'c-' ,'LineWidth',2) 6 | axis([0 1 0 1]) 7 | xlabel('Recall') 8 | ylabel('Precision') 9 | hold on 10 | 11 | MSS=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\MSS\out.mat') 12 | plot(MSS.Recall ,MSS.Pre, 'g--' ,'LineWidth', 2 ) 13 | axis([0 1 0 1]) 14 | xlabel('Recall') 15 | ylabel('Precision') 16 | hold on 17 | 18 | RBD=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\RBD\out.mat') 19 | plot(RBD.Recall ,RBD.Pre, 'b-' ,'LineWidth', 2 ) 20 | axis([0 1 0 1]) 21 | xlabel('Recall') 22 | ylabel('Precision') 23 | hold on 24 | 25 | CA=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\BSCA\out.mat') 26 | plot(CA.Recall ,CA.Pre, 'b-.' ,'LineWidth',2) 27 | axis([0 1 0 1]) 28 | xlabel('Recall') 29 | ylabel('Precision') 30 | hold on 31 | 32 | BL=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\BL\out.mat') 33 | plot(BL.Recall ,BL.Pre, 'y-' ,'LineWidth', 2 ) 34 | axis([0 1 0 1]) 35 | xlabel('Recall') 36 | ylabel('Precision') 37 | hold on 38 | 39 | RRWR=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\RRWR\out.mat') 40 | plot(RRWR.Recall ,RRWR.Pre, 'g-' ,'LineWidth',2) 41 | axis([0 1 0 1]) 42 | xlabel('Recall') 43 | ylabel('Precision') 44 | hold on 45 | 46 | FCNN=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\FCNN\out.mat') 47 | plot(FCNN.Recall ,FCNN.Pre, 'c-.' ,'LineWidth', 2 ) 48 | axis([0 1 0 1]) 49 | xlabel('Recall') 50 | ylabel('Precision') 51 | hold on 52 | 53 | DSS=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\DSS\out.mat') 54 | plot(DSS.Recall ,DSS.Pre, 'y-.' ,'LineWidth', 2 ) 55 | axis([0 1 0 1]) 56 | xlabel('Recall') 57 | ylabel('Precision') 58 | hold on 59 | 60 | MTMR= load('E:\WORK2\AAAI+IEEETran\AAAI compare data\821saliencymap\MTMR\out.mat') 61 | plot(MTMR.Recall ,MTMR.Pre, 'm-.' ,'LineWidth', 2 ) 62 | axis([0 1 0 1]) 63 | xlabel('Recall') 64 | ylabel('Precision') 65 | hold on 66 | 67 | Old=load('E:\WORK1\2MIPR\our\1\out.mat') 68 | plot(Old.Recall ,Old.Pre, 'b-' ,'LineWidth',2) 69 | axis([0 1 0 1]) 70 | xlabel('Recall') 71 | ylabel('Precision') 72 | hold on 73 | 74 | Our=load('E:\WORK1\lambda=0.01;lambda1=0.7;lambda2=0.01;lambda3=0.5;lambda4=0.5;\out.mat') 75 | plot(Our.Recall ,Our.Pre, 'r-' ,'LineWidth',2) 76 | axis([0 1 0 1]) 77 | xlabel('Recall') 78 | ylabel('Precision') 79 | hold on 80 | 81 | methods={'MR','MSS','RBD','CA','BL','RRWR','FCNN','DSS','MTMR','M3S-NIR','Our'}; 82 | precisions = {MR.FMeasureF, MSS.FMeasureF, RBD.FMeasureF, CA.FMeasureF, BL.FMeasureF, RRWR.FMeasureF, FCNN.FMeasureF, DSS.FMeasureF, MTMR.FMeasureF,Old.FMeasureF,Our.FMeasureF}; 83 | 84 | 85 | for i = 1:size(methods, 2) 86 | legendlabel{i} = [methods{i} ' [' sprintf('%.3f', precisions{i}) ']']; 87 | end 88 | legend(legendlabel, 'Location', 'SouthWest', 'FontSize', 13) 89 | 90 | -------------------------------------------------------------------------------- /Code/M3S-NIR/compare_PRF1k.m: -------------------------------------------------------------------------------- 1 | clc 2 | clear 3 | figure; 4 | MR=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\MR\out.mat') 5 | plot(MR.Recall ,MR.Pre, 'c-' ,'LineWidth',2) 6 | axis([0 1 0 1]) 7 | xlabel('Recall') 8 | ylabel('Precision') 9 | hold on 10 | 11 | MSS=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\MSS\out.mat') 12 | plot(MSS.Recall ,MSS.Pre, 'g--' ,'LineWidth', 2 ) 13 | axis([0 1 0 1]) 14 | xlabel('Recall') 15 | ylabel('Precision') 16 | hold on 17 | 18 | RBD=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\RBD\out.mat') 19 | plot(RBD.Recall ,RBD.Pre, 'b-' ,'LineWidth', 2 ) 20 | axis([0 1 0 1]) 21 | xlabel('Recall') 22 | ylabel('Precision') 23 | hold on 24 | 25 | CA=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\BSCA\out.mat') 26 | plot(CA.Recall ,CA.Pre, 'b-.' ,'LineWidth',2) 27 | axis([0 1 0 1]) 28 | xlabel('Recall') 29 | ylabel('Precision') 30 | hold on 31 | 32 | BL=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\BL\out.mat') 33 | plot(BL.Recall ,BL.Pre, 'y-' ,'LineWidth', 2 ) 34 | axis([0 1 0 1]) 35 | xlabel('Recall') 36 | ylabel('Precision') 37 | hold on 38 | 39 | RRWR=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\RRWR\out.mat') 40 | plot(RRWR.Recall ,RRWR.Pre, 'g-' ,'LineWidth',2) 41 | axis([0 1 0 1]) 42 | xlabel('Recall') 43 | ylabel('Precision') 44 | hold on 45 | 46 | % FCNN=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\FCNN\out.mat') 47 | % plot(FCNN.Recall ,FCNN.Pre, 'c-.' ,'LineWidth', 2 ) 48 | % axis([0 1 0 1]) 49 | % xlabel('Recall') 50 | % ylabel('Precision') 51 | % hold on 52 | 53 | DSS=load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\DSS\out.mat') 54 | plot(DSS.Recall ,DSS.Pre, 'y-.' ,'LineWidth', 2 ) 55 | axis([0 1 0 1]) 56 | xlabel('Recall') 57 | ylabel('Precision') 58 | hold on 59 | 60 | MTMR= load('E:\WORK2\AAAI+IEEETran\AAAI compare data\1000saliencymap\RGBT\MTMR\out.mat') 61 | plot(MTMR.Recall ,MTMR.Pre, 'm-.' ,'LineWidth', 2 ) 62 | axis([0 1 0 1]) 63 | xlabel('Recall') 64 | ylabel('Precision') 65 | hold on 66 | 67 | 68 | Old=load('E:\WORK1\2MIPR\our\1k\out.mat') 69 | plot(Old.Recall ,Old.Pre, 'b-' ,'LineWidth',2) 70 | axis([0 1 0 1]) 71 | xlabel('Recall') 72 | ylabel('Precision') 73 | hold on 74 | 75 | Our=load('E:\WORK1\code\theta1=40;theta2=40;gama=1.5;\out.mat') 76 | plot( Our.Recall , Our.Pre, 'r-' ,'LineWidth',2) 77 | axis([0 1 0 1]) 78 | xlabel('Recall') 79 | ylabel('Precision') 80 | hold on 81 | 82 | methods={'MR','MSS','RBD','CA','BL','RRWR','DSS','MTMR','M3S-NIR','Our'}; 83 | precisions = {MR.FMeasureF, MSS.FMeasureF, RBD.FMeasureF, CA.FMeasureF, BL.FMeasureF, RRWR.FMeasureF, DSS.FMeasureF, MTMR.FMeasureF,Old.FMeasureF, Our.FMeasureF}; 84 | 85 | 86 | 87 | 88 | for i = 1:size(methods, 2) 89 | legendlabel{i} = [methods{i} ' [' sprintf('%.3f', precisions{i}) ']']; 90 | end 91 | legend(legendlabel, 'Location', 'SouthWest', 'FontSize', 13) 92 | 93 | -------------------------------------------------------------------------------- /Code/M3S-NIR/descendPropagation.m: -------------------------------------------------------------------------------- 1 | function propagatedData = descendPropagation(feat,initData,Nsample,featDim) 2 | %% Calculate the propagated reconstruction errors. 3 | %% Input: 4 | %Nsample:the number of supixel. 5 | % feat: superpixel feature matrix, the size of which is Nsample * 3. 6 | % initData: the initial saliency . 7 | % paramPropagate: propagation parameters. 8 | %paramPropagate.nclus = 8; [the number of clusters] 9 | %paramPropagate.maxIter=200; Max number of k-means iterations 10 | %% Output: 11 | % propagatedData: the propagated saliency. 12 | %%%%%%%%%%%%%%%% 13 | paramPropagate.nclus = 8; 14 | paramPropagate.maxIter=200; 15 | paramPropagate.lamna=0.5; 16 | %% calculate the cluster centers 17 | centers = form_codebook(feat', paramPropagate.nclus,paramPropagate.maxIter); 18 | %% label each superpixel according to the cluster centers. 19 | [ featLabel ] = labelCluster( centers, feat', Nsample, paramPropagate.nclus ); 20 | %% a row vector(1*3),each element is the average of the column of feature 21 | meanfeat = mean(feat,1); 22 | %% sum of the variance of each feature %% 23 | sig2 = zeros(1,featDim); %a row vector 24 | for k=1:featDim 25 | 26 | sig2(k) = norm(feat(:,k) - meanfeat(:,k))^2/Nsample; 27 | end 28 | %sigma2 = mean(sig2); 29 | sigma2 = featDim^2\(sum(sig2)); 30 | %% Calculate the Euclidean distance matrix (the diag element is zero) 31 | distMatrix = zeros(Nsample, Nsample); 32 | for i=1:Nsample 33 | for j=i+1:Nsample 34 | distMatrix(i,j) = exp(-norm(feat(i,:)-feat(j,:))^2/(2*sigma2)); 35 | distMatrix(j,i) = distMatrix(i,j); 36 | end 37 | end 38 | 39 | %% Propagation 40 | [desData desInd] = sort(initData); 41 | for i=Nsample:-1:1 42 | dataLabel = desInd(i); 43 | clusterlabel = featLabel(dataLabel); 44 | clusterbgsup = find(featLabel==clusterlabel); 45 | nInnerCluster = length(clusterbgsup); 46 | 47 | sumdist = 0; 48 | sumA = 0; 49 | for m=1:nInnerCluster 50 | M = clusterbgsup(m); 51 | sumdist = sumdist + distMatrix(dataLabel,M)*initData(M); 52 | sumA = sumA + distMatrix(dataLabel,M); 53 | end 54 | 55 | if sumA==0 56 | sumA = sumA+eps; 57 | end 58 | initData(dataLabel)=(1-paramPropagate.lamna)*initData(dataLabel) + paramPropagate.lamna/sumA*sumdist; 59 | end 60 | propagatedData = initData; 61 | propagatedData = (propagatedData - min(propagatedData(:)))/(max(propagatedData(:)) - min(propagatedData(:))); -------------------------------------------------------------------------------- /Code/M3S-NIR/form_codebook.m: -------------------------------------------------------------------------------- 1 | function centers = form_codebook(descriptors, codebook_size, Max_Iterations) 2 | %% function centers = form_codebook(descriptors, codebook_size) 3 | %% 4 | %% This function uses input features to form a codebook by k-means clustering 5 | %% algorithm. The code is modified from part of R. Fergus's bag of features code. 6 | %% The clustering is performed using VGG code from Oxford written by Mark 7 | %% Everingham. The source is provided, which must be compiled to a MEX file 8 | %% for your given platform. 9 | %% 10 | %% Function specification: 11 | %% Input 12 | %% descriptors : input features [3 * Numsup] 13 | %% codebook_size : size of codebook (the number of clusters) 14 | %% Max_Iterations : Max number of k-means iterations 15 | %% Output 16 | %% centers : formed codebook [dim x codebook size] 17 | 18 | Verbosity = 0; %% Verbsoity of Mark's code 19 | 20 | %% form options structure for clustering 21 | cluster_options.maxiters = Max_Iterations; 22 | cluster_options.verbose = Verbosity; 23 | 24 | %% call kmeans clustering routine by Mark Everingham 25 | [centers,sse] = vgg_kmeans(double(descriptors), codebook_size, cluster_options); -------------------------------------------------------------------------------- /Code/M3S-NIR/globalW.m: -------------------------------------------------------------------------------- 1 | function adjMergeW = globalW(spnum) 2 | %% global constructing graph W 3 | 4 | adjMergeW=ones(spnum,spnum); 5 | for i=1:spnum 6 | adjMergeW(i, i)=0; 7 | end 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /Code/M3S-NIR/labelCluster.m: -------------------------------------------------------------------------------- 1 | function [ featLabel ] = labelCluster( centers, allfeat, N_sample, nclus ) 2 | %% Label each superpixel according to the cluster centers. 3 | %Input: 4 | % centers: 5 | % allfeat: 6 | % nclus: 7 | distance = zeros(nclus,N_sample); 8 | for i=1:nclus 9 | for j=1:N_sample 10 | distance(i,j) = norm(allfeat(:,j)-centers(:,i)); 11 | end 12 | end 13 | 14 | [minval , featLabel] = min(distance); -------------------------------------------------------------------------------- /Code/M3S-NIR/lastCMR.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/lastCMR.m -------------------------------------------------------------------------------- /Code/M3S-NIR/main_new.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/main_new.m -------------------------------------------------------------------------------- /Code/M3S-NIR/makeweights.m: -------------------------------------------------------------------------------- 1 | function weights=makeweights(edges,vals,valScale) 2 | valDistances=sqrt(sum((vals(edges(:,1),:)-vals(edges(:,2),:)).^2,2)); 3 | valDistances=normalize(valDistances); %Normalize to [0,1] 4 | weights=exp(-valScale*valDistances); 5 | -------------------------------------------------------------------------------- /Code/M3S-NIR/normalize.m: -------------------------------------------------------------------------------- 1 | function normVals=normalize(newVals,oldVals) 2 | %function normVals=normalize(newVals,oldVals) normalizes the range of 3 | %newVals to the range of oldVals such that every column is normalized 4 | %independantly 5 | % 6 | %Inputs: newVals - NxP matrix of new values to be normalized to the 7 | % range of oldVals 8 | % oldVals - Optional NxP matrix of original values (for 9 | % arbitrary K). Defaults to normalizing range to [0,1] 10 | % 11 | %Outputs: normVals - NxP matrix of newVals normalized (columnwise) to 12 | % the range of oldVals 13 | % 14 | % 15 | %5/13/03 - Leo Grady 16 | 17 | % Copyright (C) 2002, 2003 Leo Grady 18 | % Computer Vision and Computational Neuroscience Lab 19 | % Department of Cognitive and Neural Systems 20 | % Boston University 21 | % Boston, MA 02215 22 | % 23 | % This program is free software; you can redistribute it and/or 24 | % modify it under the terms of the GNU General Public License 25 | % as published by the Free Software Foundation; either version 2 26 | % of the License, or (at your option) any later version. 27 | % 28 | % This program is distributed in the hope that it will be useful, 29 | % but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | % GNU General Public License for more details. 32 | % 33 | % You should have received a copy of the GNU General Public License 34 | % along with this program; if not, write to the Free Software 35 | % Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 36 | % 37 | % Date - $Id: normalize.m,v 1.2 2003/08/21 17:29:29 lgrady Exp $ 38 | %========================================================================% 39 | 40 | %Initialize 41 | [N P]=size(newVals); 42 | 43 | %Supply optional argument, if required 44 | if nargin == 1 45 | oldVals=[zeros(1,P);ones(1,P)]; 46 | end 47 | 48 | %Find original minima/maxima 49 | minVal=min(oldVals,[],1); 50 | maxVal=max(oldVals,[],1); 51 | 52 | %Find current minima/maxima 53 | minNewVal=min(newVals,[],1); 54 | maxNewVal=max(newVals,[],1); 55 | 56 | %Perform normalization 57 | warning off MATLAB:divideByZero %Error for divide by zero handled below 58 | normVals=newVals-ones(N,1)*minNewVal; 59 | normVals=normVals.*(ones(N,1)*(maxVal-minVal))./ ... 60 | (ones(N,1)*max(normVals,[],1)); 61 | normVals=normVals+ones(N,1)*minVal; 62 | warning on MATLAB:divideByZero 63 | 64 | %Error check for completely uniform inputs 65 | uniformIndex=find(minNewVal==maxNewVal); 66 | normVals(:,uniformIndex)=ones(N,1)*minNewVal(:,uniformIndex); 67 | -------------------------------------------------------------------------------- /Code/M3S-NIR/readme.txt: -------------------------------------------------------------------------------- 1 | The code is for paper "Saliency Detection via Graph-Based Manifold Ranking" 2 | by Chuan Yang, Lihe Zhang, Huchuan Lu, Ming-Hsuan Yang, and Xiang Ruan 3 | To appear in Proceedings of IEEE Conference on Computer Vision and Pattern Recognition (CVPR 2013), Portland, June, 2013. 4 | written by Chuan Yang 5 | Email: ycscience86@gmail.com 6 | ****************************************************************************************************************** 7 | The code is tested on Windows XP with MATLAB R2010b. 8 | ****************************************************************************************************************** 9 | Usage: 10 | >put the test images into file '\test' 11 | >run 'demo.m' 12 | ****************************************************************************************************************** 13 | Note: We observe that some images on the MSRA dataset are surrounded with artificial frames, 14 | which will invalidate the used boundary prior. Thus, we run a pre-processing to remove such obvious frames. 15 | 16 | Procedures: 17 | 1. compute a binary edge map of the image using the canny method. 18 | 2. if a rectangle is detected in a band of 30 pixels in width along the four sides of the edge map (i.e. we assume that the frame is not wider than 30 pixels), we will cut the aera outside the rectangle from the image. 19 | 20 | The file 'removeframe.m' is the pre-processing code. 21 | 22 | ****************************************************************************************************************** 23 | We use the SLIC superpixel software to generate superpixels (http://ivrg.epfl.ch/supplementary_material/RK_SLICSuperpixels/index.html) 24 | and some graph functions in the Graph Analysis Toolbox (http://eslab.bu.edu/software/graphanalysis/). 25 | 26 | Note: The running time reported in our paper does not include the time of the pre-processing and the running time of the superpixel generation is computed by using the SLIC Windows GUI based executable. -------------------------------------------------------------------------------- /Code/M3S-NIR/removeframe.m: -------------------------------------------------------------------------------- 1 | function [S_input,input_img1,input_img2,w]=removeframe(img1,img2,S_imname) 2 | threshold=0.6; 3 | S_input=imread(S_imname); 4 | input_img1=imread(img1); 5 | input_img2=imread(img2); 6 | S_input=im2double(S_input); 7 | input_img1=im2double(input_img1); 8 | input_img2=im2double(input_img2); 9 | gray=rgb2gray(S_input); 10 | edgemap = edge(gray,'canny'); 11 | [m,n]=size(edgemap); 12 | flagt=0; 13 | flagd=0; 14 | flagr=0; 15 | flagl=0; 16 | t=1; 17 | d=1; 18 | l=1; 19 | r=1; 20 | 21 | for k=1:30 % we assume that the frame is not wider than 30 pixels. 22 | pbt=mean(edgemap(k,:)); 23 | pbd=mean(edgemap(m-k+1,:)); 24 | pbl=mean(edgemap(:,k)); 25 | pbr=mean(edgemap(:,n-k+1)); 26 | if pbt>threshold 27 | t=k; 28 | flagt=1; 29 | end 30 | if pbd>threshold 31 | d=k; 32 | flagd=1; 33 | end 34 | if pbl>threshold 35 | l=k; 36 | flagl=1; 37 | end 38 | if pbr>threshold 39 | r=k; 40 | flagr=1; 41 | end 42 | end 43 | 44 | flagrm=flagt+flagd+flagl+flagr; 45 | % we assume that there exists a frame when one more lines parallel to the image side are detected 46 | if flagrm>1 47 | maxwidth=max([t,d,l,r]); 48 | % 49 | if t==1 50 | t=maxwidth; 51 | end 52 | if d==1 53 | d=maxwidth; 54 | end 55 | if l==1 56 | l=maxwidth; 57 | end 58 | if r==1 59 | r=maxwidth; 60 | end 61 | S_input=S_input(t:m-d+1,l:n-r+1,:); 62 | input_img1=input_img1(t:m-d+1,l:n-r+1,:); 63 | input_img2=input_img2(t:m-d+1,l:n-r+1,:); 64 | w=[m,n,t,m-d+1,l,n-r+1]; 65 | else 66 | w=[m,n,1,m,1,n]; 67 | end 68 | % outname=[S_imname(1:end-4) '.bmp']; 69 | % imwrite(S_input,outname); 70 | 71 | 72 | -------------------------------------------------------------------------------- /Code/M3S-NIR/softthreshold.m: -------------------------------------------------------------------------------- 1 | 2 | function z = softthreshold(e1,e2 ,e3,x,y,r,spnum) 3 | z=zeros(spnum,1); 4 | z1=(x*e1+y*e2)/(x+y); 5 | z2=r/(2*(x+y)); 6 | for i=1:spnum 7 | if (z1(i,1) - e3(i,1) < -z2) 8 | s(i) = z1(i,1) + z2; 9 | else if (z1(i,1) - e3(i,1) > z2) 10 | s(i)= z1(i,1) - z2; 11 | else 12 | s(i) = e3(i,1); 13 | end 14 | end 15 | z(i,1)=s(i); 16 | 17 | end -------------------------------------------------------------------------------- /Code/M3S-NIR/twoCMR.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/twoCMR.m -------------------------------------------------------------------------------- /Code/M3S-NIR/vgg_argparse.m: -------------------------------------------------------------------------------- 1 | function [opts,rem_opts] = vgg_argparse(opts,varargin) 2 | 3 | %VGG_ARGPARSE Parse variable arguments into a structure 4 | % opts = vgg_argparse(inopts,varargin) 5 | % inopts: structure (cells array) listing valid members and default values 6 | % varargin: variable arguments of form '',,... 7 | % opts: opts modified by varargin 8 | % 9 | % Example: 10 | % function f = foo(varargin) 11 | % opts = vgg_argparse(struct('maxiters',10,'verbose',0), varargin) 12 | % ... 13 | % 14 | % An unknown option (ie, present in varargin but absent in inopts) 15 | % causes an error. Calling the function as 16 | % [opts,rem_opts] = vgg_argparse(inopts,varargin) returns the unknown 17 | % option(s) in rem_opts for later use rather than causes an error. 18 | % 19 | % May also use OPTS = VGG_ARGPARSE(OPTS, ASTRUCT) where ASTRUCT is a struct 20 | % of options. 21 | 22 | % Author: Mark Everingham 23 | % modified by werner, Jan 03 24 | % Date: 16 Jan 02 25 | 26 | if iscell(opts) 27 | opts=struct(opts{:}); 28 | end 29 | 30 | if length(varargin) & iscell(varargin{1}) 31 | if isempty(varargin{1}) 32 | inopts = struct([]); 33 | else 34 | inopts=struct(varargin{1}{:}); 35 | end 36 | else 37 | if isempty(varargin) 38 | inopts = struct([]); 39 | elseif isstruct(varargin{1}) 40 | inopts = varargin{1}; 41 | else 42 | inopts=struct(varargin{:}); 43 | end 44 | end 45 | 46 | rem_opts = []; 47 | fn = fieldnames(inopts); 48 | for i=1:length(fn) 49 | if isfield(opts,fn{i}) 50 | %opts.(fn{i})=inopts.(fn{i}); 51 | opts = setfield(opts,fn{i},getfield(inopts,fn{i})); 52 | else 53 | if nargout < 2 54 | error(sprintf('bad argument: ''%s''', fn{i})); 55 | else 56 | %rem_opts.(fn{i}) = inopts.(fn{i}); 57 | rem_opts = setfield(rem_opts,fn{i},getfield(inopts,fn{i})); 58 | end 59 | end 60 | end -------------------------------------------------------------------------------- /Code/M3S-NIR/vgg_kmeans.m: -------------------------------------------------------------------------------- 1 | function [CX, sse] = vgg_kmeans(X, nclus, varargin) 2 | 3 | % VGG_KMEANS initialize K-means clustering 4 | % [CX, sse] = vgg_kmeans(X, nclus, optname, optval, ...) 5 | % 6 | % - X: input points (one per column) 7 | % - nclus: number of clusters 8 | % - opts (defaults): 9 | % maxiters (inf): maxmimum number of iterations 10 | % mindelta (eps): minimum change in SSE per iteration 11 | % verbose (1): 1=print progress 12 | % 13 | % - CX: cluster centers 14 | % - sse: SSE 15 | 16 | % Author: Mark Everingham 17 | % Date: 13 Jan 03 18 | 19 | opts = struct('maxiters', inf, 'mindelta', eps, 'verbose', 1); 20 | if nargin > 2 21 | opts=vgg_argparse(opts,varargin); 22 | end 23 | 24 | perm=randperm(size(X,2)); 25 | CX=X(:,perm(1:nclus)); 26 | 27 | sse0 = inf; 28 | iter = 0; 29 | while iter < opts.maxiters 30 | 31 | tic; 32 | [CX, sse] = vgg_kmiter(X, CX); 33 | t=toc; 34 | 35 | if opts.verbose 36 | fprintf('iter %d: sse = %g (%g secs)\n', iter, sse, t) 37 | end 38 | 39 | if sse0-sse < opts.mindelta 40 | break 41 | end 42 | 43 | sse0=sse; 44 | iter=iter+1; 45 | 46 | end 47 | 48 | -------------------------------------------------------------------------------- /Code/M3S-NIR/vgg_kmiter.cxx: -------------------------------------------------------------------------------- 1 | #include "mex.h" 2 | #include 3 | #include 4 | 5 | // [CX', sse] = vgg_kmiter(X, CX) 6 | // X is DxN matrix of N D-dim points, stored in columns 7 | // CX is DxM matrix of M cluster centres 8 | // out: 9 | // CX' is DxM matrix of new cluster centres 10 | // sse is sum of squared distances to (old?) cluster centres 11 | 12 | void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) 13 | { 14 | const double *px, *pcx; 15 | double *pcxp; 16 | 17 | int npts, dim, nclus; 18 | const double *X, *CX; 19 | double *CXp, *psse, *CN; 20 | double d, dmin; 21 | int i, j, k, c; 22 | 23 | if (nrhs != 2) 24 | mexErrMsgTxt("two input arguments expected."); 25 | if (nlhs != 2) 26 | mexErrMsgTxt("two output arguments expected."); 27 | 28 | if (!mxIsDouble(prhs[0]) || mxIsComplex(prhs[0]) || 29 | mxGetNumberOfDimensions(prhs[0]) != 2) 30 | mexErrMsgTxt("input 1 (X) must be a real double matrix"); 31 | 32 | dim = mxGetM(prhs[0]); 33 | npts = mxGetN(prhs[0]); 34 | 35 | if (!mxIsDouble(prhs[1]) || mxIsComplex(prhs[1]) || 36 | mxGetNumberOfDimensions(prhs[1]) != 2 || 37 | mxGetM(prhs[1]) != dim) 38 | mexErrMsgTxt("input 2 (CX) must be a real double matrix compatible with input 1 (X)"); 39 | 40 | nclus = mxGetN(prhs[1]); 41 | 42 | plhs[0] = mxCreateDoubleMatrix(dim, nclus, mxREAL); 43 | CXp = mxGetPr(plhs[0]); 44 | 45 | plhs[1] = mxCreateScalarDouble(0.0); 46 | psse = mxGetPr(plhs[1]); 47 | 48 | X = mxGetPr(prhs[0]); 49 | CX = mxGetPr(prhs[1]); 50 | 51 | CN = (double *) calloc(nclus, sizeof(double)); 52 | 53 | memset(CXp, 0, dim * nclus * sizeof(double)); 54 | 55 | for (i = 0, px = X; i < npts; i++, px += dim) 56 | { 57 | dmin = DBL_MAX; 58 | c = 0; 59 | for (j = 0, pcx = CX; j < nclus; j++, pcx += dim) 60 | { 61 | d = 0.0; 62 | for (k = 0; k < dim; k++) 63 | d += (px[k] - pcx[k]) * (px[k] - pcx[k]); 64 | if (d < dmin) 65 | { 66 | dmin = d; 67 | c = j; 68 | } 69 | } 70 | 71 | *psse += dmin; 72 | 73 | CN[c]++; 74 | pcxp = CXp + c * dim; 75 | for (k = 0; k < dim; k++) 76 | pcxp[k] += px[k]; 77 | } 78 | 79 | for (j = 0, pcxp = CXp; j < nclus; j++, pcxp += dim) 80 | { 81 | if (CN[j]) 82 | { 83 | for (k = 0; k < dim; k++) 84 | pcxp[k] /= CN[j]; 85 | } 86 | } 87 | 88 | free(CN); 89 | } 90 | -------------------------------------------------------------------------------- /Code/M3S-NIR/vgg_kmiter.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/vgg_kmiter.dll -------------------------------------------------------------------------------- /Code/M3S-NIR/vgg_kmiter.mexglx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/vgg_kmiter.mexglx -------------------------------------------------------------------------------- /Code/M3S-NIR/vgg_kmiter.mexw64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/vgg_kmiter.mexw64 -------------------------------------------------------------------------------- /Code/M3S-NIR/weightc.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/M3S-NIR/weightc.m -------------------------------------------------------------------------------- /Code/MIED/README.md: -------------------------------------------------------------------------------- 1 | # Multi-interactive-Encoder-decoder-Network-for-RGBT-Salient-Object-Detection 2 | The pytorch implementation of Multi-interactive Encoder-decoder Network for RGBT Salient Object Detection 3 | 4 | ## For test 5 | 6 | The saved model can be downloaded at 7 | 8 | https://pan.baidu.com/s/1Nf_tRfYd0hlKXHR1nmP8bA with verification code 'be8z' 9 | 10 | The saliency maps of VT821 can be downloaded at 11 | 12 | https://pan.baidu.com/s/1X94GlsNBrX1OdXm0B3k29A with verification code '7qk1' 13 | 14 | ## For train 15 | 16 | The pretrained model(vgg16) can be downloaded at 17 | 18 | https://pan.baidu.com/s/1GHX_-vMO9Po2s4TiDsRLGQ with verification code 'nxdy' 19 | 20 | The trainset(VT1000, ECSSD and corresponding synthetic thermal images) can be downloaded at 21 | 22 | https://pan.baidu.com/s/1ZyWrN7NXO3M-45pBT1rflA with verification code '5uge' 23 | -------------------------------------------------------------------------------- /Code/MIED/eval.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import glob2 4 | import os 5 | import scipy.io as sio # mat 6 | eps = 2.2204e-16 7 | 8 | 9 | def parameter(): 10 | p = {} 11 | p['gtThreshold'] = 0.5 12 | p['beta'] = np.sqrt(0.3) 13 | p['thNum'] = 100 14 | p['thList'] = np.linspace(0, 1, p['thNum']) 15 | 16 | return p 17 | 18 | 19 | def im2double(im): 20 | return cv2.normalize(im.astype('float'), 21 | None, 22 | 0.0, 1.0, 23 | cv2.NORM_MINMAX) 24 | 25 | 26 | def prCount(gtMask, curSMap, p): 27 | gtH, gtW = gtMask.shape[0:2] 28 | algH, algW = curSMap.shape[0:2] 29 | 30 | if gtH != algH or gtW != algW: 31 | curSMap = cv2.resize(curSMap, (gtW, gtH)) 32 | 33 | gtMask = (gtMask >= p['gtThreshold']).astype(np.float32) 34 | gtInd = np.where(gtMask > 0) 35 | gtCnt = np.sum(gtMask) 36 | 37 | if gtCnt == 0: 38 | prec = [] 39 | recall = [] 40 | else: 41 | hitCnt = np.zeros((p['thNum'], 1), np.float32) 42 | algCnt = np.zeros((p['thNum'], 1), np.float32) 43 | 44 | for k, curTh in enumerate(p['thList']): 45 | thSMap = (curSMap >= curTh).astype(np.float32) 46 | hitCnt[k] = np.sum(thSMap[gtInd]) 47 | algCnt[k] = np.sum(thSMap) 48 | 49 | prec = hitCnt / (algCnt+eps) 50 | recall = hitCnt / gtCnt 51 | 52 | return prec, recall 53 | 54 | 55 | def PR_Curve(resDir, gtDir): 56 | p = parameter() 57 | beta = p['beta'] 58 | gtImgs = glob2.iglob(gtDir + '/*.jpg') ######## 59 | 60 | prec = [] 61 | recall = [] 62 | 63 | for gtName in gtImgs: 64 | dir, name = os.path.split(gtName) 65 | mapName = os.path.join(resDir,name[:-4]+'.png') 66 | #print(mapName) 67 | curMap = im2double(cv2.imread(mapName, cv2.IMREAD_GRAYSCALE)) 68 | 69 | curGT = im2double(cv2.imread(gtName, cv2.IMREAD_GRAYSCALE)) 70 | 71 | if curMap.shape[0] != curGT.shape[0]: 72 | curMap = cv2.resize(curMap, (curGT.shape[1], curGT.shape[2])) 73 | 74 | curPrec, curRecall = prCount(curGT, curMap, p) 75 | 76 | prec.append(curPrec) 77 | recall.append(curRecall) 78 | 79 | 80 | prec = np.hstack(prec[:]) 81 | recall = np.hstack(recall[:]) 82 | 83 | prec = np.mean(prec, 1) 84 | recall = np.mean(recall, 1) 85 | 86 | # compute the max F-Score 87 | score = (1+beta**2)*prec*recall / (beta**2*prec + recall) 88 | curTh = np.argmax(score) 89 | curScore = np.max(score) 90 | res = {} 91 | res['prec'] = prec 92 | res['recall'] = recall 93 | res['curScore'] = curScore 94 | res['curTh'] = curTh 95 | res['fscore']=score 96 | 97 | 98 | return res 99 | 100 | 101 | def MAE_Value(resDir, gtDir): 102 | p = parameter() 103 | gtThreshold = p['gtThreshold'] 104 | 105 | gtImgs = glob2.iglob(gtDir + '/*.jpg') 106 | 107 | MAE = [] 108 | 109 | 110 | for gtName in gtImgs: 111 | dir, name = os.path.split(gtName) 112 | mapName= os.path.join(resDir,name[:-4]+'.png') ###### 113 | 114 | #print(mapName) 115 | if os.path.exists(mapName) is False: 116 | mapName = mapName.replace('.png', '.jpg') 117 | if os.path.exists(mapName) is False: 118 | mapName = mapName.replace('.jpg','.bmp') 119 | 120 | curMap = im2double(cv2.imread(mapName, cv2.IMREAD_GRAYSCALE)) 121 | 122 | curGT = im2double(cv2.imread(gtName, cv2.IMREAD_GRAYSCALE)) 123 | curGT = (curGT >= gtThreshold).astype(np.float32) 124 | 125 | if curMap.shape[0] != curGT.shape[0]: 126 | curMap = cv2.resize(curMap, (curGT.shape[1], curGT.shape[2])) 127 | 128 | diff = np.abs(curMap - curGT) 129 | 130 | MAE.append(np.mean(diff)) 131 | 132 | return np.mean(MAE) 133 | 134 | 135 | if __name__ == "__main__": 136 | 137 | method = 'out' 138 | resDir = 'path/to/result' 139 | gtDir = 'path/to/gt' 140 | mae = MAE_Value(resDir, gtDir) 141 | pr = PR_Curve(resDir, gtDir) 142 | 143 | FMeasureF = pr['curScore'] 144 | print('max F:', pr['curScore']) 145 | print('MAE:', mae) 146 | mat_path = './' + method + '.mat' 147 | print(mat_path) 148 | sio.savemat(mat_path, {'Pre': pr['prec'], 'Recall': pr['recall'],'Fscore':pr['fscore'],'maxF_index':pr['curTh'],'FMeasureF': FMeasureF,'MAE':mae}) 149 | -------------------------------------------------------------------------------- /Code/MIED/lib/data_prefetcher.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | class DataPrefetcher(object): 4 | def __init__(self, loader): 5 | self.loader = iter(loader) 6 | self.stream = torch.cuda.Stream() 7 | self.preload() 8 | 9 | def preload(self): 10 | try: 11 | self.next_rgb, self.next_t, self.next_gt,_,_ = next(self.loader) 12 | except StopIteration: 13 | self.next_rgb = None 14 | self.next_t = None 15 | self.next_gt = None 16 | return 17 | 18 | with torch.cuda.stream(self.stream): 19 | self.next_rgb = self.next_rgb.cuda(non_blocking=True).float() 20 | self.next_t = self.next_t.cuda(non_blocking=True).float() 21 | self.next_gt = self.next_gt.cuda(non_blocking=True).float() 22 | #self.next_rgb = self.next_rgb #if need 23 | #self.next_t = self.next_t #if need 24 | #self.next_gt = self.next_gt # if need 25 | 26 | def next(self): 27 | torch.cuda.current_stream().wait_stream(self.stream) 28 | rgb = self.next_rgb 29 | t= self.next_t 30 | gt = self.next_gt 31 | self.preload() 32 | return rgb, t, gt -------------------------------------------------------------------------------- /Code/MIED/lib/dataset.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | import os 4 | import cv2 5 | import numpy as np 6 | try: 7 | from . import transform 8 | except: 9 | import transform 10 | from torch.utils.data import Dataset 11 | 12 | #BGR 13 | mean_rgb = np.array([[[0.551*255, 0.619*255, 0.532*255]]]) 14 | mean_t =np.array([[[0.341*255, 0.360*255, 0.753*255]]]) 15 | std_rgb = np.array([[[0.241 * 255, 0.236 * 255, 0.244 * 255]]]) 16 | std_t = np.array([[[0.208 * 255, 0.269 * 255, 0.241 * 255]]]) 17 | 18 | 19 | class Data(Dataset): 20 | def __init__(self, root,mode='train'): 21 | self.samples = [] 22 | lines = os.listdir(os.path.join(root, 'GT')) 23 | for line in lines: 24 | rgbpath = os.path.join(root, 'RGB', line[:-4]+'.jpg') 25 | tpath = os.path.join(root, 'T', line[:-4]+'.jpg') 26 | maskpath = os.path.join(root, 'GT', line) 27 | self.samples.append([rgbpath,tpath,maskpath]) 28 | 29 | if mode == 'train': 30 | self.transform = transform.Compose( transform.Normalize(mean1=mean_rgb,mean2=mean_t,std1=std_rgb,std2=std_t), 31 | transform.Resize(400,400), 32 | transform.RandomHorizontalFlip(),transform.ToTensor()) 33 | 34 | elif mode == 'test': 35 | self.transform = transform.Compose( transform.Normalize(mean1=mean_rgb,mean2=mean_t,std1=std_rgb,std2=std_t), 36 | transform.Resize(400,400), 37 | transform.ToTensor()) 38 | else: 39 | raise ValueError 40 | 41 | def __getitem__(self, idx): 42 | rgbpath,tpath,maskpath = self.samples[idx] 43 | rgb = cv2.imread(rgbpath).astype(np.float32) 44 | t = cv2.imread(tpath).astype(np.float32) 45 | mask = cv2.imread(maskpath).astype(np.float32) 46 | H, W, C = mask.shape 47 | rgb,t,mask = self.transform(rgb,t,mask) 48 | return rgb,t,mask, (H, W), maskpath.split('/')[-1] 49 | 50 | def __len__(self): 51 | return len(self.samples) -------------------------------------------------------------------------------- /Code/MIED/lib/transform.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | #coding=utf-8 3 | 4 | import cv2 5 | import torch 6 | import numpy as np 7 | 8 | class Compose(object): 9 | def __init__(self, *ops): 10 | self.ops = ops 11 | 12 | def __call__(self, rgb,t, mask): 13 | for op in self.ops: 14 | rgb,t, mask = op(rgb,t, mask) 15 | return rgb,t, mask 16 | 17 | 18 | 19 | class Normalize(object): 20 | def __init__(self, mean1,mean2, std1,std2): 21 | self.mean1 = mean1 22 | self.mean2 = mean2 23 | self.std1 = std1 24 | self.std2 = std2 25 | 26 | def __call__(self, rgb,t, mask): 27 | rgb = (rgb - self.mean1)/self.std1 28 | t = (t - self.mean2) / self.std2 29 | mask /= 255 30 | return rgb,t, mask 31 | 32 | class Minusmean(object): 33 | def __init__(self, mean1,mean2): 34 | self.mean1 = mean1 35 | self.mean2 = mean2 36 | 37 | def __call__(self, rgb,t, mask): 38 | rgb = rgb - self.mean1 39 | t = t - self.mean2 40 | mask /= 255 41 | return rgb,t, mask 42 | 43 | 44 | class Resize(object): 45 | def __init__(self, H, W): 46 | self.H = H 47 | self.W = W 48 | 49 | def __call__(self, rgb,t, mask): 50 | rgb = cv2.resize(rgb, dsize=(self.W, self.H), interpolation=cv2.INTER_LINEAR) 51 | t = cv2.resize(t, dsize=(self.W, self.H), interpolation=cv2.INTER_LINEAR) 52 | mask = cv2.resize( mask, dsize=(self.W, self.H), interpolation=cv2.INTER_LINEAR) 53 | return rgb,t, mask 54 | 55 | class RandomCrop(object): 56 | def __init__(self, H, W): 57 | self.H = H 58 | self.W = W 59 | 60 | def __call__(self, rgb,t, mask): 61 | H,W,_ = rgb.shape 62 | xmin = np.random.randint(W-self.W+1) 63 | ymin = np.random.randint(H-self.H+1) 64 | rgb = rgb[ymin:ymin+self.H, xmin:xmin+self.W, :] 65 | t = t[ymin:ymin + self.H, xmin:xmin + self.W, :] 66 | mask = mask[ymin:ymin+self.H, xmin:xmin+self.W, :] 67 | return rgb,t, mask 68 | 69 | class RandomHorizontalFlip(object): 70 | def __call__(self, rgb,t, mask): 71 | if np.random.randint(2)==1: 72 | rgb = rgb[:,::-1,:].copy() 73 | t = t[:, ::-1, :].copy() 74 | mask = mask[:,::-1,:].copy() 75 | return rgb,t, mask 76 | 77 | class ToTensor(object): 78 | def __call__(self, rgb,t, mask): 79 | rgb = torch.from_numpy(rgb) 80 | rgb = rgb.permute(2, 0, 1) 81 | t = torch.from_numpy(t) 82 | t = t.permute(2, 0, 1) 83 | mask = torch.from_numpy(mask) 84 | mask = mask.permute(2, 0, 1) 85 | return rgb,t,mask.mean(dim=0, keepdim=True) -------------------------------------------------------------------------------- /Code/MIED/net.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | import vgg 5 | 6 | def convblock(in_,out_,ks,st,pad): 7 | return nn.Sequential( 8 | nn.Conv2d(in_,out_,ks,st,pad), 9 | nn.BatchNorm2d(out_), 10 | nn.ReLU(inplace=True) 11 | ) 12 | 13 | class GFB(nn.Module): 14 | def __init__(self,in_1,in_2): 15 | super(GFB, self).__init__() 16 | self.ca1 = CA(2*in_1) 17 | self.conv1 = convblock(2*in_1,128, 3, 1, 1) 18 | self.conv_globalinfo = convblock(512,128,3, 1, 1) 19 | self.ca2 = CA(in_2) 20 | self.conv_curfeat =convblock(in_2,128,3,1,1) 21 | self.conv_out= convblock(128,in_2,3,1,1) 22 | 23 | def forward(self, pre1,pre2,cur,global_info): 24 | cur_size = cur.size()[2:] 25 | pre = self.ca1(torch.cat((pre1,pre2),1)) 26 | pre =self.conv1(F.interpolate(pre,cur_size,mode='bilinear',align_corners=True)) 27 | 28 | global_info = self.conv_globalinfo(F.interpolate(global_info,cur_size,mode='bilinear',align_corners=True)) 29 | cur_feat =self.conv_curfeat(self.ca2(cur)) 30 | fus = pre + cur_feat + global_info 31 | return self.conv_out(fus) 32 | 33 | 34 | class GlobalInfo(nn.Module): 35 | def __init__(self): 36 | super(GlobalInfo, self).__init__() 37 | self.ca = CA(1024) 38 | self.de_chan = convblock(1024,256,3,1,1) 39 | 40 | self.b0 = nn.Sequential( 41 | nn.AdaptiveMaxPool2d(13), 42 | nn.Conv2d(256,128,1,1,0,bias=False), 43 | nn.ReLU(inplace=True) 44 | ) 45 | 46 | self.b1 = nn.Sequential( 47 | nn.AdaptiveMaxPool2d(9), 48 | nn.Conv2d(256,128,1,1,0,bias=False), 49 | nn.ReLU(inplace=True) 50 | ) 51 | self.b2 = nn.Sequential( 52 | nn.AdaptiveMaxPool2d(5), 53 | nn.Conv2d(256, 128, 1, 1, 0,bias=False), 54 | nn.ReLU(inplace=True) 55 | ) 56 | self.b3 = nn.Sequential( 57 | nn.AdaptiveMaxPool2d(1), 58 | nn.Conv2d(256, 128, 1, 1, 0,bias=False), 59 | nn.ReLU(inplace=True) 60 | ) 61 | self.fus = convblock(768,512,1,1,0) 62 | 63 | def forward(self, rgb,t): 64 | x_size=rgb.size()[2:] 65 | x=self.ca(torch.cat((rgb,t),1)) 66 | x=self.de_chan(x) 67 | b0 = F.interpolate(self.b0(x),x_size,mode='bilinear',align_corners=True) 68 | b1 = F.interpolate(self.b1(x),x_size,mode='bilinear',align_corners=True) 69 | b2 = F.interpolate(self.b2(x),x_size,mode='bilinear',align_corners=True) 70 | b3 = F.interpolate(self.b3(x),x_size,mode='bilinear',align_corners=True) 71 | out = self.fus(torch.cat((b0,b1,b2,b3,x),1)) 72 | return out 73 | 74 | class CA(nn.Module): 75 | def __init__(self,in_ch): 76 | super(CA, self).__init__() 77 | self.avg_weight = nn.AdaptiveAvgPool2d(1) 78 | self.max_weight = nn.AdaptiveMaxPool2d(1) 79 | self.fus = nn.Sequential( 80 | nn.Conv2d(in_ch, in_ch // 2, 1, 1, 0), 81 | nn.ReLU(), 82 | nn.Conv2d(in_ch // 2, in_ch, 1, 1, 0), 83 | ) 84 | self.c_mask = nn.Sigmoid() 85 | def forward(self, x): 86 | avg_map_c = self.avg_weight(x) 87 | max_map_c = self.max_weight(x) 88 | c_mask = self.c_mask(torch.add(self.fus(avg_map_c), self.fus(max_map_c))) 89 | return torch.mul(x, c_mask) 90 | 91 | class FinalScore(nn.Module): 92 | def __init__(self): 93 | super(FinalScore, self).__init__() 94 | self.ca =CA(256) 95 | self.score = nn.Conv2d(256, 1, 1, 1, 0) 96 | def forward(self,f1,f2,xsize): 97 | f1 = torch.cat((f1,f2),1) 98 | f1 = self.ca(f1) 99 | score = F.interpolate(self.score(f1), xsize, mode='bilinear', align_corners=True) 100 | return score 101 | 102 | class Decoder(nn.Module): 103 | def __init__(self): 104 | super(Decoder, self).__init__() 105 | self.global_info =GlobalInfo() 106 | self.score_global = nn.Conv2d(512, 1, 1, 1, 0) 107 | 108 | self.gfb4_1 = GFB(512,512) 109 | self.gfb3_1= GFB(512,256) 110 | self.gfb2_1= GFB(256,128) 111 | 112 | self.gfb4_2 = GFB(512, 512) #1/8 113 | self.gfb3_2 = GFB(512, 256)#1/4 114 | self.gfb2_2 = GFB(256, 128)#1/2 115 | 116 | self.score_1=nn.Conv2d(128, 1, 1, 1, 0) 117 | self.score_2 = nn.Conv2d(128, 1, 1, 1, 0) 118 | 119 | self.refine =FinalScore() 120 | 121 | 122 | def forward(self,rgb,t): 123 | xsize=rgb[0].size()[2:] 124 | global_info =self.global_info(rgb[4],t[4]) # 512 1/16 125 | d1=self.gfb4_1(global_info,t[4],rgb[3],global_info) 126 | d2=self.gfb4_2(global_info, rgb[4], t[3], global_info) 127 | #print(d1.shape,d2.shape) 128 | d3= self.gfb3_1(d1, d2,rgb[2],global_info) 129 | d4 = self.gfb3_2(d2, d1, t[2], global_info) 130 | d5 = self.gfb2_1(d3, d4, rgb[1], global_info) 131 | d6 = self.gfb2_2(d4, d3, t[1], global_info) #1/2 128 132 | 133 | score_global = self.score_global(global_info) 134 | 135 | score1=self.score_1(F.interpolate(d5,xsize,mode='bilinear',align_corners=True)) 136 | score2 = self.score_2(F.interpolate(d6, xsize, mode='bilinear', align_corners=True)) 137 | score =self.refine(d5,d6,xsize) 138 | return score,score1,score2,score_global 139 | 140 | class Mnet(nn.Module): 141 | def __init__(self): 142 | super(Mnet,self).__init__() 143 | self.rgb_net= vgg.a_vgg16() 144 | self.t_net= vgg.a_vgg16() #两个分支 145 | self.decoder=Decoder() 146 | 147 | for m in self.modules(): 148 | if isinstance(m, nn.Conv2d): 149 | m.weight.data.normal_(0, 0.01) 150 | elif isinstance(m, nn.BatchNorm2d): 151 | m.weight.data.fill_(1) 152 | m.bias.data.zero_() 153 | 154 | def forward(self,rgb,t): 155 | rgb_f= self.rgb_net(rgb) 156 | t_f= self.t_net(t) 157 | score,score1,score2,score_g =self.decoder(rgb_f,t_f) 158 | return score,score1,score2,score_g 159 | 160 | def load_pretrained_model(self): 161 | st=torch.load("vgg16.pth") 162 | st2={} 163 | for key in st.keys(): 164 | st2['base.'+key]=st[key] 165 | self.rgb_net.load_state_dict(st2) 166 | self.t_net.load_state_dict(st2) 167 | print('loading pretrained model success!') 168 | 169 | 170 | if __name__=="__main__": 171 | a=torch.rand(1,3,400,400) 172 | b=a 173 | net=Mnet() 174 | c,c1,c2=net(a,b) 175 | print(c.shape) 176 | print(c1.shape) 177 | print(c2.shape) -------------------------------------------------------------------------------- /Code/MIED/test.py: -------------------------------------------------------------------------------- 1 | coding='utf-8' 2 | import os 3 | from torch.utils.data import DataLoader 4 | from lib.dataset import Data 5 | import torch.nn.functional as F 6 | import torch 7 | import cv2 8 | import time 9 | from net import Mnet 10 | import numpy as np 11 | os.environ["CUDA_VISIBLE_DEVICES"] = "4" 12 | if __name__ == '__main__': 13 | model_path='./model/normal.pth' 14 | out_path = './output' 15 | data = Data(root='path/to/test_images',mode='test') 16 | loader = DataLoader(data, batch_size=1,shuffle=False) 17 | net = Mnet().cuda() 18 | print('loading model from %s...' % model_path) 19 | net.load_state_dict(torch.load(model_path)) 20 | if not os.path.exists(out_path): os.mkdir(out_path) 21 | time_s = time.time() 22 | img_num = len(loader) 23 | net.eval() 24 | with torch.no_grad(): 25 | for rgb, t, _ , (H, W), name in loader: 26 | print(name[0]) 27 | score, score1, score2,score_g = net(rgb.cuda().float(), t.cuda().float()) 28 | score = F.interpolate(score, size=(H, W), mode='bilinear') 29 | pred = np.squeeze(torch.sigmoid(score).cpu().data.numpy()) 30 | multi_fuse = 255 * pred 31 | cv2.imwrite(os.path.join(out_path, name[0][:-4] + '.png'), multi_fuse) 32 | time_e = time.time() 33 | print('speed: %f FPS' % (img_num / (time_e - time_s))) 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /Code/MIED/train.py: -------------------------------------------------------------------------------- 1 | coding='utf-8' 2 | import os 3 | from net import Mnet 4 | import torch 5 | import random 6 | import numpy as np 7 | import torch.optim as optim 8 | from torch.utils.data import DataLoader 9 | from lib.dataset import Data 10 | from lib.data_prefetcher import DataPrefetcher 11 | from torch.nn import functional as F 12 | import cv2 13 | import scipy.io as sio 14 | from eval import PR_Curve,MAE_Value 15 | os.environ["CUDA_VISIBLE_DEVICES"] = "2" 16 | 17 | def my_loss1(score,score1,score2,score_g,label): 18 | sal_loss2 = F.binary_cross_entropy_with_logits(score1, label, reduction='mean') 19 | sal_loss3 = F.binary_cross_entropy_with_logits(score2, label, reduction='mean') 20 | sal_loss1 = F.binary_cross_entropy_with_logits(score, label, reduction='mean') 21 | label_g = F.interpolate(label, [25, 25], mode='bilinear', align_corners=True) 22 | sal_loss_g = F.binary_cross_entropy_with_logits(score_g, label_g, reduction='mean') 23 | return sal_loss1 + sal_loss2 + sal_loss3 + sal_loss_g 24 | 25 | 26 | if __name__ == '__main__': 27 | random.seed(118) 28 | np.random.seed(118) 29 | torch.manual_seed(118) 30 | torch.cuda.manual_seed(118) 31 | torch.cuda.manual_seed_all(118) 32 | 33 | # dataset 34 | img_root = 'path/to/imgs' 35 | save_path = './model' 36 | if not os.path.exists(save_path): os.mkdir(save_path) 37 | lr = 0.001 #2 38 | batch_size = 4 39 | epoch = 20 40 | lr_dec=[11,16] 41 | data = Data(img_root) 42 | loader = DataLoader(data, batch_size=batch_size, shuffle=True, num_workers=1) 43 | 44 | net = Mnet().cuda() 45 | net.load_pretrained_model() 46 | optimizer = optim.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr=lr, weight_decay=0.0005,momentum=0.9) 47 | iter_num = len(loader) 48 | net.train() 49 | for epochi in range(1, epoch + 1): 50 | 51 | if epochi in lr_dec : 52 | lr=lr/10 53 | optimizer = optim.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr=lr, weight_decay=0.0005,momentum=0.9) 54 | print(lr) 55 | 56 | prefetcher = DataPrefetcher(loader) 57 | rgb, t, label = prefetcher.next() 58 | r_sal_loss = 0 59 | net.zero_grad() 60 | i = 0 61 | while rgb is not None: 62 | i+=1 63 | score, score1, score2,g= net(rgb, t) 64 | sal_loss= my_loss1( score,score1,score2,g,label) 65 | #sal_loss = F.binary_cross_entropy_with_logits(score, label, reduction='mean') 66 | r_sal_loss += sal_loss.data 67 | sal_loss.backward() 68 | optimizer.step() 69 | optimizer.zero_grad() 70 | if i % 200 == 0: 71 | print('epoch: [%2d/%2d], iter: [%5d/%5d] || loss : %5.4f' % ( 72 | epochi, epoch, i, iter_num, r_sal_loss / 200)) 73 | #print(weit) 74 | #print(weit.max()) 75 | r_sal_loss = 0 76 | rgb, t, label = prefetcher.next() 77 | torch.save(net.state_dict(), '%s/normal.pth' % (save_path)) -------------------------------------------------------------------------------- /Code/MIED/vgg.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | # vgg16 3 | def vgg(cfg, i, batch_norm=False): 4 | layers = [] 5 | in_channels = i 6 | stage = 1 7 | for v in cfg: 8 | if v == 'M': 9 | stage += 1 10 | if stage == 6: 11 | layers += [nn.MaxPool2d(kernel_size=3, stride=1, padding=1)] 12 | else: 13 | layers += [nn.MaxPool2d(kernel_size=3, stride=2, padding=1)] 14 | else: 15 | if stage == 6: 16 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) 17 | else: 18 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) 19 | if batch_norm: 20 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 21 | else: 22 | layers += [conv2d, nn.ReLU(inplace=True)] 23 | in_channels = v 24 | return layers 25 | 26 | class a_vgg16(nn.Module): 27 | def __init__(self): 28 | super(a_vgg16, self).__init__() 29 | self.cfg =[64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'] 30 | self.extract = [3, 8, 15, 22, 29] # [ 8, 15, 22, 29] 31 | # 64:1 -->128:1/2 -->256:1/4 -->512 :1/8 --> 512:1/16 -->M-> 512,1/16 32 | self.base = nn.ModuleList(vgg(self.cfg, 3)) 33 | for m in self.modules(): 34 | if isinstance(m, nn.Conv2d): 35 | m.weight.data.normal_(0, 0.01) 36 | elif isinstance(m, nn.BatchNorm2d): 37 | m.weight.data.fill_(1) 38 | m.bias.data.zero_() 39 | 40 | def forward(self, x): 41 | tmp_x = [] 42 | for k in range(len(self.base)): 43 | x = self.base[k](x) 44 | if k in self.extract: 45 | tmp_x.append(x) #collect feature maps 1(64) 1/2(128) 1/4(256) 1/8(512) 1/16(512) 46 | return tmp_x 47 | 48 | -------------------------------------------------------------------------------- /Code/MTMR/AdjcProcloop.m: -------------------------------------------------------------------------------- 1 | function adjcMerge = AdjcProcloop(M,N) 2 | % $Description: 3 | % -compute the adjacent matrix 4 | % $Agruments 5 | % Input; 6 | % -M: superpixel label matrix 7 | % -N: superpixel number 8 | % Output: 9 | % -adjcMerge: adjacent matrix 10 | 11 | adjcMerge = zeros(N,N); 12 | [m n] = size(M); 13 | 14 | for i = 1:m-1 15 | for j = 1:n-1 16 | if(M(i,j)~=M(i,j+1)) 17 | adjcMerge(M(i,j),M(i,j+1)) = 1; 18 | adjcMerge(M(i,j+1),M(i,j)) = 1; 19 | end; 20 | if(M(i,j)~=M(i+1,j)) 21 | adjcMerge(M(i,j),M(i+1,j)) = 1; 22 | adjcMerge(M(i+1,j),M(i,j)) = 1; 23 | end; 24 | if(M(i,j)~=M(i+1,j+1)) 25 | adjcMerge(M(i,j),M(i+1,j+1)) = 1; 26 | adjcMerge(M(i+1,j+1),M(i,j)) = 1; 27 | end; 28 | if(M(i+1,j)~=M(i,j+1)) 29 | adjcMerge(M(i+1,j),M(i,j+1)) = 1; 30 | adjcMerge(M(i,j+1),M(i+1,j)) = 1; 31 | end; 32 | end; 33 | end; 34 | bd=unique([M(1,:),M(m,:),M(:,1)',M(:,n)']); 35 | for i=1:length(bd) 36 | for j=i+1:length(bd) 37 | adjcMerge(bd(i),bd(j))=1; 38 | adjcMerge(bd(j),bd(i))=1; 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /Code/MTMR/CMR.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/MTMR/CMR.m -------------------------------------------------------------------------------- /Code/MTMR/DEMO11.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/MTMR/DEMO11.m -------------------------------------------------------------------------------- /Code/MTMR/OursPRF.m: -------------------------------------------------------------------------------- 1 | load('E:\experiment\cvpr13\multiM1\saliencymap\927\temp\test\mat\0.1.mat') 2 | plot(Recall ,Pre, 'm-' ,'LineWidth', 2 ) 3 | axis([0 1 0 1]) 4 | xlabel('Recall') 5 | ylabel('Precision') 6 | hold on 7 | 8 | load('E:\experiment\cvpr13\multiM1\saliencymap\927\temp\test\mat\0.5.mat') 9 | plot(Recall ,Pre, 'b-' ,'LineWidth', 2 ) 10 | axis([0 1 0 1]) 11 | xlabel('Recall') 12 | ylabel('Precision') 13 | hold on 14 | 15 | load('E:\experiment\cvpr13\multiM1\saliencymap\927\temp\test\mat\0.6.mat') 16 | plot(Recall ,Pre, 'r-' ,'LineWidth', 2 ) 17 | axis([0 1 0 1]) 18 | xlabel('Recall') 19 | ylabel('Precision') 20 | hold on 21 | 22 | load('E:\experiment\cvpr13\multiM1\saliencymap\927\temp\test\mat\1.mat') 23 | plot(Recall ,Pre, 'y-' ,'LineWidth', 2 ) 24 | axis([0 1 0 1]) 25 | xlabel('Recall') 26 | ylabel('Precision') 27 | hold on 28 | 29 | load('E:\experiment\cvpr13\multiM1\saliencymap\927\temp\test\mat\0.9.mat') 30 | plot(Recall ,Pre, 'g-' ,'LineWidth', 2 ) 31 | axis([0 1 0 1]) 32 | xlabel('Recall') 33 | ylabel('Precision') 34 | hold on 35 | 36 | % hleg=legend('Ours(G)','Ours(T)','Ours(GT)','Location','SouthWest'); 37 | % set(hleg,'FontSize',8) 38 | 39 | -------------------------------------------------------------------------------- /Code/MTMR/Plot_PreRecallThousand_contrast.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/MTMR/Plot_PreRecallThousand_contrast.m -------------------------------------------------------------------------------- /Code/MTMR/ReadDAT.m: -------------------------------------------------------------------------------- 1 | function B = ReadDAT(image_size,data_path) 2 | % $Description: 3 | % -read the superpixel labels from .dat file 4 | % $Agruments 5 | % Input; 6 | % -image_size: [width height] 7 | % -data_path: the path of the .dat file 8 | % Output: 9 | % -label matrix width*height 10 | 11 | row = image_size(1); 12 | colomn = image_size(2); 13 | fid = fopen(data_path,'r'); 14 | A = fread(fid, row * colomn, 'uint32')'; 15 | A = A + 1; 16 | B = reshape(A,[colomn, row]); 17 | B = B'; 18 | fclose(fid); -------------------------------------------------------------------------------- /Code/MTMR/SLICSuperpixelSegmentation.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/MTMR/SLICSuperpixelSegmentation.exe -------------------------------------------------------------------------------- /Code/MTMR/adjacency.m: -------------------------------------------------------------------------------- 1 | function W=adjacency(edges,weights,N) 2 | %Function W=adjacency(edges,weights,N) computes the weighted 3 | % adjacency matrix of a point and edge set. For an unweighted 4 | % matrix, set weights equal to ones(1,M) where M is the number 5 | % of edges in the graph. 6 | % 7 | %Inputs: edges - A Mx2 list of M edges indexing into points 8 | % weights - The weights used to determine matrix values. 9 | % If not specified, uses vector of all ones 10 | % N - Optional number of nodes in the graph. Used if 11 | % N > max(edges) (i.e., isolated nodes are present) 12 | % 13 | %Outputs: W - Adjacency matrix 14 | % 15 | % 16 | %Note: Adjacency matrix is not of the Seidel type 17 | % (i.e. all values are positive) 18 | %5/19/03 - Leo Grady 19 | 20 | % Copyright (C) 2002, 2003 Leo Grady 21 | % Computer Vision and Computational Neuroscience Lab 22 | % Department of Cognitive and Neural Systems 23 | % Boston University 24 | % Boston, MA 02215 25 | % 26 | % This program is free software; you can redistribute it and/or 27 | % modify it under the terms of the GNU General Public License 28 | % as published by the Free Software Foundation; either version 2 29 | % of the License, or (at your option) any later version. 30 | % 31 | % This program is distributed in the hope that it will be useful, 32 | % but WITHOUT ANY WARRANTY; without even the implied warranty of 33 | % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 34 | % GNU General Public License for more details. 35 | % 36 | % You should have received a copy of the GNU General Public License 37 | % along with this program; if not, write to the Free Software 38 | % Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 39 | % 40 | % Date - $Id: adjacency.m,v 1.2 2003/08/21 17:29:29 lgrady Exp $ 41 | %========================================================================% 42 | 43 | %If weights are not specified, use unity weighting 44 | if nargin == 1 45 | weights=ones(size(edges,1),1); 46 | end 47 | 48 | %If N is not specified, use maximum values of edges 49 | if nargin < 3 50 | N=max(max(edges)); 51 | end 52 | 53 | %Build sparse adjacency matrix 54 | W=sparse([edges(:,1);edges(:,2)],[edges(:,2);edges(:,1)], ... 55 | [weights;weights],N,N); 56 | -------------------------------------------------------------------------------- /Code/MTMR/colorspace.m: -------------------------------------------------------------------------------- 1 | function varargout = colorspace(Conversion,varargin) 2 | %COLORSPACE Convert a color image between color representations. 3 | % B = COLORSPACE(S,A) converts the color representation of image A 4 | % where S is a string specifying the conversion. S tells the 5 | % source and destination color spaces, S = 'dest<-src', or 6 | % alternatively, S = 'src->dest'. Supported color spaces are 7 | % 8 | % 'RGB' R'G'B' Red Green Blue (ITU-R BT.709 gamma-corrected) 9 | % 'YPbPr' Luma (ITU-R BT.601) + Chroma 10 | % 'YCbCr'/'YCC' Luma + Chroma ("digitized" version of Y'PbPr) 11 | % 'YUV' NTSC PAL Y'UV Luma + Chroma 12 | % 'YIQ' NTSC Y'IQ Luma + Chroma 13 | % 'YDbDr' SECAM Y'DbDr Luma + Chroma 14 | % 'JPEGYCbCr' JPEG-Y'CbCr Luma + Chroma 15 | % 'HSV'/'HSB' Hue Saturation Value/Brightness 16 | % 'HSL'/'HLS'/'HSI' Hue Saturation Luminance/Intensity 17 | % 'XYZ' CIE XYZ 18 | % 'Lab' CIE L*a*b* (CIELAB) 19 | % 'Luv' CIE L*u*v* (CIELUV) 20 | % 'Lch' CIE L*ch (CIELCH) 21 | % 22 | % All conversions assume 2 degree observer and D65 illuminant. Color 23 | % space names are case insensitive. When R'G'B' is the source or 24 | % destination, it can be omitted. For example 'yuv<-' is short for 25 | % 'yuv<-rgb'. 26 | % 27 | % MATLAB uses two standard data formats for R'G'B': double data with 28 | % intensities in the range 0 to 1, and uint8 data with integer-valued 29 | % intensities from 0 to 255. As MATLAB's native datatype, double data is 30 | % the natural choice, and the R'G'B' format used by colorspace. However, 31 | % for memory and computational performance, some functions also operate 32 | % with uint8 R'G'B'. Given uint8 R'G'B' color data, colorspace will 33 | % first cast it to double R'G'B' before processing. 34 | % 35 | % If A is an Mx3 array, like a colormap, B will also have size Mx3. 36 | % 37 | % [B1,B2,B3] = COLORSPACE(S,A) specifies separate output channels. 38 | % COLORSPACE(S,A1,A2,A3) specifies separate input channels. 39 | 40 | % Pascal Getreuer 2005-2006 41 | 42 | %%% Input parsing %%% 43 | if nargin < 2, error('Not enough input arguments.'); end 44 | [SrcSpace,DestSpace] = parse(Conversion); 45 | 46 | if nargin == 2 47 | Image = varargin{1}; 48 | elseif nargin >= 3 49 | Image = cat(3,varargin{:}); 50 | else 51 | error('Invalid number of input arguments.'); 52 | end 53 | 54 | FlipDims = (size(Image,3) == 1); 55 | 56 | if FlipDims, Image = permute(Image,[1,3,2]); end 57 | if ~isa(Image,'double'), Image = double(Image)/255; end 58 | if size(Image,3) ~= 3, error('Invalid input size.'); end 59 | 60 | SrcT = gettransform(SrcSpace); 61 | DestT = gettransform(DestSpace); 62 | 63 | if ~ischar(SrcT) & ~ischar(DestT) 64 | % Both source and destination transforms are affine, so they 65 | % can be composed into one affine operation 66 | T = [DestT(:,1:3)*SrcT(:,1:3),DestT(:,1:3)*SrcT(:,4)+DestT(:,4)]; 67 | Temp = zeros(size(Image)); 68 | Temp(:,:,1) = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3) + T(10); 69 | Temp(:,:,2) = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3) + T(11); 70 | Temp(:,:,3) = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3) + T(12); 71 | Image = Temp; 72 | elseif ~ischar(DestT) 73 | Image = rgb(Image,SrcSpace); 74 | Temp = zeros(size(Image)); 75 | Temp(:,:,1) = DestT(1)*Image(:,:,1) + DestT(4)*Image(:,:,2) + DestT(7)*Image(:,:,3) + DestT(10); 76 | Temp(:,:,2) = DestT(2)*Image(:,:,1) + DestT(5)*Image(:,:,2) + DestT(8)*Image(:,:,3) + DestT(11); 77 | Temp(:,:,3) = DestT(3)*Image(:,:,1) + DestT(6)*Image(:,:,2) + DestT(9)*Image(:,:,3) + DestT(12); 78 | Image = Temp; 79 | else 80 | Image = feval(DestT,Image,SrcSpace); 81 | end 82 | 83 | %%% Output format %%% 84 | if nargout > 1 85 | varargout = {Image(:,:,1),Image(:,:,2),Image(:,:,3)}; 86 | else 87 | if FlipDims, Image = permute(Image,[1,3,2]); end 88 | varargout = {Image}; 89 | end 90 | 91 | return; 92 | 93 | 94 | function [SrcSpace,DestSpace] = parse(Str) 95 | % Parse conversion argument 96 | 97 | if isstr(Str) 98 | Str = lower(strrep(strrep(Str,'-',''),' ','')); 99 | k = find(Str == '>'); 100 | 101 | if length(k) == 1 % Interpret the form 'src->dest' 102 | SrcSpace = Str(1:k-1); 103 | DestSpace = Str(k+1:end); 104 | else 105 | k = find(Str == '<'); 106 | 107 | if length(k) == 1 % Interpret the form 'dest<-src' 108 | DestSpace = Str(1:k-1); 109 | SrcSpace = Str(k+1:end); 110 | else 111 | error(['Invalid conversion, ''',Str,'''.']); 112 | end 113 | end 114 | 115 | SrcSpace = alias(SrcSpace); 116 | DestSpace = alias(DestSpace); 117 | else 118 | SrcSpace = 1; % No source pre-transform 119 | DestSpace = Conversion; 120 | if any(size(Conversion) ~= 3), error('Transformation matrix must be 3x3.'); end 121 | end 122 | return; 123 | 124 | 125 | function Space = alias(Space) 126 | Space = strrep(Space,'cie',''); 127 | 128 | if isempty(Space) 129 | Space = 'rgb'; 130 | end 131 | 132 | switch Space 133 | case {'ycbcr','ycc'} 134 | Space = 'ycbcr'; 135 | case {'hsv','hsb'} 136 | Space = 'hsv'; 137 | case {'hsl','hsi','hls'} 138 | Space = 'hsl'; 139 | case {'rgb','yuv','yiq','ydbdr','ycbcr','jpegycbcr','xyz','lab','luv','lch'} 140 | return; 141 | end 142 | return; 143 | 144 | 145 | function T = gettransform(Space) 146 | % Get a colorspace transform: either a matrix describing an affine transform, 147 | % or a string referring to a conversion subroutine 148 | switch Space 149 | case 'ypbpr' 150 | T = [0.299,0.587,0.114,0;-0.1687367,-0.331264,0.5,0;0.5,-0.418688,-0.081312,0]; 151 | case 'yuv' 152 | % R'G'B' to NTSC/PAL YUV 153 | % Wikipedia: http://en.wikipedia.org/wiki/YUV 154 | T = [0.299,0.587,0.114,0;-0.147,-0.289,0.436,0;0.615,-0.515,-0.100,0]; 155 | case 'ydbdr' 156 | % R'G'B' to SECAM YDbDr 157 | % Wikipedia: http://en.wikipedia.org/wiki/YDbDr 158 | T = [0.299,0.587,0.114,0;-0.450,-0.883,1.333,0;-1.333,1.116,0.217,0]; 159 | case 'yiq' 160 | % R'G'B' in [0,1] to NTSC YIQ in [0,1];[-0.595716,0.595716];[-0.522591,0.522591]; 161 | % Wikipedia: http://en.wikipedia.org/wiki/YIQ 162 | T = [0.299,0.587,0.114,0;0.595716,-0.274453,-0.321263,0;0.211456,-0.522591,0.311135,0]; 163 | case 'ycbcr' 164 | % R'G'B' (range [0,1]) to ITU-R BRT.601 (CCIR 601) Y'CbCr 165 | % Wikipedia: http://en.wikipedia.org/wiki/YCbCr 166 | % Poynton, Equation 3, scaling of R'G'B to Y'PbPr conversion 167 | T = [65.481,128.553,24.966,16;-37.797,-74.203,112.0,128;112.0,-93.786,-18.214,128]; 168 | case 'jpegycbcr' 169 | % Wikipedia: http://en.wikipedia.org/wiki/YCbCr 170 | T = [0.299,0.587,0.114,0;-0.168736,-0.331264,0.5,0.5;0.5,-0.418688,-0.081312,0.5]*255; 171 | case {'rgb','xyz','hsv','hsl','lab','luv','lch'} 172 | T = Space; 173 | otherwise 174 | error(['Unknown color space, ''',Space,'''.']); 175 | end 176 | return; 177 | 178 | 179 | function Image = rgb(Image,SrcSpace) 180 | % Convert to Rec. 709 R'G'B' from 'SrcSpace' 181 | switch SrcSpace 182 | case 'rgb' 183 | return; 184 | case 'hsv' 185 | % Convert HSV to R'G'B' 186 | Image = huetorgb((1 - Image(:,:,2)).*Image(:,:,3),Image(:,:,3),Image(:,:,1)); 187 | case 'hsl' 188 | % Convert HSL to R'G'B' 189 | L = Image(:,:,3); 190 | Delta = Image(:,:,2).*min(L,1-L); 191 | Image = huetorgb(L-Delta,L+Delta,Image(:,:,1)); 192 | case {'xyz','lab','luv','lch'} 193 | % Convert to CIE XYZ 194 | Image = xyz(Image,SrcSpace); 195 | % Convert XYZ to RGB 196 | T = [3.240479,-1.53715,-0.498535;-0.969256,1.875992,0.041556;0.055648,-0.204043,1.057311]; 197 | R = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3); % R 198 | G = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3); % G 199 | B = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3); % B 200 | % Desaturate and rescale to constrain resulting RGB values to [0,1] 201 | AddWhite = -min(min(min(R,G),B),0); 202 | Scale = max(max(max(R,G),B)+AddWhite,1); 203 | R = (R + AddWhite)./Scale; 204 | G = (G + AddWhite)./Scale; 205 | B = (B + AddWhite)./Scale; 206 | % Apply gamma correction to convert RGB to Rec. 709 R'G'B' 207 | Image(:,:,1) = gammacorrection(R); % R' 208 | Image(:,:,2) = gammacorrection(G); % G' 209 | Image(:,:,3) = gammacorrection(B); % B' 210 | otherwise % Conversion is through an affine transform 211 | T = gettransform(SrcSpace); 212 | temp = inv(T(:,1:3)); 213 | T = [temp,-temp*T(:,4)]; 214 | R = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3) + T(10); 215 | G = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3) + T(11); 216 | B = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3) + T(12); 217 | AddWhite = -min(min(min(R,G),B),0); 218 | Scale = max(max(max(R,G),B)+AddWhite,1); 219 | R = (R + AddWhite)./Scale; 220 | G = (G + AddWhite)./Scale; 221 | B = (B + AddWhite)./Scale; 222 | Image(:,:,1) = R; 223 | Image(:,:,2) = G; 224 | Image(:,:,3) = B; 225 | end 226 | 227 | % Clip to [0,1] 228 | Image = min(max(Image,0),1); 229 | return; 230 | 231 | 232 | function Image = xyz(Image,SrcSpace) 233 | % Convert to CIE XYZ from 'SrcSpace' 234 | WhitePoint = [0.950456,1,1.088754]; 235 | 236 | switch SrcSpace 237 | case 'xyz' 238 | return; 239 | case 'luv' 240 | % Convert CIE L*uv to XYZ 241 | WhitePointU = (4*WhitePoint(1))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 242 | WhitePointV = (9*WhitePoint(2))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 243 | L = Image(:,:,1); 244 | Y = (L + 16)/116; 245 | Y = invf(Y)*WhitePoint(2); 246 | U = Image(:,:,2)./(13*L + 1e-6*(L==0)) + WhitePointU; 247 | V = Image(:,:,3)./(13*L + 1e-6*(L==0)) + WhitePointV; 248 | Image(:,:,1) = -(9*Y.*U)./((U-4).*V - U.*V); % X 249 | Image(:,:,2) = Y; % Y 250 | Image(:,:,3) = (9*Y - (15*V.*Y) - (V.*Image(:,:,1)))./(3*V); % Z 251 | case {'lab','lch'} 252 | Image = lab(Image,SrcSpace); 253 | % Convert CIE L*ab to XYZ 254 | fY = (Image(:,:,1) + 16)/116; 255 | fX = fY + Image(:,:,2)/500; 256 | fZ = fY - Image(:,:,3)/200; 257 | Image(:,:,1) = WhitePoint(1)*invf(fX); % X 258 | Image(:,:,2) = WhitePoint(2)*invf(fY); % Y 259 | Image(:,:,3) = WhitePoint(3)*invf(fZ); % Z 260 | otherwise % Convert from some gamma-corrected space 261 | % Convert to Rec. 701 R'G'B' 262 | Image = rgb(Image,SrcSpace); 263 | % Undo gamma correction 264 | R = invgammacorrection(Image(:,:,1)); 265 | G = invgammacorrection(Image(:,:,2)); 266 | B = invgammacorrection(Image(:,:,3)); 267 | % Convert RGB to XYZ 268 | T = inv([3.240479,-1.53715,-0.498535;-0.969256,1.875992,0.041556;0.055648,-0.204043,1.057311]); 269 | Image(:,:,1) = T(1)*R + T(4)*G + T(7)*B; % X 270 | Image(:,:,2) = T(2)*R + T(5)*G + T(8)*B; % Y 271 | Image(:,:,3) = T(3)*R + T(6)*G + T(9)*B; % Z 272 | end 273 | return; 274 | 275 | 276 | function Image = hsv(Image,SrcSpace) 277 | % Convert to HSV 278 | Image = rgb(Image,SrcSpace); 279 | V = max(Image,[],3); 280 | S = (V - min(Image,[],3))./(V + (V == 0)); 281 | Image(:,:,1) = rgbtohue(Image); 282 | Image(:,:,2) = S; 283 | Image(:,:,3) = V; 284 | return; 285 | 286 | 287 | function Image = hsl(Image,SrcSpace) 288 | % Convert to HSL 289 | switch SrcSpace 290 | case 'hsv' 291 | % Convert HSV to HSL 292 | MaxVal = Image(:,:,3); 293 | MinVal = (1 - Image(:,:,2)).*MaxVal; 294 | L = 0.5*(MaxVal + MinVal); 295 | temp = min(L,1-L); 296 | Image(:,:,2) = 0.5*(MaxVal - MinVal)./(temp + (temp == 0)); 297 | Image(:,:,3) = L; 298 | otherwise 299 | Image = rgb(Image,SrcSpace); % Convert to Rec. 701 R'G'B' 300 | % Convert R'G'B' to HSL 301 | MinVal = min(Image,[],3); 302 | MaxVal = max(Image,[],3); 303 | L = 0.5*(MaxVal + MinVal); 304 | temp = min(L,1-L); 305 | S = 0.5*(MaxVal - MinVal)./(temp + (temp == 0)); 306 | Image(:,:,1) = rgbtohue(Image); 307 | Image(:,:,2) = S; 308 | Image(:,:,3) = L; 309 | end 310 | return; 311 | 312 | 313 | function Image = lab(Image,SrcSpace) 314 | % Convert to CIE L*a*b* (CIELAB) 315 | WhitePoint = [0.950456,1,1.088754]; 316 | 317 | switch SrcSpace 318 | case 'lab' 319 | return; 320 | case 'lch' 321 | % Convert CIE L*CH to CIE L*ab 322 | C = Image(:,:,2); 323 | Image(:,:,2) = cos(Image(:,:,3)*pi/180).*C; % a* 324 | Image(:,:,3) = sin(Image(:,:,3)*pi/180).*C; % b* 325 | otherwise 326 | Image = xyz(Image,SrcSpace); % Convert to XYZ 327 | % Convert XYZ to CIE L*a*b* 328 | X = Image(:,:,1)/WhitePoint(1); 329 | Y = Image(:,:,2)/WhitePoint(2); 330 | Z = Image(:,:,3)/WhitePoint(3); 331 | fX = f(X); 332 | fY = f(Y); 333 | fZ = f(Z); 334 | Image(:,:,1) = 116*fY - 16; % L* 335 | Image(:,:,2) = 500*(fX - fY); % a* 336 | Image(:,:,3) = 200*(fY - fZ); % b* 337 | end 338 | return; 339 | 340 | 341 | function Image = luv(Image,SrcSpace) 342 | % Convert to CIE L*u*v* (CIELUV) 343 | WhitePoint = [0.950456,1,1.088754]; 344 | WhitePointU = (4*WhitePoint(1))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 345 | WhitePointV = (9*WhitePoint(2))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 346 | 347 | Image = xyz(Image,SrcSpace); % Convert to XYZ 348 | U = (4*Image(:,:,1))./(Image(:,:,1) + 15*Image(:,:,2) + 3*Image(:,:,3)); 349 | V = (9*Image(:,:,2))./(Image(:,:,1) + 15*Image(:,:,2) + 3*Image(:,:,3)); 350 | Y = Image(:,:,2)/WhitePoint(2); 351 | L = 116*f(Y) - 16; 352 | Image(:,:,1) = L; % L* 353 | Image(:,:,2) = 13*L.*(U - WhitePointU); % u* 354 | Image(:,:,3) = 13*L.*(V - WhitePointV); % v* 355 | return; 356 | 357 | 358 | function Image = lch(Image,SrcSpace) 359 | % Convert to CIE L*ch 360 | Image = lab(Image,SrcSpace); % Convert to CIE L*ab 361 | H = atan2(Image(:,:,3),Image(:,:,2)); 362 | H = H*180/pi + 360*(H < 0); 363 | Image(:,:,2) = sqrt(Image(:,:,2).^2 + Image(:,:,3).^2); % C 364 | Image(:,:,3) = H; % H 365 | return; 366 | 367 | 368 | function Image = huetorgb(m0,m2,H) 369 | % Convert HSV or HSL hue to RGB 370 | N = size(H); 371 | H = min(max(H(:),0),360)/60; 372 | m0 = m0(:); 373 | m2 = m2(:); 374 | F = H - round(H/2)*2; 375 | M = [m0, m0 + (m2-m0).*abs(F), m2]; 376 | Num = length(m0); 377 | j = [2 1 0;1 2 0;0 2 1;0 1 2;1 0 2;2 0 1;2 1 0]*Num; 378 | k = floor(H) + 1; 379 | Image = reshape([M(j(k,1)+(1:Num).'),M(j(k,2)+(1:Num).'),M(j(k,3)+(1:Num).')],[N,3]); 380 | return; 381 | 382 | 383 | function H = rgbtohue(Image) 384 | % Convert RGB to HSV or HSL hue 385 | [M,i] = sort(Image,3); 386 | i = i(:,:,3); 387 | Delta = M(:,:,3) - M(:,:,1); 388 | Delta = Delta + (Delta == 0); 389 | R = Image(:,:,1); 390 | G = Image(:,:,2); 391 | B = Image(:,:,3); 392 | H = zeros(size(R)); 393 | k = (i == 1); 394 | H(k) = (G(k) - B(k))./Delta(k); 395 | k = (i == 2); 396 | H(k) = 2 + (B(k) - R(k))./Delta(k); 397 | k = (i == 3); 398 | H(k) = 4 + (R(k) - G(k))./Delta(k); 399 | H = 60*H + 360*(H < 0); 400 | H(Delta == 0) = nan; 401 | return; 402 | 403 | 404 | function Rp = gammacorrection(R) 405 | Rp = real(1.099*R.^0.45 - 0.099); 406 | i = (R < 0.018); 407 | Rp(i) = 4.5138*R(i); 408 | return; 409 | 410 | 411 | function R = invgammacorrection(Rp) 412 | R = real(((Rp + 0.099)/1.099).^(1/0.45)); 413 | i = (R < 0.018); 414 | R(i) = Rp(i)/4.5138; 415 | return; 416 | 417 | 418 | function fY = f(Y) 419 | fY = real(Y.^(1/3)); 420 | i = (Y < 0.008856); 421 | fY(i) = Y(i)*(841/108) + (4/29); 422 | return; 423 | 424 | 425 | function Y = invf(fY) 426 | Y = fY.^3; 427 | i = (Y < 0.008856); 428 | Y(i) = (fY(i) - 4/29)*(108/841); 429 | return; 430 | -------------------------------------------------------------------------------- /Code/MTMR/compare_PRF.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/MTMR/compare_PRF.m -------------------------------------------------------------------------------- /Code/MTMR/main_new - 备份.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | global imgRoot; 3 | global imgRoot1; 4 | global imgRoot2; 5 | global supdir; 6 | global saldir; 7 | global theta1; 8 | global theta2; 9 | global spnumber; 10 | global Gama; 11 | global beta; 12 | 13 | spnumber=300; 14 | u=0.02; 15 | u1=0.06; 16 | lambda=0.03; 17 | theta1=24; 18 | theta2=12; 19 | Gama=5; 20 | beta=0.25; 21 | 22 | A=cell(169,10); 23 | temp=1; 24 | imgRoot='./share/1/'; 25 | imgRoot1='./test/1/G/';% test grayscale image path 26 | imgRoot2='./test/1/T/';% test grayscale image path 27 | supdir='./superpixels/1/';% the superpixel label file path 28 | mkdir(imgRoot); 29 | mkdir(supdir); 30 | 31 | % for u1=0.4:0.4 32 | saldir=['./saliencymap/1/'];% the output path of the saliency map 33 | mkdir(saldir); 34 | sal=dir([saldir '*' 'png']); 35 | if(length(sal)<1642) 36 | DEMO11(lambda,u,u1); 37 | end 38 | % cd 'C:/Users/wgz/Desktop/multiM2/'; 39 | % [Pre,Recall,PreF,RecallF,FMeasureF]=Plot_PreRecallThousand_contrast(u1,lambda,saldir); 40 | 41 | % end 42 | -------------------------------------------------------------------------------- /Code/MTMR/main_new.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/MTMR/main_new.m -------------------------------------------------------------------------------- /Code/MTMR/makeweights.m: -------------------------------------------------------------------------------- 1 | function weights=makeweights(edges,vals,valScale) 2 | valDistances=sqrt(sum((vals(edges(:,1),:)-vals(edges(:,2),:)).^2,2)); 3 | valDistances=normalize(valDistances); %Normalize to [0,1] 4 | weights=exp(-valScale*valDistances); 5 | -------------------------------------------------------------------------------- /Code/MTMR/normalize.m: -------------------------------------------------------------------------------- 1 | function normVals=normalize(newVals,oldVals) 2 | %function normVals=normalize(newVals,oldVals) normalizes the range of 3 | %newVals to the range of oldVals such that every column is normalized 4 | %independantly 5 | % 6 | %Inputs: newVals - NxP matrix of new values to be normalized to the 7 | % range of oldVals 8 | % oldVals - Optional NxP matrix of original values (for 9 | % arbitrary K). Defaults to normalizing range to [0,1] 10 | % 11 | %Outputs: normVals - NxP matrix of newVals normalized (columnwise) to 12 | % the range of oldVals 13 | % 14 | % 15 | %5/13/03 - Leo Grady 16 | 17 | % Copyright (C) 2002, 2003 Leo Grady 18 | % Computer Vision and Computational Neuroscience Lab 19 | % Department of Cognitive and Neural Systems 20 | % Boston University 21 | % Boston, MA 02215 22 | % 23 | % This program is free software; you can redistribute it and/or 24 | % modify it under the terms of the GNU General Public License 25 | % as published by the Free Software Foundation; either version 2 26 | % of the License, or (at your option) any later version. 27 | % 28 | % This program is distributed in the hope that it will be useful, 29 | % but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | % GNU General Public License for more details. 32 | % 33 | % You should have received a copy of the GNU General Public License 34 | % along with this program; if not, write to the Free Software 35 | % Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 36 | % 37 | % Date - $Id: normalize.m,v 1.2 2003/08/21 17:29:29 lgrady Exp $ 38 | %========================================================================% 39 | 40 | %Initialize 41 | [N P]=size(newVals); 42 | 43 | %Supply optional argument, if required 44 | if nargin == 1 45 | oldVals=[zeros(1,P);ones(1,P)]; 46 | end 47 | 48 | %Find original minima/maxima 49 | minVal=min(oldVals,[],1); 50 | maxVal=max(oldVals,[],1); 51 | 52 | %Find current minima/maxima 53 | minNewVal=min(newVals,[],1); 54 | maxNewVal=max(newVals,[],1); 55 | 56 | %Perform normalization 57 | warning off MATLAB:divideByZero %Error for divide by zero handled below 58 | normVals=newVals-ones(N,1)*minNewVal; 59 | normVals=normVals.*(ones(N,1)*(maxVal-minVal))./ ... 60 | (ones(N,1)*max(normVals,[],1)); 61 | normVals=normVals+ones(N,1)*minVal; 62 | warning on MATLAB:divideByZero 63 | 64 | %Error check for completely uniform inputs 65 | uniformIndex=find(minNewVal==maxNewVal); 66 | normVals(:,uniformIndex)=ones(N,1)*minNewVal(:,uniformIndex); 67 | -------------------------------------------------------------------------------- /Code/MTMR/readme.txt: -------------------------------------------------------------------------------- 1 | The code is for paper "Saliency Detection via Graph-Based Manifold Ranking" 2 | by Chuan Yang, Lihe Zhang, Huchuan Lu, Ming-Hsuan Yang, and Xiang Ruan 3 | To appear in Proceedings of IEEE Conference on Computer Vision and Pattern Recognition (CVPR 2013), Portland, June, 2013. 4 | written by Chuan Yang 5 | Email: ycscience86@gmail.com 6 | ****************************************************************************************************************** 7 | The code is tested on Windows XP with MATLAB R2010b. 8 | ****************************************************************************************************************** 9 | Usage: 10 | >put the test images into file '\test' 11 | >run 'demo.m' 12 | ****************************************************************************************************************** 13 | Note: We observe that some images on the MSRA dataset are surrounded with artificial frames, 14 | which will invalidate the used boundary prior. Thus, we run a pre-processing to remove such obvious frames. 15 | 16 | Procedures: 17 | 1. compute a binary edge map of the image using the canny method. 18 | 2. if a rectangle is detected in a band of 30 pixels in width along the four sides of the edge map (i.e. we assume that the frame is not wider than 30 pixels), we will cut the aera outside the rectangle from the image. 19 | 20 | The file 'removeframe.m' is the pre-processing code. 21 | 22 | ****************************************************************************************************************** 23 | We use the SLIC superpixel software to generate superpixels (http://ivrg.epfl.ch/supplementary_material/RK_SLICSuperpixels/index.html) 24 | and some graph functions in the Graph Analysis Toolbox (http://eslab.bu.edu/software/graphanalysis/). 25 | 26 | Note: The running time reported in our paper does not include the time of the pre-processing and the running time of the superpixel generation is computed by using the SLIC Windows GUI based executable. -------------------------------------------------------------------------------- /Code/MTMR/removeframe.m: -------------------------------------------------------------------------------- 1 | function [S_input,input_img1,input_img2,w]=removeframe(img1,img2,S_imname) 2 | threshold=0.6; 3 | S_input=imread(S_imname); 4 | input_img1=imread(img1); 5 | input_img2=imread(img2); 6 | S_input=im2double(S_input); 7 | input_img1=im2double(input_img1); 8 | input_img2=im2double(input_img2); 9 | gray=rgb2gray(S_input); 10 | edgemap = edge(gray,'canny'); 11 | [m,n]=size(edgemap); 12 | flagt=0; 13 | flagd=0; 14 | flagr=0; 15 | flagl=0; 16 | t=1; 17 | d=1; 18 | l=1; 19 | r=1; 20 | 21 | for k=1:30 % we assume that the frame is not wider than 30 pixels. 22 | pbt=mean(edgemap(k,:)); 23 | pbd=mean(edgemap(m-k+1,:)); 24 | pbl=mean(edgemap(:,k)); 25 | pbr=mean(edgemap(:,n-k+1)); 26 | if pbt>threshold 27 | t=k; 28 | flagt=1; 29 | end 30 | if pbd>threshold 31 | d=k; 32 | flagd=1; 33 | end 34 | if pbl>threshold 35 | l=k; 36 | flagl=1; 37 | end 38 | if pbr>threshold 39 | r=k; 40 | flagr=1; 41 | end 42 | end 43 | 44 | flagrm=flagt+flagd+flagl+flagr; 45 | % we assume that there exists a frame when one more lines parallel to the image side are detected 46 | if flagrm>1 47 | maxwidth=max([t,d,l,r]); 48 | % 49 | if t==1 50 | t=maxwidth; 51 | end 52 | if d==1 53 | d=maxwidth; 54 | end 55 | if l==1 56 | l=maxwidth; 57 | end 58 | if r==1 59 | r=maxwidth; 60 | end 61 | S_input=S_input(t:m-d+1,l:n-r+1,:); 62 | input_img1=input_img1(t:m-d+1,l:n-r+1,:); 63 | input_img2=input_img2(t:m-d+1,l:n-r+1,:); 64 | w=[m,n,t,m-d+1,l,n-r+1]; 65 | else 66 | w=[m,n,1,m,1,n]; 67 | end 68 | % outname=[S_imname(1:end-4) '.bmp']; 69 | % imwrite(S_input,outname); 70 | 71 | 72 | -------------------------------------------------------------------------------- /Code/SGDL/AdjcProcloop.m: -------------------------------------------------------------------------------- 1 | function adjcMerge = AdjcProcloop(M,N) 2 | % $Description: 3 | % -compute the adjacent matrix 4 | % $Agruments 5 | % Input; 6 | % -M: superpixel label matrix 7 | % -N: superpixel number 8 | % Output: 9 | % -adjcMerge: adjacent matrix 10 | 11 | adjcMerge = zeros(N,N); 12 | [m n] = size(M); 13 | 14 | for i = 1:m-1 15 | for j = 1:n-1 16 | if(M(i,j)~=M(i,j+1)) 17 | adjcMerge(M(i,j),M(i,j+1)) = 1; 18 | adjcMerge(M(i,j+1),M(i,j)) = 1; 19 | end; 20 | if(M(i,j)~=M(i+1,j)) 21 | adjcMerge(M(i,j),M(i+1,j)) = 1; 22 | adjcMerge(M(i+1,j),M(i,j)) = 1; 23 | end; 24 | if(M(i,j)~=M(i+1,j+1)) 25 | adjcMerge(M(i,j),M(i+1,j+1)) = 1; 26 | adjcMerge(M(i+1,j+1),M(i,j)) = 1; 27 | end; 28 | if(M(i+1,j)~=M(i,j+1)) 29 | adjcMerge(M(i+1,j),M(i,j+1)) = 1; 30 | adjcMerge(M(i,j+1),M(i+1,j)) = 1; 31 | end; 32 | end; 33 | end; 34 | bd=unique([M(1,:),M(m,:),M(:,1)',M(:,n)']); 35 | for i=1:length(bd) 36 | for j=i+1:length(bd) 37 | adjcMerge(bd(i),bd(j))=1; 38 | adjcMerge(bd(j),bd(i))=1; 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /Code/SGDL/ComputeAffinityMatrix.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/SGDL/ComputeAffinityMatrix.m -------------------------------------------------------------------------------- /Code/SGDL/ExtractFCNfeature.m: -------------------------------------------------------------------------------- 1 | function [meanVgg1,meanVgg2] = ExtractFCNfeature(matRoot,imname,pixelList,m,n) 2 | 3 | layer1 = 32; 4 | layer2 = 6; 5 | 6 | padding = [0 100 100 100 100 ... 7 | 52 52 52 52 52 ... 8 | 26 26 26 26 26 26 26 ... 9 | 14 14 14 14 14 14 14 ... 10 | 7 7 7 7 7 7 7 ... 11 | 4]; 12 | 13 | 14 | matName1 = [matRoot,'32','\',imname,'.mat']; 15 | eval(['load ',matName1,';']); 16 | temp = layer32; % feat 17 | 18 | vgg_feat1 = temp(padding(layer1):end-padding(layer1)+1,padding(layer1):end-padding(layer1)+1,:); 19 | vgg_feat1 = double(imresize(vgg_feat1,[m,n])); 20 | meanVgg1 = GetMeanColor(vgg_feat1,pixelList,'vgg'); 21 | matName2 = [matRoot,'6','\',imname,'.mat']; 22 | eval(['load ',matName2,';']); 23 | temp = layer6; 24 | 25 | vgg_feat2 = temp(padding(layer2):end-padding(layer2)+1,padding(layer2):end-padding(layer2)+1,:); 26 | vgg_feat2 = double(imresize(vgg_feat2,[m,n])); 27 | meanVgg2 = GetMeanColor(vgg_feat2,pixelList,'vgg'); -------------------------------------------------------------------------------- /Code/SGDL/FCN.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/SGDL/FCN.zip -------------------------------------------------------------------------------- /Code/SGDL/Find_Edge_Superpixels.m: -------------------------------------------------------------------------------- 1 | function EdgSup = Find_Edge_Superpixels( Labels, K, height, width , Wcon, ConPix ) 2 | %% 3 | % obtain the indication of edge super-pixels 4 | % Input: 5 | % Labels: the super-pixel label obtained from SLIC 6 | % K: the number of super-pixels 7 | % height: the height of the image 8 | % width: the width of the image 9 | % Wcon: the affinity weight on the edge of the graph 10 | % ConPix: one layer neighbour relationship of super-pixels 11 | % Output: 12 | % EdgSup: the edge superpixel is indicated by value 1, 13 | % the superpixel in the edge frame is indicated by value 2. 14 | %%%%==================================================================== 15 | EdgSup=zeros( K,1); Check=0; 16 | for i=1:height 17 | EdgSup ( Labels( i,1 )+1 ) =1; 18 | EdgSup ( Labels(i, width) +1 )=1; 19 | end 20 | for i=1:width 21 | EdgSup (Labels(1,i) +1 )= 1 ; 22 | EdgSup (Labels(height, i) +1 ) =1; 23 | end 24 | EdgSupSecond = EdgSup; 25 | for j=1:K 26 | if EdgSup(j)==1 27 | for z=1:K 28 | if ( ConPix(j,z)>0 ) && ( EdgSup(z)==0 ) 29 | Check = Check + Wcon(j,z); 30 | EdgSupSecond( z ) = 1; 31 | end 32 | end 33 | if Check > 13 % heuristic threshold to discard the frame 34 | return; 35 | end 36 | end 37 | end 38 | EdgSup = EdgSup + EdgSupSecond; 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /Code/SGDL/GetEdges.m: -------------------------------------------------------------------------------- 1 | function my_edges = GetEdges(seed_fore,seed_back) 2 | 3 | ind_fore = find(seed_fore==1); 4 | ind_back = find(seed_back==1); 5 | my_edges = []; 6 | edges = zeros(length(ind_back),2); 7 | edges(:,2) = ind_back; 8 | for i = 1:length(ind_fore) 9 | edges(:,1) = ind_fore(i); 10 | my_edges = [my_edges;edges]; 11 | end -------------------------------------------------------------------------------- /Code/SGDL/GetMeanColor.m: -------------------------------------------------------------------------------- 1 | function meanCol = GetMeanColor(image, pixelList,feat_type) 2 | % Code Author: Wangjiang Zhu 3 | % Email: wangjiang88119@gmail.com 4 | % Date: 3/24/2014 5 | 6 | [h, w, chn] = size(image); 7 | if ~strcmp(feat_type,'vgg') 8 | image = uint8(image*255); 9 | end 10 | tmpImg=reshape(image, h*w, chn); 11 | spNum = length(pixelList); 12 | meanCol=zeros(spNum, chn); 13 | for i=1:spNum 14 | meanCol(i, :)=mean(tmpImg(pixelList{i},:), 1); 15 | end 16 | if chn ==1 %for gray images 17 | meanCol = repmat(meanCol, [1, 3]); 18 | end -------------------------------------------------------------------------------- /Code/SGDL/Learn.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/SGDL/Learn.m -------------------------------------------------------------------------------- /Code/SGDL/ReadDAT.m: -------------------------------------------------------------------------------- 1 | function B = ReadDAT(image_size,data_path) 2 | % $Description: 3 | % -read the superpixel labels from .dat file 4 | % $Agruments 5 | % Input; 6 | % -image_size: [width height] 7 | % -data_path: the path of the .dat file 8 | % Output: 9 | % -label matrix width*height 10 | 11 | row = image_size(1); 12 | colomn = image_size(2); 13 | fid = fopen(data_path,'r'); 14 | A = fread(fid, row * colomn, 'uint32')'; 15 | A = A + 1; 16 | B = reshape(A,[colomn, row]); 17 | B = B'; 18 | fclose(fid); -------------------------------------------------------------------------------- /Code/SGDL/SLICSuperpixelSegmentation.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/SGDL/SLICSuperpixelSegmentation.exe -------------------------------------------------------------------------------- /Code/SGDL/adjacency.m: -------------------------------------------------------------------------------- 1 | function W=adjacency(edges,weights,N) 2 | %Function W=adjacency(edges,weights,N) computes the weighted 3 | % adjacency matrix of a point and edge set. For an unweighted 4 | % matrix, set weights equal to ones(1,M) where M is the number 5 | % of edges in the graph. 6 | % 7 | %Inputs: edges - A Mx2 list of M edges indexing into points 8 | % weights - The weights used to determine matrix values. 9 | % If not specified, uses vector of all ones 10 | % N - Optional number of nodes in the graph. Used if 11 | % N > max(edges) (i.e., isolated nodes are present) 12 | % 13 | %Outputs: W - Adjacency matrix 14 | % 15 | % 16 | %Note: Adjacency matrix is not of the Seidel type 17 | % (i.e. all values are positive) 18 | %5/19/03 - Leo Grady 19 | 20 | % Copyright (C) 2002, 2003 Leo Grady 21 | % Computer Vision and Computational Neuroscience Lab 22 | % Department of Cognitive and Neural Systems 23 | % Boston University 24 | % Boston, MA 02215 25 | % 26 | % This program is free software; you can redistribute it and/or 27 | % modify it under the terms of the GNU General Public License 28 | % as published by the Free Software Foundation; either version 2 29 | % of the License, or (at your option) any later version. 30 | % 31 | % This program is distributed in the hope that it will be useful, 32 | % but WITHOUT ANY WARRANTY; without even the implied warranty of 33 | % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 34 | % GNU General Public License for more details. 35 | % 36 | % You should have received a copy of the GNU General Public License 37 | % along with this program; if not, write to the Free Software 38 | % Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 39 | % 40 | % Date - $Id: adjacency.m,v 1.2 2003/08/21 17:29:29 lgrady Exp $ 41 | %========================================================================% 42 | 43 | %If weights are not specified, use unity weighting 44 | if nargin == 1 45 | weights=ones(size(edges,1),1); 46 | end 47 | 48 | %If N is not specified, use maximum values of edges 49 | if nargin < 3 50 | N=max(max(edges)); 51 | end 52 | 53 | %Build sparse adjacency matrix 54 | W=sparse([edges(:,1);edges(:,2)],[edges(:,2);edges(:,1)], ... 55 | [weights;weights],N,N); 56 | -------------------------------------------------------------------------------- /Code/SGDL/colorspace.m: -------------------------------------------------------------------------------- 1 | function varargout = colorspace(Conversion,varargin) 2 | %COLORSPACE Convert a color image between color representations. 3 | % B = COLORSPACE(S,A) converts the color representation of image A 4 | % where S is a string specifying the conversion. S tells the 5 | % source and destination color spaces, S = 'dest<-src', or 6 | % alternatively, S = 'src->dest'. Supported color spaces are 7 | % 8 | % 'RGB' R'G'B' Red Green Blue (ITU-R BT.709 gamma-corrected) 9 | % 'YPbPr' Luma (ITU-R BT.601) + Chroma 10 | % 'YCbCr'/'YCC' Luma + Chroma ("digitized" version of Y'PbPr) 11 | % 'YUV' NTSC PAL Y'UV Luma + Chroma 12 | % 'YIQ' NTSC Y'IQ Luma + Chroma 13 | % 'YDbDr' SECAM Y'DbDr Luma + Chroma 14 | % 'JPEGYCbCr' JPEG-Y'CbCr Luma + Chroma 15 | % 'HSV'/'HSB' Hue Saturation Value/Brightness 16 | % 'HSL'/'HLS'/'HSI' Hue Saturation Luminance/Intensity 17 | % 'XYZ' CIE XYZ 18 | % 'Lab' CIE L*a*b* (CIELAB) 19 | % 'Luv' CIE L*u*v* (CIELUV) 20 | % 'Lch' CIE L*ch (CIELCH) 21 | % 22 | % All conversions assume 2 degree observer and D65 illuminant. Color 23 | % space names are case insensitive. When R'G'B' is the source or 24 | % destination, it can be omitted. For example 'yuv<-' is short for 25 | % 'yuv<-rgb'. 26 | % 27 | % MATLAB uses two standard data formats for R'G'B': double data with 28 | % intensities in the range 0 to 1, and uint8 data with integer-valued 29 | % intensities from 0 to 255. As MATLAB's native datatype, double data is 30 | % the natural choice, and the R'G'B' format used by colorspace. However, 31 | % for memory and computational performance, some functions also operate 32 | % with uint8 R'G'B'. Given uint8 R'G'B' color data, colorspace will 33 | % first cast it to double R'G'B' before processing. 34 | % 35 | % If A is an Mx3 array, like a colormap, B will also have size Mx3. 36 | % 37 | % [B1,B2,B3] = COLORSPACE(S,A) specifies separate output channels. 38 | % COLORSPACE(S,A1,A2,A3) specifies separate input channels. 39 | 40 | % Pascal Getreuer 2005-2006 41 | 42 | %%% Input parsing %%% 43 | % if 0 44 | if nargin < 2, error('Not enough input arguments.'); end 45 | [SrcSpace,DestSpace] = parse(Conversion); 46 | 47 | if nargin == 2 48 | Image = varargin{1}; 49 | elseif nargin >= 3 50 | Image = cat(3,varargin{:}); 51 | else 52 | error('Invalid number of input arguments.'); 53 | end 54 | 55 | FlipDims = (size(Image,3) == 1); 56 | 57 | if FlipDims, Image = permute(Image,[1,3,2]); end 58 | if ~isa(Image,'double'), Image = double(Image)/255; end 59 | if size(Image,3) ~= 3, error('Invalid input size.'); end 60 | 61 | SrcT = gettransform(SrcSpace); 62 | DestT = gettransform(DestSpace); 63 | 64 | if ~ischar(SrcT) & ~ischar(DestT) 65 | % Both source and destination transforms are affine, so they 66 | % can be composed into one affine operation 67 | T = [DestT(:,1:3)*SrcT(:,1:3),DestT(:,1:3)*SrcT(:,4)+DestT(:,4)]; 68 | Temp = zeros(size(Image)); 69 | Temp(:,:,1) = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3) + T(10); 70 | Temp(:,:,2) = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3) + T(11); 71 | Temp(:,:,3) = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3) + T(12); 72 | Image = Temp; 73 | elseif ~ischar(DestT) 74 | Image = rgb(Image,SrcSpace); 75 | Temp = zeros(size(Image)); 76 | Temp(:,:,1) = DestT(1)*Image(:,:,1) + DestT(4)*Image(:,:,2) + DestT(7)*Image(:,:,3) + DestT(10); 77 | Temp(:,:,2) = DestT(2)*Image(:,:,1) + DestT(5)*Image(:,:,2) + DestT(8)*Image(:,:,3) + DestT(11); 78 | Temp(:,:,3) = DestT(3)*Image(:,:,1) + DestT(6)*Image(:,:,2) + DestT(9)*Image(:,:,3) + DestT(12); 79 | Image = Temp; 80 | else 81 | Image = feval(DestT,Image,SrcSpace); 82 | end 83 | 84 | %%% Output format %%% 85 | if nargout > 1 86 | varargout = {Image(:,:,1),Image(:,:,2),Image(:,:,3)}; 87 | else 88 | if FlipDims, Image = permute(Image,[1,3,2]); end 89 | varargout = {Image}; 90 | end 91 | 92 | return; 93 | 94 | 95 | function [SrcSpace,DestSpace] = parse(Str) 96 | % Parse conversion argument 97 | 98 | if isstr(Str) 99 | Str = lower(strrep(strrep(Str,'-',''),' ','')); 100 | k = find(Str == '>'); 101 | 102 | if length(k) == 1 % Interpret the form 'src->dest' 103 | SrcSpace = Str(1:k-1); 104 | DestSpace = Str(k+1:end); 105 | else 106 | k = find(Str == '<'); 107 | 108 | if length(k) == 1 % Interpret the form 'dest<-src' 109 | DestSpace = Str(1:k-1); 110 | SrcSpace = Str(k+1:end); 111 | else 112 | error(['Invalid conversion, ''',Str,'''.']); 113 | end 114 | end 115 | 116 | SrcSpace = alias(SrcSpace); 117 | DestSpace = alias(DestSpace); 118 | else 119 | SrcSpace = 1; % No source pre-transform 120 | DestSpace = Conversion; 121 | if any(size(Conversion) ~= 3), error('Transformation matrix must be 3x3.'); end 122 | end 123 | return; 124 | 125 | 126 | function Space = alias(Space) 127 | Space = strrep(Space,'cie',''); 128 | 129 | if isempty(Space) 130 | Space = 'rgb'; 131 | end 132 | 133 | switch Space 134 | case {'ycbcr','ycc'} 135 | Space = 'ycbcr'; 136 | case {'hsv','hsb'} 137 | Space = 'hsv'; 138 | case {'hsl','hsi','hls'} 139 | Space = 'hsl'; 140 | case {'rgb','yuv','yiq','ydbdr','ycbcr','jpegycbcr','xyz','lab','luv','lch'} 141 | return; 142 | end 143 | return; 144 | 145 | 146 | function T = gettransform(Space) 147 | % Get a colorspace transform: either a matrix describing an affine transform, 148 | % or a string referring to a conversion subroutine 149 | switch Space 150 | case 'ypbpr' 151 | T = [0.299,0.587,0.114,0;-0.1687367,-0.331264,0.5,0;0.5,-0.418688,-0.081312,0]; 152 | case 'yuv' 153 | % R'G'B' to NTSC/PAL YUV 154 | % Wikipedia: http://en.wikipedia.org/wiki/YUV 155 | T = [0.299,0.587,0.114,0;-0.147,-0.289,0.436,0;0.615,-0.515,-0.100,0]; 156 | case 'ydbdr' 157 | % R'G'B' to SECAM YDbDr 158 | % Wikipedia: http://en.wikipedia.org/wiki/YDbDr 159 | T = [0.299,0.587,0.114,0;-0.450,-0.883,1.333,0;-1.333,1.116,0.217,0]; 160 | case 'yiq' 161 | % R'G'B' in [0,1] to NTSC YIQ in [0,1];[-0.595716,0.595716];[-0.522591,0.522591]; 162 | % Wikipedia: http://en.wikipedia.org/wiki/YIQ 163 | T = [0.299,0.587,0.114,0;0.595716,-0.274453,-0.321263,0;0.211456,-0.522591,0.311135,0]; 164 | case 'ycbcr' 165 | % R'G'B' (range [0,1]) to ITU-R BRT.601 (CCIR 601) Y'CbCr 166 | % Wikipedia: http://en.wikipedia.org/wiki/YCbCr 167 | % Poynton, Equation 3, scaling of R'G'B to Y'PbPr conversion 168 | T = [65.481,128.553,24.966,16;-37.797,-74.203,112.0,128;112.0,-93.786,-18.214,128]; 169 | case 'jpegycbcr' 170 | % Wikipedia: http://en.wikipedia.org/wiki/YCbCr 171 | T = [0.299,0.587,0.114,0;-0.168736,-0.331264,0.5,0.5;0.5,-0.418688,-0.081312,0.5]*255; 172 | case {'rgb','xyz','hsv','hsl','lab','luv','lch'} 173 | T = Space; 174 | otherwise 175 | error(['Unknown color space, ''',Space,'''.']); 176 | end 177 | return; 178 | 179 | 180 | function Image = rgb(Image,SrcSpace) 181 | % Convert to Rec. 709 R'G'B' from 'SrcSpace' 182 | switch SrcSpace 183 | case 'rgb' 184 | return; 185 | case 'hsv' 186 | % Convert HSV to R'G'B' 187 | Image = huetorgb((1 - Image(:,:,2)).*Image(:,:,3),Image(:,:,3),Image(:,:,1)); 188 | case 'hsl' 189 | % Convert HSL to R'G'B' 190 | L = Image(:,:,3); 191 | Delta = Image(:,:,2).*min(L,1-L); 192 | Image = huetorgb(L-Delta,L+Delta,Image(:,:,1)); 193 | case {'xyz','lab','luv','lch'} 194 | % Convert to CIE XYZ 195 | Image = xyz(Image,SrcSpace); 196 | % Convert XYZ to RGB 197 | T = [3.240479,-1.53715,-0.498535;-0.969256,1.875992,0.041556;0.055648,-0.204043,1.057311]; 198 | R = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3); % R 199 | G = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3); % G 200 | B = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3); % B 201 | % Desaturate and rescale to constrain resulting RGB values to [0,1] 202 | AddWhite = -min(min(min(R,G),B),0); 203 | Scale = max(max(max(R,G),B)+AddWhite,1); 204 | R = (R + AddWhite)./Scale; 205 | G = (G + AddWhite)./Scale; 206 | B = (B + AddWhite)./Scale; 207 | % Apply gamma correction to convert RGB to Rec. 709 R'G'B' 208 | Image(:,:,1) = gammacorrection(R); % R' 209 | Image(:,:,2) = gammacorrection(G); % G' 210 | Image(:,:,3) = gammacorrection(B); % B' 211 | otherwise % Conversion is through an affine transform 212 | T = gettransform(SrcSpace); 213 | temp = inv(T(:,1:3)); 214 | T = [temp,-temp*T(:,4)]; 215 | R = T(1)*Image(:,:,1) + T(4)*Image(:,:,2) + T(7)*Image(:,:,3) + T(10); 216 | G = T(2)*Image(:,:,1) + T(5)*Image(:,:,2) + T(8)*Image(:,:,3) + T(11); 217 | B = T(3)*Image(:,:,1) + T(6)*Image(:,:,2) + T(9)*Image(:,:,3) + T(12); 218 | AddWhite = -min(min(min(R,G),B),0); 219 | Scale = max(max(max(R,G),B)+AddWhite,1); 220 | R = (R + AddWhite)./Scale; 221 | G = (G + AddWhite)./Scale; 222 | B = (B + AddWhite)./Scale; 223 | Image(:,:,1) = R; 224 | Image(:,:,2) = G; 225 | Image(:,:,3) = B; 226 | end 227 | 228 | % Clip to [0,1] 229 | Image = min(max(Image,0),1); 230 | return; 231 | 232 | 233 | function Image = xyz(Image,SrcSpace) 234 | % Convert to CIE XYZ from 'SrcSpace' 235 | WhitePoint = [0.950456,1,1.088754]; 236 | 237 | switch SrcSpace 238 | case 'xyz' 239 | return; 240 | case 'luv' 241 | % Convert CIE L*uv to XYZ 242 | WhitePointU = (4*WhitePoint(1))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 243 | WhitePointV = (9*WhitePoint(2))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 244 | L = Image(:,:,1); 245 | Y = (L + 16)/116; 246 | Y = invf(Y)*WhitePoint(2); 247 | U = Image(:,:,2)./(13*L + 1e-6*(L==0)) + WhitePointU; 248 | V = Image(:,:,3)./(13*L + 1e-6*(L==0)) + WhitePointV; 249 | Image(:,:,1) = -(9*Y.*U)./((U-4).*V - U.*V); % X 250 | Image(:,:,2) = Y; % Y 251 | Image(:,:,3) = (9*Y - (15*V.*Y) - (V.*Image(:,:,1)))./(3*V); % Z 252 | case {'lab','lch'} 253 | Image = lab(Image,SrcSpace); 254 | % Convert CIE L*ab to XYZ 255 | fY = (Image(:,:,1) + 16)/116; 256 | fX = fY + Image(:,:,2)/500; 257 | fZ = fY - Image(:,:,3)/200; 258 | Image(:,:,1) = WhitePoint(1)*invf(fX); % X 259 | Image(:,:,2) = WhitePoint(2)*invf(fY); % Y 260 | Image(:,:,3) = WhitePoint(3)*invf(fZ); % Z 261 | otherwise % Convert from some gamma-corrected space 262 | % Convert to Rec. 701 R'G'B' 263 | Image = rgb(Image,SrcSpace); 264 | % Undo gamma correction 265 | R = invgammacorrection(Image(:,:,1)); 266 | G = invgammacorrection(Image(:,:,2)); 267 | B = invgammacorrection(Image(:,:,3)); 268 | % Convert RGB to XYZ 269 | T = inv([3.240479,-1.53715,-0.498535;-0.969256,1.875992,0.041556;0.055648,-0.204043,1.057311]); 270 | Image(:,:,1) = T(1)*R + T(4)*G + T(7)*B; % X 271 | Image(:,:,2) = T(2)*R + T(5)*G + T(8)*B; % Y 272 | Image(:,:,3) = T(3)*R + T(6)*G + T(9)*B; % Z 273 | end 274 | return; 275 | 276 | 277 | function Image = hsv(Image,SrcSpace) 278 | % Convert to HSV 279 | Image = rgb(Image,SrcSpace); 280 | V = max(Image,[],3); 281 | S = (V - min(Image,[],3))./(V + (V == 0)); 282 | Image(:,:,1) = rgbtohue(Image); 283 | Image(:,:,2) = S; 284 | Image(:,:,3) = V; 285 | return; 286 | 287 | 288 | function Image = hsl(Image,SrcSpace) 289 | % Convert to HSL 290 | switch SrcSpace 291 | case 'hsv' 292 | % Convert HSV to HSL 293 | MaxVal = Image(:,:,3); 294 | MinVal = (1 - Image(:,:,2)).*MaxVal; 295 | L = 0.5*(MaxVal + MinVal); 296 | temp = min(L,1-L); 297 | Image(:,:,2) = 0.5*(MaxVal - MinVal)./(temp + (temp == 0)); 298 | Image(:,:,3) = L; 299 | otherwise 300 | Image = rgb(Image,SrcSpace); % Convert to Rec. 701 R'G'B' 301 | % Convert R'G'B' to HSL 302 | MinVal = min(Image,[],3); 303 | MaxVal = max(Image,[],3); 304 | L = 0.5*(MaxVal + MinVal); 305 | temp = min(L,1-L); 306 | S = 0.5*(MaxVal - MinVal)./(temp + (temp == 0)); 307 | Image(:,:,1) = rgbtohue(Image); 308 | Image(:,:,2) = S; 309 | Image(:,:,3) = L; 310 | end 311 | return; 312 | 313 | 314 | function Image = lab(Image,SrcSpace) 315 | % Convert to CIE L*a*b* (CIELAB) 316 | WhitePoint = [0.950456,1,1.088754]; 317 | 318 | switch SrcSpace 319 | case 'lab' 320 | return; 321 | case 'lch' 322 | % Convert CIE L*CH to CIE L*ab 323 | C = Image(:,:,2); 324 | Image(:,:,2) = cos(Image(:,:,3)*pi/180).*C; % a* 325 | Image(:,:,3) = sin(Image(:,:,3)*pi/180).*C; % b* 326 | otherwise 327 | Image = xyz(Image,SrcSpace); % Convert to XYZ 328 | % Convert XYZ to CIE L*a*b* 329 | X = Image(:,:,1)/WhitePoint(1); 330 | Y = Image(:,:,2)/WhitePoint(2); 331 | Z = Image(:,:,3)/WhitePoint(3); 332 | fX = f(X); 333 | fY = f(Y); 334 | fZ = f(Z); 335 | Image(:,:,1) = 116*fY - 16; % L* 336 | Image(:,:,2) = 500*(fX - fY); % a* 337 | Image(:,:,3) = 200*(fY - fZ); % b* 338 | end 339 | return; 340 | 341 | 342 | function Image = luv(Image,SrcSpace) 343 | % Convert to CIE L*u*v* (CIELUV) 344 | WhitePoint = [0.950456,1,1.088754]; 345 | WhitePointU = (4*WhitePoint(1))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 346 | WhitePointV = (9*WhitePoint(2))./(WhitePoint(1) + 15*WhitePoint(2) + 3*WhitePoint(3)); 347 | 348 | Image = xyz(Image,SrcSpace); % Convert to XYZ 349 | U = (4*Image(:,:,1))./(Image(:,:,1) + 15*Image(:,:,2) + 3*Image(:,:,3)); 350 | V = (9*Image(:,:,2))./(Image(:,:,1) + 15*Image(:,:,2) + 3*Image(:,:,3)); 351 | Y = Image(:,:,2)/WhitePoint(2); 352 | L = 116*f(Y) - 16; 353 | Image(:,:,1) = L; % L* 354 | Image(:,:,2) = 13*L.*(U - WhitePointU); % u* 355 | Image(:,:,3) = 13*L.*(V - WhitePointV); % v* 356 | return; 357 | 358 | 359 | function Image = lch(Image,SrcSpace) 360 | % Convert to CIE L*ch 361 | Image = lab(Image,SrcSpace); % Convert to CIE L*ab 362 | H = atan2(Image(:,:,3),Image(:,:,2)); 363 | H = H*180/pi + 360*(H < 0); 364 | Image(:,:,2) = sqrt(Image(:,:,2).^2 + Image(:,:,3).^2); % C 365 | Image(:,:,3) = H; % H 366 | return; 367 | 368 | 369 | function Image = huetorgb(m0,m2,H) 370 | % Convert HSV or HSL hue to RGB 371 | N = size(H); 372 | H = min(max(H(:),0),360)/60; 373 | m0 = m0(:); 374 | m2 = m2(:); 375 | F = H - round(H/2)*2; 376 | M = [m0, m0 + (m2-m0).*abs(F), m2]; 377 | Num = length(m0); 378 | j = [2 1 0;1 2 0;0 2 1;0 1 2;1 0 2;2 0 1;2 1 0]*Num; 379 | k = floor(H) + 1; 380 | Image = reshape([M(j(k,1)+(1:Num).'),M(j(k,2)+(1:Num).'),M(j(k,3)+(1:Num).')],[N,3]); 381 | return; 382 | 383 | 384 | function H = rgbtohue(Image) 385 | % Convert RGB to HSV or HSL hue 386 | [M,i] = sort(Image,3); 387 | i = i(:,:,3); 388 | Delta = M(:,:,3) - M(:,:,1); 389 | Delta = Delta + (Delta == 0); 390 | R = Image(:,:,1); 391 | G = Image(:,:,2); 392 | B = Image(:,:,3); 393 | H = zeros(size(R)); 394 | k = (i == 1); 395 | H(k) = (G(k) - B(k))./Delta(k); 396 | k = (i == 2); 397 | H(k) = 2 + (B(k) - R(k))./Delta(k); 398 | k = (i == 3); 399 | H(k) = 4 + (R(k) - G(k))./Delta(k); 400 | H = 60*H + 360*(H < 0); 401 | H(Delta == 0) = nan; 402 | return; 403 | 404 | 405 | function Rp = gammacorrection(R) 406 | Rp = real(1.099*R.^0.45 - 0.099); 407 | i = (R < 0.018); 408 | Rp(i) = 4.5138*R(i); 409 | return; 410 | 411 | 412 | function R = invgammacorrection(Rp) 413 | R = real(((Rp + 0.099)/1.099).^(1/0.45)); 414 | i = (R < 0.018); 415 | R(i) = Rp(i)/4.5138; 416 | return; 417 | 418 | 419 | function fY = f(Y) 420 | fY = real(Y.^(1/3)); 421 | i = (Y < 0.008856); 422 | fY(i) = Y(i)*(841/108) + (4/29); 423 | return; 424 | 425 | 426 | function Y = invf(fY) 427 | Y = fY.^3; 428 | i = (Y < 0.008856); 429 | Y(i) = (fY(i) - 4/29)*(108/841); 430 | return; 431 | % end -------------------------------------------------------------------------------- /Code/SGDL/demo.m: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lz118/RGBT-Salient-Object-Detection/a38d4118176b3ffdfb3285e7b2abcf5c6b0adb19/Code/SGDL/demo.m -------------------------------------------------------------------------------- /Code/SGDL/find_connect_superpixel_DoubleIn_Opposite.m: -------------------------------------------------------------------------------- 1 | function [ ConPix0, ConPixSecond ] = find_connect_superpixel_DoubleIn_Opposite(labels, K, height ,width ) 2 | %% 3 | % obtain the neighbour relationship of the super-pixels 4 | % Input: 5 | % labels: the super-pixel label obtained from SLIC 6 | % K: the number of super-pixels 7 | % height: the height of the image 8 | % width: the width of the image 9 | % Output: 10 | % ConPix0: the one layer neighbour relationship 11 | % ConPixSecond: the two layer neighbour relationship 12 | %%%%===================================================== 13 | ConPix=zeros(K,K); 14 | %the one outerboundary super 15 | for i=1:height-1 16 | for j=1:width-1 17 | if labels(i,j)~=labels(i,j+1) 18 | ConPix(labels(i,j)+1 ,labels(i,j+1)+1 )=1; 19 | end 20 | if labels(i,j)~=labels(i+1,j) 21 | ConPix(labels(i,j)+1 ,labels(i+1,j)+1 )=1; 22 | end 23 | end 24 | if labels(i,j+1)~=labels(i+1,j+1) 25 | ConPix(labels(i,j+1)+1 ,labels(i+1,j+1 )+1 )=1; 26 | end 27 | end 28 | for j=1:width-1 29 | if labels(height,j)~=labels(height,j+1) 30 | ConPix( labels(height,j)+1,labels(height,j+1)+1 )=1; 31 | end 32 | end 33 | for i=1:height-1 34 | for j=1:width-1 35 | if labels(i,j)~=labels(i+1,j+1) 36 | ConPix( labels(i,j)+1,labels(i+1,j+1)+1 )=1; 37 | end 38 | end 39 | end 40 | for i=1:height-1 41 | for j=2:width 42 | if labels(i,j)~=labels(i+1,j-1) 43 | ConPix( labels(i,j)+1,labels(i+1,j-1)+1 )=1; 44 | end 45 | end 46 | end 47 | ConPix0 = ConPix + ConPix'; 48 | % connect the super-pixel on the opposite boundary 49 | for j=1:width 50 | ConPix( labels(1,j)+1 , labels(height,j)+1 ) = 1; 51 | end 52 | for j=1:height 53 | ConPix( labels(j,1)+1 , labels(j,width)+1 ) = 1 ; 54 | end 55 | ConPix=ConPix+ConPix'; 56 | % find the second outerboundary superpixel 57 | ConPixSecond = ConPix; 58 | for i=1:K 59 | siteline=find( ConPix(i,:)>0 ); 60 | lenthsiteline=length(siteline); 61 | for j=1:lenthsiteline 62 | ConPixSecond(i,:)= ConPixSecond(i,:)+ ConPix( siteline( j ), :); 63 | end 64 | end 65 | % find third outerboundary superpixel 66 | % ConPixTid = ConPixSecond; 67 | % for i=1:K 68 | % siteline=find( ConPixSecond(i,:)>0 ); 69 | % lenthsiteline=length(siteline); 70 | % for j=1:lenthsiteline 71 | % ConPixTid(i,:)= ConPixTid(i,:)+ ConPix( siteline( j ), :); 72 | % end 73 | % end 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /Code/SGDL/makeweights.m: -------------------------------------------------------------------------------- 1 | function weights=makeweights(edges,vals,valScale) 2 | valDistances=sqrt(sum((vals(edges(:,1),:)-vals(edges(:,2),:)).^2,2)); 3 | valDistances=normalize(valDistances); %Normalize to [0,1] 4 | weights=exp(-valScale*valDistances); 5 | -------------------------------------------------------------------------------- /Code/SGDL/normalize.m: -------------------------------------------------------------------------------- 1 | function normVals=normalize(newVals,oldVals) 2 | %function normVals=normalize(newVals,oldVals) normalizes the range of 3 | %newVals to the range of oldVals such that every column is normalized 4 | %independantly 5 | % 6 | %Inputs: newVals - NxP matrix of new values to be normalized to the 7 | % range of oldVals 8 | % oldVals - Optional NxP matrix of original values (for 9 | % arbitrary K). Defaults to normalizing range to [0,1] 10 | % 11 | %Outputs: normVals - NxP matrix of newVals normalized (columnwise) to 12 | % the range of oldVals 13 | % 14 | % 15 | %5/13/03 - Leo Grady 16 | 17 | % Copyright (C) 2002, 2003 Leo Grady 18 | % Computer Vision and Computational Neuroscience Lab 19 | % Department of Cognitive and Neural Systems 20 | % Boston University 21 | % Boston, MA 02215 22 | % 23 | % This program is free software; you can redistribute it and/or 24 | % modify it under the terms of the GNU General Public License 25 | % as published by the Free Software Foundation; either version 2 26 | % of the License, or (at your option) any later version. 27 | % 28 | % This program is distributed in the hope that it will be useful, 29 | % but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | % GNU General Public License for more details. 32 | % 33 | % You should have received a copy of the GNU General Public License 34 | % along with this program; if not, write to the Free Software 35 | % Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 36 | % 37 | % Date - $Id: normalize.m,v 1.2 2003/08/21 17:29:29 lgrady Exp $ 38 | %========================================================================% 39 | 40 | %Initialize 41 | [N P]=size(newVals); 42 | 43 | %Supply optional argument, if required 44 | if nargin == 1 45 | oldVals=[zeros(1,P);ones(1,P)]; 46 | end 47 | 48 | %Find original minima/maxima 49 | minVal=min(oldVals,[],1); 50 | maxVal=max(oldVals,[],1); 51 | 52 | %Find current minima/maxima 53 | minNewVal=min(newVals,[],1); 54 | maxNewVal=max(newVals,[],1); 55 | 56 | %Perform normalization 57 | warning off MATLAB:divideByZero %Error for divide by zero handled below 58 | normVals=newVals-ones(N,1)*minNewVal; 59 | normVals=normVals.*(ones(N,1)*(maxVal-minVal))./ ... 60 | (ones(N,1)*max(normVals,[],1)); 61 | normVals=normVals+ones(N,1)*minVal; 62 | warning on MATLAB:divideByZero 63 | 64 | %Error check for completely uniform inputs 65 | uniformIndex=find(minNewVal==maxNewVal); 66 | normVals(:,uniformIndex)=ones(N,1)*minNewVal(:,uniformIndex); 67 | -------------------------------------------------------------------------------- /Code/SGDL/readme.txt: -------------------------------------------------------------------------------- 1 | The code is for paper "RGB-T Image Saliency Detection via Collaborative Graph Learning" 2 | by Zhengzheng Tu, Tian Xia, Chenglong Li, Xiaoxiao Wang, Yan Ma and Jin Tang 3 | To appear in Proceedings of IEEE TRANSACTIONS ON MULTIMEDIA,2019. 4 | written by Tian Xia 5 | ****************************************************************************************************************** 6 | The code is tested on Windows10 with MATLAB R2016a. 7 | ****************************************************************************************************************** 8 | Usage: 9 | >put the test images(included RGB and Thermal image) into file 'test' 10 | We extracted the FCN-32S feature of RGB and Thermal image( https://github.com/DUTFangXiang/ExtractFCNFeature) 11 | >put the FCN-32S features into file 'FCN-feature' 12 | >run 'demo.m' 13 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # RGBT-Salient-Object-Detection 2 | 3 | A resource collection of RGBT salient object detection. 4 | 5 | 6 | 7 | ## Paper 8 | 9 | 1. A Unified RGB-T Saliency Detection Benchmark: Dataset, Baselines, Analysis and A Novel Approach. 10 | 2. RGB-T Image Saliency Detection via Collaborative Graph Learning. 11 | 3. M3S-NIR: Multi-Modal Multi-Scale Noise-Insensitive Ranking for RGB-T Saliency Detection. 12 | 4. Multi-Interactive Dual-Decoder for RGB-Thermal Salient Object Detection. 13 | 5. RGBT Salient Object Detection: A Large-scale Dataset and Benchmark. 14 | 15 | All the paper can be downloaded at https://pan.baidu.com/s/1IHHwHTh1In3dlWMKNtCK6A [evls] 16 | 17 | 18 | 19 | ## Dataset 20 | 21 | - VT821 https://pan.baidu.com/s/1ksuUr3cr6_-fZAsSUp0n0w [9yqv] 22 | 23 | - VT1000 https://pan.baidu.com/s/1i7gfrHoaaRuateMXBxvmMw [tb6l] 24 | 25 | - VT5000 https://pan.baidu.com/s/196S1GcnI56Vn6fLO3oXb5Q [y9jj] 26 | 27 | 28 | 29 | 30 | ## MTMR 31 | 32 | - Saliency maps in all datasets: https://pan.baidu.com/s/1nmooebYg31R8Z6p3cxb7sA [aki9] 33 | 34 | 35 | 36 | ## M3S-NIR 37 | 38 | - Saliency maps in all datasets: https://pan.baidu.com/s/1nqa5jLGp82oSR16TPMOIDw [d121] 39 | 40 | 41 | ## SGDL 42 | 43 | - The pretrained FCN model can be downloaded at：https://pan.baidu.com/s/1t7zqoO0f-e7P2yDfhvwuiw [utq1] 44 | 45 | - Saliency maps in all datasets: https://pan.baidu.com/s/1N_Y9apeyBcGdS4BzkZQryg [jnt0] 46 | 47 | 48 | ## ADF 49 | 50 | - The pretrained model(VGG16) and the saved model can be downloaded at https://pan.baidu.com/s/1IE1H99EyVcAzpDwmDdDNYw [0cs2] 51 | 52 | - Saliency maps in all datasets: https://pan.baidu.com/s/17vugADwbbMzgncUutpZjGw [9w8p] 53 | 54 | ## MIDD 55 | 56 | - Project https://github.com/lz118/Multi-interactive-Dual-decoder 57 | --------------------------------------------------------------------------------