├── .gitignore ├── Demo.ipynb ├── IV_images ├── IR1.png ├── IR10.png ├── IR11.png ├── IR12.png ├── IR13.png ├── IR14.png ├── IR15.png ├── IR16.png ├── IR17.png ├── IR18.png ├── IR19.png ├── IR2.png ├── IR20.png ├── IR21.png ├── IR3.png ├── IR4.png ├── IR5.png ├── IR6.png ├── IR7.png ├── IR8.png ├── IR9.png ├── VIS1.jpg ├── VIS1.png ├── VIS10.png ├── VIS11.png ├── VIS12.png ├── VIS13.png ├── VIS14.png ├── VIS15.png ├── VIS16.png ├── VIS17.png ├── VIS18.png ├── VIS19.png ├── VIS2.png ├── VIS20.png ├── VIS21.png ├── VIS3.jpg ├── VIS3.png ├── VIS4.jpg ├── VIS4.png ├── VIS5.jpg ├── VIS5.png ├── VIS6.png ├── VIS7.png ├── VIS8.png └── VIS9.png ├── README.md └── vggfusion.py /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | 3 | *.pyc 4 | -------------------------------------------------------------------------------- /IV_images/IR1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR1.png -------------------------------------------------------------------------------- /IV_images/IR10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR10.png -------------------------------------------------------------------------------- /IV_images/IR11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR11.png -------------------------------------------------------------------------------- /IV_images/IR12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR12.png -------------------------------------------------------------------------------- /IV_images/IR13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR13.png -------------------------------------------------------------------------------- /IV_images/IR14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR14.png -------------------------------------------------------------------------------- /IV_images/IR15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR15.png -------------------------------------------------------------------------------- /IV_images/IR16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR16.png -------------------------------------------------------------------------------- /IV_images/IR17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR17.png -------------------------------------------------------------------------------- /IV_images/IR18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR18.png -------------------------------------------------------------------------------- /IV_images/IR19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR19.png -------------------------------------------------------------------------------- /IV_images/IR2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR2.png -------------------------------------------------------------------------------- /IV_images/IR20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR20.png -------------------------------------------------------------------------------- /IV_images/IR21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR21.png -------------------------------------------------------------------------------- /IV_images/IR3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR3.png -------------------------------------------------------------------------------- /IV_images/IR4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR4.png -------------------------------------------------------------------------------- /IV_images/IR5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR5.png -------------------------------------------------------------------------------- /IV_images/IR6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR6.png -------------------------------------------------------------------------------- /IV_images/IR7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR7.png -------------------------------------------------------------------------------- /IV_images/IR8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR8.png -------------------------------------------------------------------------------- /IV_images/IR9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/IR9.png -------------------------------------------------------------------------------- /IV_images/VIS1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS1.jpg -------------------------------------------------------------------------------- /IV_images/VIS1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS1.png -------------------------------------------------------------------------------- /IV_images/VIS10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS10.png -------------------------------------------------------------------------------- /IV_images/VIS11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS11.png -------------------------------------------------------------------------------- /IV_images/VIS12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS12.png -------------------------------------------------------------------------------- /IV_images/VIS13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS13.png -------------------------------------------------------------------------------- /IV_images/VIS14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS14.png -------------------------------------------------------------------------------- /IV_images/VIS15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS15.png -------------------------------------------------------------------------------- /IV_images/VIS16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS16.png -------------------------------------------------------------------------------- /IV_images/VIS17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS17.png -------------------------------------------------------------------------------- /IV_images/VIS18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS18.png -------------------------------------------------------------------------------- /IV_images/VIS19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS19.png -------------------------------------------------------------------------------- /IV_images/VIS2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS2.png -------------------------------------------------------------------------------- /IV_images/VIS20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS20.png -------------------------------------------------------------------------------- /IV_images/VIS21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS21.png -------------------------------------------------------------------------------- /IV_images/VIS3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS3.jpg -------------------------------------------------------------------------------- /IV_images/VIS3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS3.png -------------------------------------------------------------------------------- /IV_images/VIS4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS4.jpg -------------------------------------------------------------------------------- /IV_images/VIS4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS4.png -------------------------------------------------------------------------------- /IV_images/VIS5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS5.jpg -------------------------------------------------------------------------------- /IV_images/VIS5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS5.png -------------------------------------------------------------------------------- /IV_images/VIS6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS6.png -------------------------------------------------------------------------------- /IV_images/VIS7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS7.png -------------------------------------------------------------------------------- /IV_images/VIS8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS8.png -------------------------------------------------------------------------------- /IV_images/VIS9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GrimReaperSam/imagefusion_pytorch/43b3ddac9141f0b1e1039f301bb3bb5373881c33/IV_images/VIS9.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Infrared and Visible Image Fusion using a Deep Learning Framework - Pytorch Implementation 2 | 3 | This is a pytorch implementation of Li H, Wu X J, Kittler J. Infrared and Visible Image Fusion using a Deep Learning Framework[J]. arXiv preprint arXiv:1804.06992, 2018.(https://arxiv.org/abs/1804.06992) 4 | 5 | ### Dataset 6 | 7 | There are pairs of visible and infrared images in the `IV_image` folder. 8 | You can find them, and others in the dataset used by the original paper here: [dataset](https://figshare.com/articles/dataset/TNO_Image_Fusion_Dataset/1008029) 9 | 10 | ### Requirements 11 | You will need the following tools to run this code: 12 | - [SPORCO](https://github.com/bwohlberg/sporco/) 13 | - [PyTorch](https://pytorch.org/) 14 | -------------------------------------------------------------------------------- /vggfusion.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sporco.util import tikhonov_filter 3 | 4 | import torch 5 | from torchvision.models.vgg import vgg19 6 | 7 | def lowpass(s, lda, npad): 8 | return tikhonov_filter(s, lda, npad) 9 | 10 | def c3(s): 11 | if s.ndim == 2: 12 | s3 = np.dstack([s, s, s]) 13 | else: 14 | s3 = s 15 | return np.rollaxis(s3, 2, 0)[None, :, :, :] 16 | 17 | def l1_features(out): 18 | h, w, d = out.shape 19 | A_temp = np.zeros((h+2, w+2)) 20 | 21 | l1_norm = np.sum(np.abs(out), axis=2) 22 | A_temp[1:h+1, 1:w+1] = l1_norm 23 | return A_temp 24 | 25 | def fusion_strategy(feat_a, feat_b, source_a, source_b, unit): 26 | 27 | m, n = feat_a.shape 28 | m1, n1 = source_a.shape[:2] 29 | weight_ave_temp1 = np.zeros((m1, n1)) 30 | weight_ave_temp2 = np.zeros((m1, n1)) 31 | 32 | for i in range(1, m): 33 | for j in range(1, n): 34 | A1 = feat_a[i-1:i+1, j-1:j+1].sum() / 9 35 | A2 = feat_b[i-1:i+1, j-1:j+1].sum() / 9 36 | 37 | weight_ave_temp1[(i-2)*unit+1:(i-1)*unit+1, (j-2)*unit+1:(j-1)*unit+1] = A1 / (A1+A2) 38 | weight_ave_temp2[(i-2)*unit+1:(i-1)*unit+1, (j-2)*unit+1:(j-1)*unit+1] = A2 / (A1+A2) 39 | 40 | if source_a.ndim == 3: 41 | weight_ave_temp1 = weight_ave_temp1[:, :, None] 42 | source_a_fuse = source_a * weight_ave_temp1 43 | if source_b.ndim == 3: 44 | weight_ave_temp2 = weight_ave_temp2[:, :, None] 45 | source_b_fuse = source_b * weight_ave_temp2 46 | 47 | if source_a.ndim == 3 or source_b.ndim == 3: 48 | gen = np.atleast_3d(source_a_fuse) + np.atleast_3d(source_b_fuse) 49 | else: 50 | gen = source_a_fuse + source_b_fuse 51 | 52 | return gen 53 | 54 | def get_activation(model, layer_numbers, input_image): 55 | outs = [] 56 | out = input_image 57 | for i in range(max(layer_numbers)+1): 58 | with torch.no_grad(): 59 | out = model.features[i](out) 60 | if i in layer_numbers: 61 | outs.append(np.rollaxis(out.detach().cpu().numpy()[0], 0, 3)) 62 | return outs 63 | 64 | def fuse(vis, ir, model=None): 65 | npad = 16 66 | lda = 5 67 | vis_low, vis_high = lowpass(vis.astype(np.float32)/255, lda, npad) 68 | ir_low, ir_high = lowpass(ir.astype(np.float32)/255, lda, npad) 69 | 70 | if model is None: 71 | model = vgg19(True) 72 | model.cuda().eval() 73 | relus = [2, 7, 12, 21] 74 | unit_relus = [1, 2, 4, 8] 75 | 76 | vis_in = torch.from_numpy(c3(vis_high)).cuda() 77 | ir_in = torch.from_numpy(c3(ir_high)).cuda() 78 | 79 | relus_vis = get_activation(model, relus, vis_in) 80 | relus_ir = get_activation(model, relus, ir_in) 81 | 82 | vis_feats = [l1_features(out) for out in relus_vis] 83 | ir_feats = [l1_features(out) for out in relus_ir] 84 | 85 | saliencies = [] 86 | saliency_max = None 87 | for idx in range(len(relus)): 88 | saliency_current = fusion_strategy(vis_feats[idx], ir_feats[idx], vis_high, ir_high, unit_relus[idx]) 89 | saliencies.append(saliency_current) 90 | 91 | if saliency_max is None: 92 | saliency_max = saliency_current 93 | else: 94 | saliency_max = np.maximum(saliency_max, saliency_current) 95 | 96 | if vis_low.ndim == 3 or ir_low.ndim == 3: 97 | low_fused = np.atleast_3d(vis_low) + np.atleast_3d(ir_low) 98 | else: 99 | low_fused = vis_low + ir_low 100 | low_fused = low_fused / 2 101 | high_fused = saliency_max 102 | return low_fused + high_fused 103 | 104 | --------------------------------------------------------------------------------