├── README.md ├── inputImages ├── img1.jpg ├── img10.jpg ├── img11.jpg ├── img12.jpg ├── img13.jpg ├── img14.jpg ├── img15.jpg ├── img16.jpg ├── img17.jpg ├── img18.jpg ├── img19.jpg ├── img2.jpg ├── img20.jpg ├── img21.jpg ├── img22.jpg ├── img3.jpg ├── img4.jpg ├── img5.jpg ├── img6.jpg ├── img7.jpg ├── img8.jpg └── img9.jpg ├── model.t7b ├── run.lua └── vis.lua /README.md: -------------------------------------------------------------------------------- 1 | # VisualBackProp 2 | VisualBackProp - visualization method for convolutional neural networks 3 | 4 | # Description 5 | Detailed description of the VisualBackProp can be found in: 6 | https://arxiv.org/abs/1611.05418 7 | 8 | Content: 9 | 10 | * inputImages - folder with input images 11 | * outputImages - folder for output images 12 | * vis.lua - implementation of VisualBackProp 13 | * run.lua - loads images, runs visualization method, saves images 14 | * model.t7b - model trained to predict steering wheel angle 15 | 16 | # Usage 17 | `mkdir outputImages` 18 | 19 | `qlua run.lua` 20 | 21 | Script generates set of 3 images for each input image. Each set contain: 22 | 23 | * Input image with visualization mask overlaid in red 24 | * Averaged feature maps 25 | * Intermediate masks 26 | -------------------------------------------------------------------------------- /inputImages/img1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img1.jpg -------------------------------------------------------------------------------- /inputImages/img10.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img10.jpg -------------------------------------------------------------------------------- /inputImages/img11.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img11.jpg -------------------------------------------------------------------------------- /inputImages/img12.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img12.jpg -------------------------------------------------------------------------------- /inputImages/img13.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img13.jpg -------------------------------------------------------------------------------- /inputImages/img14.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img14.jpg -------------------------------------------------------------------------------- /inputImages/img15.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img15.jpg -------------------------------------------------------------------------------- /inputImages/img16.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img16.jpg -------------------------------------------------------------------------------- /inputImages/img17.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img17.jpg -------------------------------------------------------------------------------- /inputImages/img18.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img18.jpg -------------------------------------------------------------------------------- /inputImages/img19.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img19.jpg -------------------------------------------------------------------------------- /inputImages/img2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img2.jpg -------------------------------------------------------------------------------- /inputImages/img20.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img20.jpg -------------------------------------------------------------------------------- /inputImages/img21.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img21.jpg -------------------------------------------------------------------------------- /inputImages/img22.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img22.jpg -------------------------------------------------------------------------------- /inputImages/img3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img3.jpg -------------------------------------------------------------------------------- /inputImages/img4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img4.jpg -------------------------------------------------------------------------------- /inputImages/img5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img5.jpg -------------------------------------------------------------------------------- /inputImages/img6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img6.jpg -------------------------------------------------------------------------------- /inputImages/img7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img7.jpg -------------------------------------------------------------------------------- /inputImages/img8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img8.jpg -------------------------------------------------------------------------------- /inputImages/img9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/inputImages/img9.jpg -------------------------------------------------------------------------------- /model.t7b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbojarski/VisualBackProp/6f35369e1e381ceb9861f7d5c6268279d25f3c0a/model.t7b -------------------------------------------------------------------------------- /run.lua: -------------------------------------------------------------------------------- 1 | require "torch" 2 | require "image" 3 | require "cudnn" 4 | 5 | dofile 'vis.lua' 6 | 7 | ------------------------------------------------------------------------------------ 8 | 9 | local outputImages = "outputImages/" 10 | local inputImages = "inputImages/" 11 | local imgFileName = "img" 12 | local outFileName = "out" 13 | local fmapFileName = "fmap" 14 | local maskFileName = "mask" 15 | local imgExt = ".jpg" 16 | 17 | local imgCnt = 22 18 | local imgCh = 1 19 | local imgH = 125 20 | local imgW = 640 21 | 22 | local imgBatch = torch.CudaTensor(imgCnt, imgCh, imgH, imgW) 23 | 24 | ------------------------------------------------------------------------------------ 25 | 26 | local function getImages(n, visMask, fMaps, fMapsM) 27 | local imgOut = torch.CudaTensor(3, imgH, imgW) 28 | local w = visMask:size(4) 29 | local h = visMask:size(3) 30 | local fMapsImg = torch.ones(#fMaps * h + (#fMaps - 1) * 2, w):cuda() 31 | local fMapsImgM = torch.ones(#fMaps * h + (#fMaps - 1) * 2, w):cuda() 32 | --normalize and scale averaged feature maps and intermediate visualization masks 33 | for i = 1, #fMaps do 34 | local min = fMaps[i][n][1]:min() 35 | local max = fMaps[i][n][1]:max() 36 | fMaps[i][n][1]:add(-min) 37 | fMaps[i][n][1]:div(max - min) 38 | local min = fMapsM[i][n][1]:min() 39 | local max = fMapsM[i][n][1]:max() 40 | fMapsM[i][n][1]:add(-min) 41 | fMapsM[i][n][1]:div(max - min) 42 | fMapsImg:narrow(1, 1 + (i - 1) * (h + 2), h): 43 | copy(image.scale(fMaps[i][n][1]:float(), w, h):cuda()) 44 | fMapsImgM:narrow(1, 1 + (i - 1) * (h + 2), h): 45 | copy(image.scale(fMapsM[i][n][1]:float(), w, h):cuda()) 46 | end 47 | --overlay visualization mask over the input images 48 | imgOut[1]:copy(imgBatch[n][1]):add(visMask[n]) 49 | imgOut[2]:copy(imgBatch[n][1]):add(-visMask[n]) 50 | imgOut[3]:copy(imgBatch[n][1]):add(-visMask[n]) 51 | imgOut:clamp(0, 1) 52 | return imgOut, fMapsImg, fMapsImgM 53 | end 54 | 55 | ------------------------------------------------------------------------------------ 56 | 57 | print("Loading model...") 58 | model = torch.load("model.t7b") 59 | 60 | print("Loading images...") 61 | for i = 1, 22 do 62 | imgBatch[i]:copy(image.load(inputImages .. imgFileName .. tostring(i) .. imgExt)) 63 | end 64 | 65 | print("Generating visualization masks...") 66 | local visMask, fMaps, fMapsM = getVisMask(model:get(1), imgBatch) 67 | 68 | print("Saving images...") 69 | for i = 1, 22 do 70 | local img1, img2, img3 = getImages(i, visMask, fMaps, fMapsM) 71 | image.save(outputImages..outFileName..tostring(i)..imgExt, img1) 72 | image.save(outputImages..fmapFileName..tostring(i)..imgExt, img2) 73 | image.save(outputImages..maskFileName..tostring(i)..imgExt, img3) 74 | end 75 | 76 | -------------------------------------------------------------------------------- /vis.lua: -------------------------------------------------------------------------------- 1 | require "torch" 2 | require "cutorch" 3 | require "nn" 4 | require "cunn" 5 | require "cudnn" 6 | 7 | ------------------------------------------------------------------------------------ 8 | 9 | function getVisMask(mod, idata) 10 | --do the forward pass through the feature extractor (convolutional layers) 11 | mod:forward(idata) 12 | local layersReLU = mod:findModules('cudnn.ReLU') 13 | local layersConv = mod:findModules('cudnn.SpatialConvolution') 14 | local mask = nil 15 | local sum = {} 16 | local sumUp = {} 17 | local fMaps = {} 18 | local fMapsMasked = {} 19 | --process feature maps 20 | for i = #layersReLU, 1, -1 do 21 | --sum all the feature maps at each level 22 | sum[i] = layersReLU[i].output:sum(2) 23 | --calculate the dimension of scaled up map 24 | local w = (layersReLU[i].output:size(3) - 1) * layersConv[i].dW + layersConv[i].kW 25 | local h = (layersReLU[i].output:size(4) - 1) * layersConv[i].dH + layersConv[i].kH 26 | fMaps[i] = sum[i]:clone() 27 | --pointwise multiplication 28 | if i < #layersReLU then 29 | sum[i]:cmul(sumUp[i + 1]) 30 | end 31 | --save intermediate mask 32 | fMapsMasked[i] = sum[i]:clone() 33 | --scale up intermediate mask using deconvolution 34 | if i > 1 then 35 | local adjw = layersReLU[i - 1].output:size(3) - w 36 | local adjh = layersReLU[i - 1].output:size(4) - h 37 | local mmUp = nn.SpatialFullConvolution(1, 1, 38 | layersConv[i].kW, 39 | layersConv[i].kH, 40 | layersConv[i].dW, 41 | layersConv[i].dH, 42 | 0, 0, adjh, adjw) 43 | mmUp:cuda() 44 | mmUp:parameters()[1]:fill(1) 45 | mmUp:parameters()[2]:fill(0) 46 | sumUp[i] = mmUp:forward(sum[i]) 47 | else 48 | local adjw = idata:size(3) - w 49 | local adjh = idata:size(4) - h 50 | local mmUp = nn.SpatialFullConvolution(1, 1, 51 | layersConv[i].kW, 52 | layersConv[i].kH, 53 | layersConv[i].dW, 54 | layersConv[i].dH, 55 | 0, 0, adjh, adjw) 56 | mmUp:cuda() 57 | mmUp:parameters()[1]:fill(1) 58 | mmUp:parameters()[2]:fill(0) 59 | sumUp[i] = mmUp:forward(sum[i]) 60 | end 61 | end 62 | --assign output - visualization mask 63 | local out = sumUp[1] 64 | --normalize mask to range 0-1 65 | local omin = torch.min(out, 3):min(4):mul(-1) 66 | local omax = torch.max(out, 3):max(4):add(omin) 67 | out:add(torch.expand(omin:reshape(idata:size(1), 1, 1, 1), idata:size(1), 68 | idata:size(2), idata:size(3), idata:size(4))) 69 | out:cdiv(torch.expand(omax:reshape(idata:size(1), 1, 1, 1), idata:size(1), 70 | idata:size(2), idata:size(3), idata:size(4))) 71 | --return visualization mask, averaged feature maps, and intermediate masks 72 | return out, fMaps, fMapsMasked 73 | end 74 | 75 | --------------------------------------------------------------------------------