├── dev ├── imageSource.md ├── ext.lua ├── dispTop10.lua ├── getTop10.lua ├── saliencyReadme.md ├── spatialPredictionMap.lua ├── getModel.lua ├── pub1.lua ├── preditError.lua ├── person-spatialTest.lua ├── getSpatialModel.lua ├── pub2.lua ├── saliencyMap.lua ├── profile.txt ├── saliencyCam.lua └── profile-wide.lua ├── .gitignore ├── getData-public.sh ├── README.md └── getData-private.sh /dev/imageSource.md: -------------------------------------------------------------------------------- 1 | # List of images' sources 2 | 3 | - peep01.jpg: http://www.sanfranciscoize.com/2012/02/madrid-different-ways-to-share-space.html 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Files 2 | *.sw* 3 | *.png 4 | 5 | # This project untracked files 6 | /net 7 | /data 8 | /imgs 9 | 10 | # Previous project garbage 11 | dev/README.md 12 | dev/TopAccuracy.lua 13 | dev/camFind.lua 14 | dev/convert-conv-linear-to-spatial.lua 15 | dev/convertGPUmodelCPU.lua 16 | dev/demo.lua 17 | dev/init-loggers.lua 18 | dev/models.lua 19 | dev/run-gpu.lua 20 | dev/train-and-test-gpu.lua 21 | dev/usefulFunctions.lua 22 | -------------------------------------------------------------------------------- /getData-public.sh: -------------------------------------------------------------------------------- 1 | # Get data from server 2 | 3 | path='https://engineering.purdue.edu/elab/smartEYE-data/' 4 | 5 | echo '' 6 | echo 'Do you wish to download the network?' 7 | select yn in 'Yes' 'No'; do 8 | case $yn in 9 | Yes ) mkdir -p net 10 | wget $path'/net/17cate9filter.tar.gz' . 11 | tar -xzf 17cate9filter.tar.gz 12 | mv 17cate9filter net/ 13 | rm *.tar.gz 14 | break;; 15 | No ) break;; 16 | esac 17 | done 18 | -------------------------------------------------------------------------------- /dev/ext.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Get ext-ernal variables 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Jun 15 5 | -------------------------------------------------------------------------------- 6 | 7 | -- Loading images and classes, building reverse classes 8 | top10 = torch.load('../data/17cate9filter/Top10TestData.t7') 9 | classes = torch.load('../net/17cate9filter/classes.t7') 10 | revClas = {}; for a,b in ipairs(classes) do revClas[b] = a end 11 | -------------------------------------------------------------------------------- /dev/dispTop10.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Display (or save) the top10 test images 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'image' 8 | require 'ext' 9 | 10 | for label, cls in pairs(top10) do 11 | -- Display 12 | image.display{image = cls.image, legend = label, nrow = 5} 13 | 14 | -- Save 15 | -- image.saveJPG(label .. '.jpg',image.toDisplayTensor{ 16 | -- input = cls.image, legend = label, nrow = 5 17 | -- }) 18 | end 19 | -------------------------------------------------------------------------------- /dev/getTop10.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Get top 10 predictions per class 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14 5 | -------------------------------------------------------------------------------- 6 | 7 | data = torch.load('../data/17cate9filter/TestDataset.t7') 8 | cls = data.classes 9 | data.classes = nil 10 | top10data = {} 11 | 12 | for _, c in pairs(cls) do 13 | clsE = torch.Tensor(data[c].error) 14 | e, idx = clsE:sort() 15 | for i = 1, 10 do 16 | if i == 1 then 17 | top10data[c] = {} 18 | top10data[c].image = {} 19 | top10data[c].error = {} 20 | end 21 | table.insert(top10data[c].image, data[c].image[idx[i]]) 22 | table.insert(top10data[c].error, e[i]) 23 | end 24 | end 25 | 26 | torch.save('../data/17cate9filter/Top10TestData.t7', top10data) 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # smartEYE 2 | 3 | This is an implemetation of the [VADNN](http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7086900) article. 4 | This collection of scripts for [Torch7](http://torch.ch/) can compute both *bottom-up* and *top-down* visual attention saliency maps. 5 | 6 | 7 | ## You need to fetch some data 8 | 9 | You need to fetch the trained model, at least, to be able to run the live demo. 10 | 11 | ```bash 12 | ./getData-public.sh 13 | ``` 14 | 15 | 16 | ## Dependency 17 | 18 | If you want to use an USB camera, you will need to install the `camera` package 19 | 20 | ```bash 21 | luarocks install camera 22 | ``` 23 | 24 | 25 | ## How to run a live person top-down saliency demo 26 | 27 | There are available three different methods with which the final top-down saliency map is displayed. 28 | Each method can be selected by `--mode #`, where `#` can be 1, 2 or 3. 29 | The source can be a USB webcam, which is the default option, or an Ethernet camera, which can be chosen with `--eth`. 30 | For example, we can run on USB, mode 3 by typing 31 | 32 | ```bash 33 | cd dev 34 | qlua saliencyCam.lua --mode 3 35 | ``` 36 | 37 | or, we could run on Ethernet and using method 2 with 38 | 39 | ```bash 40 | qlua saliencyCam.lua --mode 2 --eth 41 | ``` 42 | -------------------------------------------------------------------------------- /getData-private.sh: -------------------------------------------------------------------------------- 1 | # Get data from server 2 | 3 | path=${MYELAB?'Need to set MYELAB'}':~/elabshare/users/atcold/Git-data/smartEYE' 4 | 5 | echo '' 6 | echo 'Do you wish to download the network?' 7 | select yn in 'Yes' 'No'; do 8 | case $yn in 9 | Yes ) scp -r $path'/net' .; break;; 10 | No ) break;; 11 | esac 12 | done 13 | 14 | echo '' 15 | echo 'Do you wish to download sample images?' 16 | select yn in 'Yes' 'No'; do 17 | case $yn in 18 | Yes ) scp -r $path'/imgs' .; break;; 19 | No ) break;; 20 | esac 21 | done 22 | 23 | echo '' 24 | echo 'Do you wish to download top10 testing data?' 25 | select yn in 'Yes' 'No'; do 26 | case $yn in 27 | Yes ) 28 | mkdir -p data/17cate9filter 29 | scp -r $path'/data/17cate9filter/Top10TestData.t7' data/17cate9filter 30 | break;; 31 | No ) break;; 32 | esac 33 | done 34 | 35 | echo '' 36 | echo 'Do you wish to download the WHOLE testing-set data?' 37 | select yn in 'Yes' 'No'; do 38 | case $yn in 39 | Yes ) 40 | mkdir -p data/17cate9filter 41 | scp -r $path'/data/17cate9filter/TestDataset.t7' data/17cate9filter 42 | break;; 43 | No ) break;; 44 | esac 45 | done 46 | -------------------------------------------------------------------------------- /dev/saliencyReadme.md: -------------------------------------------------------------------------------- 1 | # Saliency README file 2 | 3 | This file contains some information about the developing of the saliency project code 4 | 5 | ## Files 6 | 7 | ### `predictError.lua` 8 | 9 | Reproduces the *testing* step of the *network training* procedure (just to be sure I am not doing weird stuff). 10 | 11 | ### `TestDataset.t7` 12 | 13 | The whole *test dataset* has been extracted and saved into a Lua table `{}` under the entry with the labels's name. The structure is the following: each entry (`barcode`, `bootle`, `dog`, etc...) is an *array* of `{error, image}` table. `error` is the output of the *cost function*, i.e. the `LogSoftMax()` of the *correct output* and `image` contains a `float` version of the *input image*. 14 | 15 | This file contains also a `classes` entry which contains an *array* of the available classes: each *index* correspond to the correct *class index*. 16 | 17 | ### `Top10TestData.t7` 18 | 19 | The top-10 predictions per each testing class have been stored into this file. The structure is alike the `TestDataset.t7` file, but the `classes` entry is missing in this case. 20 | 21 | ### `getTop10.lua` 22 | 23 | Extracts `Top10TestData.t7` from `TestDataset.t7`. 24 | 25 | ### `dispTop10.lua` 26 | 27 | Displays all (17) categories top-10 test images and corresponding label. 28 | -------------------------------------------------------------------------------- /dev/spatialPredictionMap.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Generate spatial prediction map 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'image' 8 | require 'gnuplot' 9 | require 'imgraph' 10 | require 'getSpatialModel' 11 | 12 | img = image.load('../imgs/peep01.jpg') 13 | model:forward(img:cuda()) 14 | --gnuplot.hist(model.output[revClas.jacket]) 15 | psProb = torch.CudaTensor(#model.output) 16 | 17 | -- This has to be rewritten with FFI 18 | for i = 1, model.output:size(2) do 19 | for j = 1, model.output:size(3) do 20 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 21 | end 22 | end 23 | 24 | --gnuplot.hist(rescaled[revClas.jacket]) 25 | 26 | -- image.display{ 27 | -- image = rescaled[revClas.jacket]:float(), 28 | -- zoom = 15, 29 | -- legend = 'Spatial person with jaket' 30 | -- } 31 | 32 | colorMap = imgraph.colorize(psProb[2]:float()*255,image.jetColormap(256):float()) 33 | image.display{zoom = 0.4, nrow = 2, image = { 34 | img, 35 | image.scale(colorMap, img:size(3), img:size(2)), 36 | image.scale(colorMap, img:size(3), img:size(2), 'simple'), 37 | img*0 38 | }} 39 | -------------------------------------------------------------------------------- /dev/getModel.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Get model cleaned and ready 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'cunn' 8 | local GPUcount = cutorch.getDeviceCount() 9 | if GPUcount > 1 then 10 | io.write('Choose free GPU (1-' .. GPUcount .. ') ') 11 | cutorch.setDevice(io.read()) 12 | end 13 | 14 | oldModel = torch.load('../net/17cate9filter/model-127.net') 15 | 16 | -- Building new MLP 17 | oldMLP = oldModel.modules[2] 18 | MLP = nn.Sequential():cuda() 19 | MLP:add(oldMLP.modules[1]) -- Reshape 20 | -- 2 is Dropout 21 | MLP:add(oldMLP.modules[3]) -- Linear 22 | MLP:add(nn.ReLU():cuda()) -- 4 23 | -- 5 is Dropout 24 | MLP:add(oldMLP.modules[6]) -- Linear 25 | MLP:add(nn.ReLU():cuda()) -- ReLU 26 | -- 8 is Dropout 27 | MLP:add(oldMLP.modules[9]) -- Linear 28 | 29 | -- Creating new model 30 | model = nn.Sequential():cuda() 31 | model:add(oldModel.modules[1]) 32 | model:add(MLP) 33 | 34 | -- Removing custom updateGradInput() 35 | model.modules[1].modules[1].updateGradInput = nil 36 | 37 | -- Creating SoftMax SM module 38 | SM = nn.SoftMax():cuda() 39 | LSM = nn.LogSoftMax():cuda() 40 | LSMf = nn.LogSoftMax():float() 41 | loss = nn.ClassNLLCriterion():cuda() 42 | 43 | -- Cleaning 44 | oldModel = nil 45 | oldMLP = nil 46 | collectgarbage() 47 | -------------------------------------------------------------------------------- /dev/pub1.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Generate publication "figure 1" 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Feb 15 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'image' 8 | require 'getModel' 9 | require 'ext' 10 | 11 | -- Function definition --------------------------------------------------------- 12 | clip = function (input) 13 | input[input:gt(input:max()/3)] = input:max()/3 14 | input[input:lt(input:min()/3)] = input:min()/3 15 | return input 16 | end 17 | 18 | -- Main program ---------------------------------------------------------------- 19 | nbs = {4, 10, 1, 6} 20 | label = {'phone', 'plant', 'laptop', 'laptop'} 21 | img = {} 22 | gradInput = {} 23 | 24 | for i, n in ipairs(nbs) do 25 | img[i] = top10[label[i]].image[n] 26 | gradLoss = loss:updateGradInput(model:forward(img[i]:cuda()), revClas[label[i]]) 27 | gradInput[i] = model:updateGradInput(img[i]:cuda(), gradLoss):float() 28 | end 29 | 30 | --image.display{image = { 31 | pub1 = image.toDisplayTensor{input = { 32 | img[1], clip(gradInput[1]:clone():abs():max(1):repeatTensor(3,1,1)), 33 | img[2], clip(gradInput[2]:clone():abs():max(1):repeatTensor(3,1,1)), 34 | img[3], clip(gradInput[3]:clone():abs():max(1):repeatTensor(3,1,1)), 35 | img[4], clip(gradInput[4]:clone():abs():max(1):repeatTensor(3,1,1)), 36 | }, scaleeach = true, nrow = 4, zoom = 2, padding = 2 37 | } 38 | image.save('pub1.png',pub1) 39 | -------------------------------------------------------------------------------- /dev/preditError.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Predict error for a specific image 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'cunn' 8 | require 'ext' 9 | 10 | oldModel = torch.load('../net/17cate9filter/model-127.net') 11 | 12 | -- Building new MLP 13 | oldMLP = oldModel.modules[2] 14 | MLP = nn.Sequential():cuda() 15 | MLP:add(oldMLP.modules[1]) -- Reshape 16 | -- 2 is Dropout 17 | MLP:add(oldMLP.modules[3]) -- Linear 18 | MLP:add(nn.ReLU():cuda()) -- 4 19 | -- 5 is Dropout 20 | MLP:add(oldMLP.modules[6]) -- Linear 21 | MLP:add(nn.ReLU():cuda()) -- ReLU 22 | -- 8 is Dropout 23 | MLP:add(oldMLP.modules[9]) -- Linear 24 | --MLP:add(nn:SoftMax():cuda()) 25 | MLP:add(nn:LogSoftMax():cuda()) 26 | 27 | model = nn.Sequential():cuda() 28 | model:add(oldModel.modules[1]) 29 | model:add(MLP) 30 | 31 | -- Disabling on oldModel 32 | oldMLP.modules[2].train = false 33 | oldMLP.modules[5].train = false 34 | oldMLP.modules[8].train = false 35 | LSM = nn.LogSoftMax():float() -- explains the discrepancy in the predictions values 36 | 37 | crit = nn.ClassNLLCriterion():cuda() 38 | 39 | for label, class in pairs(top10) do 40 | print('Prediction for ' .. label) 41 | for n, img in ipairs(class.image) do 42 | newE = crit:forward(model:forward(img:cuda()), revClas[label]) 43 | oldE = crit:forward(LSM:forward(oldModel:forward(img:cuda()):float()), revClas[label]) 44 | print(string.format( 45 | 'New pred E: %.5f, old pred E: %.5f, saved: %.5f', 46 | newE, oldE, class.error[n] 47 | )) 48 | end 49 | io.write("Press enter for next category, 'q' for quitting... ") 50 | if io.read() == 'q' then break end 51 | end 52 | 53 | -------------------------------------------------------------------------------- /dev/person-spatialTest.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Generate several spatial saliency map for a person test image 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14, Jan/Feb 15 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'image' 8 | require 'getSpatialModel' 9 | 10 | -- Function definition --------------------------------------------------------- 11 | image.fit = function (inputImage) 12 | return image.scale(inputImage:float(), img:size(3), img:size(2), 'simple') 13 | end 14 | gradManipulation = function (grad, th) 15 | return grad:float():abs():max(1):repeatTensor(3,1,1) 16 | end 17 | 18 | -- Main program ---------------------------------------------------------------- 19 | img = image.load('../imgs/peep01.jpg') 20 | model:forward(img:cuda()) 21 | 22 | -- Estimating pseudo-probability, this has to be rewritten with FFI 23 | psProb = torch.CudaTensor(#model.output) 24 | for i = 1, model.output:size(2) do 25 | for j = 1, model.output:size(3) do 26 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 27 | end 28 | end 29 | 30 | -- Top-down saliency 31 | target = psProb:clone():zero() 32 | target[2] = psProb[2]:clone():mul(-1) 33 | gradInput = model:updateGradInput(img:cuda(), target) 34 | 35 | -- Visulaisation 36 | image.display{image = gradInput:float(), zoom = 0.4, 37 | --pub2 = image.toDisplayTensor{ padding = 8, 38 | nrow = 2, scaleeach = true, 39 | legend = '"person" pseudo probability and top-down saliency map', 40 | -- input = { 41 | image = { 42 | img:float(), 43 | image.fit(image.y2jet(psProb[2]*255+1)), 44 | gradInput:float(), 45 | gradManipulation(gradInput, 0) 46 | } 47 | } 48 | --image.save('pub2.png',image.scale(pub2,pub2:size(3)/2,pub2:size(2)/2)) 49 | -------------------------------------------------------------------------------- /dev/getSpatialModel.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Spatialise a MLP 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14, Feb 15 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'sys' 8 | 9 | if sys.execute('uname -a'):find('tegra') then 10 | require 'cudnn' 11 | model = torch.load('../net/17cate9filter/model-127.net.tegra'):cuda() 12 | SM = cudnn.SoftMax():cuda() 13 | else 14 | 15 | -- Getting model and renaming 16 | require 'getModel' 17 | 18 | oldModel = model 19 | oldMLP = oldModel.modules[2] 20 | 21 | -- Defining new MLP 22 | newMLP = nn.Sequential() 23 | newMLP:add(nn.SpatialConvolutionMM( 32, 2048, 6, 6)) 24 | newMLP:add(nn.ReLU()) 25 | newMLP:add(nn.SpatialConvolutionMM(2048, 2048, 1, 1)) 26 | newMLP:add(nn.ReLU()) 27 | newMLP:add(nn.SpatialConvolutionMM(2048, 17, 1, 1)) 28 | 29 | -- Sending MLP to Cuda 30 | newMLP:cuda() 31 | 32 | -- Copying over the weights from oldMLP 33 | -- Kernels 34 | newMLP.modules[1].weight:copy(oldMLP.modules[2].weight) 35 | newMLP.modules[3].weight:copy(oldMLP.modules[4].weight) 36 | newMLP.modules[5].weight:copy(oldMLP.modules[6].weight) 37 | -- Bias 38 | newMLP.modules[1].bias:copy(oldMLP.modules[2].bias) 39 | newMLP.modules[3].bias:copy(oldMLP.modules[4].bias) 40 | newMLP.modules[5].bias:copy(oldMLP.modules[6].bias) 41 | 42 | -- Creating new final model WITH preprocessing 43 | model = nn.Sequential():cuda() 44 | model:add(nn.AddConstant(-0.4124)):cuda() 45 | model:add(nn.MulConstant(1/0.2805)):cuda() 46 | model:add(oldModel.modules[1]) 47 | model:add(newMLP) 48 | 49 | -- Cleaning 50 | oldModel = nil 51 | oldMLP = nil 52 | collectgarbage() 53 | 54 | end 55 | 56 | -- Creating a SpatialSoftMax function 57 | nn.SpatialSoftMax = function (spatialLogit) 58 | return SM( 59 | spatialLogit 60 | :reshape(spatialLogit:size(1), spatialLogit:size(2) * spatialLogit:size(3)) 61 | :t():contiguous() 62 | ):t():reshape(spatialLogit:size()) 63 | end 64 | -------------------------------------------------------------------------------- /dev/pub2.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Generate publication "figure 2" 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14, Jan/Feb 15 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'image' 8 | require 'getSpatialModel' 9 | 10 | -- Function definition --------------------------------------------------------- 11 | image.fit = function (inputImage) 12 | return image.scale(inputImage:float(), img:size(3)/4, img:size(2)/4, 'simple') 13 | end 14 | 15 | image.scaleDown = function (inputImage) 16 | return image.scale(inputImage:float(), img:size(3)/4, img:size(2)/4) 17 | end 18 | 19 | clip = function (input) 20 | input[input:gt(input:max()/4)] = input:max()/4 21 | input[input:lt(input:min()/4)] = input:min()/4 22 | return input 23 | end 24 | 25 | gradManipulation = function (grad) 26 | out = grad:float():abs():max(1):repeatTensor(3,1,1) 27 | return clip(out) 28 | end 29 | 30 | -- Main program ---------------------------------------------------------------- 31 | img = image.load('../imgs/peep01.jpg') 32 | model:forward(img:cuda()) 33 | 34 | -- Estimating pseudo-probability, this has to be rewritten with FFI 35 | psProb = torch.CudaTensor(#model.output) 36 | for i = 1, model.output:size(2) do 37 | for j = 1, model.output:size(3) do 38 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 39 | end 40 | end 41 | 42 | -- Top-down saliency 43 | person = 2 44 | target = psProb:clone():zero() 45 | target[person] = psProb[person]:clone():mul(-1) 46 | gradInput = model:updateGradInput(img:cuda(), target) 47 | 48 | -- Visulaisation 49 | --image.display{image = gradInput:float(), zoom = 0.4, 50 | pub2 = image.toDisplayTensor{ padding = 2, 51 | nrow = 2, scaleeach = true, 52 | legend = '"person" pseudo probability and top-down saliency map', 53 | input = { 54 | -- image = { 55 | image.scaleDown(img:float()), 56 | image.fit(image.y2jet(psProb[2]*255+1)), 57 | image.scaleDown(clip(gradInput:float())), 58 | image.scaleDown(gradManipulation(gradInput)), 59 | } 60 | } 61 | image.save('pub2.png',pub2) 62 | -------------------------------------------------------------------------------- /dev/saliencyMap.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Generate saliency map 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Nov 14 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'image' 8 | require 'getModel' 9 | require 'ext' 10 | --require 'getSpatialModel' -- remove preprocessing for fairness 11 | 12 | io.write('[V]iew or [P]rofile? (V/P) ') 13 | userInput = io.read() 14 | if userInput == 'V' or userInput == 'v' then 15 | for label, class in pairs(top10) do 16 | io.write('Processing ' .. label); io.flush() 17 | for n, img in ipairs(class.image) do 18 | gradLoss = loss:updateGradInput(model:forward(img:cuda()), revClas[label]) 19 | gradInput = model:updateGradInput(img:cuda(), gradLoss) 20 | image.display{ 21 | image = { 22 | img, 23 | gradInput, 24 | gradInput:clone():max(1):repeatTensor(3,1,1), 25 | gradInput:clone():abs():max(1):repeatTensor(3,1,1) 26 | }, scaleeach = true, nrow = 2, zoom = 2, legend = label..'['..n..']' 27 | } 28 | end 29 | io.write(". Press enter for next category, 'q' for quitting... ") 30 | if io.read() == 'q' then break end 31 | end 32 | else 33 | print('Profiling...') 34 | count = 0 35 | timer = torch.Timer() 36 | for i = 1, 10 do 37 | for label, class in pairs(top10) do 38 | for n, img in ipairs(class.image) do 39 | gradLoss = loss:updateGradInput(model:forward(img:cuda()), revClas[label]) 40 | --gradLoss = model:forward(img:cuda()) -- if getSpatialModel 41 | count = count + 1 42 | end 43 | end 44 | end 45 | cutorch.synchronize() 46 | forwardTime = timer:time().real / count 47 | timer:reset() 48 | for i = 1, 10 do 49 | for label, class in pairs(top10) do 50 | for n, img in ipairs(class.image) do 51 | gradLoss = loss:updateGradInput(model:forward(img:cuda()), revClas[label]) 52 | --gradLoss = model:forward(img:cuda()) -- if getSpatialModel 53 | gradInput = model:updateGradInput(img:cuda(), gradLoss) 54 | end 55 | end 56 | end 57 | cutorch.synchronize() 58 | forwardAndGradTime = timer:time().real / count 59 | print('Forward time / sample [ms]: ', 1e3 * forwardTime) 60 | print('Forward and grad time / sample [ms]: ', 1e3 * forwardAndGradTime) 61 | print('Just grad time / sample [ms]: ', 1e3 * (forwardAndGradTime - forwardTime)) 62 | end 63 | -------------------------------------------------------------------------------- /dev/profile.txt: -------------------------------------------------------------------------------- 1 | On GPU3, 13 Feb 15 2 | -------------------------------------------------------------------------------- 3 | Eye-size (224x224) 4 | 5 | Forward time / sample [ms]: 1.7327628416174 6 | Forward and grad time / sample [ms]: 3.6317140915815 7 | Just grad time / sample [ms]: 1.8989512499641 8 | -------------------------------------------------------------------------------- 9 | Eye-size, no SoftMax(), no loss() 10 | 11 | Forward time / sample [ms]: 1.5146970748901 12 | Forward and grad time / sample [ms]: 3.175585269928 13 | Just grad time / sample [ms]: 1.6608881950378 14 | -------------------------------------------------------------------------------- 15 | Eye-size, no SoftMax(), spatial, no stat 16 | 17 | Forward time / sample [ms]: 2.0737600326538 18 | Forward and grad time / sample [ms]: 4.0523791313171 19 | Just grad time / sample [ms]: 1.9786190986633 20 | -------------------------------------------------------------------------------- 21 | Profiling Eye-size, no SoftMax(), spatial 22 | 23 | Forward time / sample [ms]: 2.1075391769409 24 | Forward and grad time / sample [ms]: 4.1152501106262 25 | Just grad time / sample [ms]: 2.0077109336853 26 | -------------------------------------------------------------------------------- 27 | Profiling 1067x1600, no SoftMax(), spatial 28 | 29 | Forward time / sample [ms]: 36.289842128754 30 | Forward and grad time / sample [ms]: 78.196549415588 31 | Just grad time / sample [ms]: 41.906707286835 32 | -------------------------------------------------------------------------------- 33 | Profiling Eye-size, spatial, bottom-up 34 | 35 | Forward time / sample [ms]: 2.1483781337738 36 | Forward and grad time / sample [ms]: 4.1458990573883 37 | Just grad time / sample [ms]: 1.9975209236145 38 | -------------------------------------------------------------------------------- 39 | Profiling 1067x1600, spatial, bottom-up 40 | 41 | Forward time / sample [ms]: 96.076610088348 42 | Forward and grad time / sample [ms]: 138.7956905365 43 | Just grad time / sample [ms]: 42.719080448151 44 | -------------------------------------------------------------------------------- 45 | Profiling Eye-size, spatial, top-down 46 | 47 | Forward time / sample [ms]: 2.1514868736267 48 | Forward and grad time / sample [ms]: 4.1713001728058 49 | Just grad time / sample [ms]: 2.0198132991791 50 | -------------------------------------------------------------------------------- 51 | Profiling 1067x1600, spatial, top-down 52 | 53 | Forward time / sample [ms]: 95.385000705719 54 | Forward and grad time / sample [ms]: 138.04469823837 55 | Just grad time / sample [ms]: 42.659697532654 56 | -------------------------------------------------------------------------------- 57 | -------------------------------------------------------------------------------- /dev/saliencyCam.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Generate saliency map from camera image (demo) 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Jan 15 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'getSpatialModel' 8 | require 'pl' 9 | 10 | -- Options --------------------------------------------------------------------- 11 | opt = lapp([[ 12 | --camRes (default ok) Camera resolution (nHD|VGA|FWVGA|ok|HD|FHD) 13 | --eth Ethernet camera 14 | --mode (default 2) Different mode 15 | ]]) 16 | 17 | -- Switch input sources 18 | res = { 19 | nHD = {w = 640, h = 360}, 20 | VGA = {w = 640, h = 480}, 21 | FWVGA = {w = 854, h = 480}, 22 | ok = {w = 1024, h = 576}, 23 | HD = {w = 1280, h = 720}, 24 | FHD = {w = 1920, h = 1080}, 25 | } 26 | 27 | if opt.eth then 28 | video = require 'libvideo_decoder' 29 | status, height, width, length = video.init('http://169.254.80.245:8081', 'mjpeg') 30 | dst = torch.ByteTensor(3, height, width) 31 | cam = {} 32 | function cam:forward() 33 | video.frame_rgb(dst) 34 | return dst:float():mul(1/255) 35 | end 36 | else 37 | require 'camera' 38 | cam = image.Camera.new{ 39 | width = res[opt.camRes].w, 40 | height = res[opt.camRes].h 41 | } 42 | end 43 | 44 | -- Auxiliary functions --------------------------------------------------------- 45 | image.fit = function (inputImage) 46 | return image.scale(inputImage:float(), img:size(3), img:size(2), 'simple') 47 | end 48 | mask = function (inputImage, mask, th) 49 | local temp 50 | if opt.mode == 1 then 51 | temp = mask:gt(th):float()[1]--:mul(.8):add(.2) 52 | temp = image.dilate(image.erode(temp, image.gaussian(10):gt(.2):float()), image.gaussian(20):gt(.2):float()) 53 | temp = temp:mul(.8):add(.2):repeatTensor(3,1,1) 54 | elseif opt.mode == 2 then 55 | mask = mask[1] 56 | k = image.gaussian{size = 7, normalize = true}:float() 57 | mask = image.convolve(mask, k, 'same'):repeatTensor(3,1,1) 58 | temp = mask:gt(th):float():mul(.8):add(.2) 59 | elseif opt.mode == 3 then 60 | temp = mask:gt(th):float():mul(.8):add(.2) 61 | end 62 | 63 | return inputImage:clone():cmul(temp) 64 | end 65 | gradManipulation = function (grad, th) 66 | return grad:float():abs():max(1):repeatTensor(3,1,1) 67 | end 68 | 69 | layout = opt.eth and 1 or 2 70 | 71 | function printLabels() 72 | if opt.eth then 73 | win.painter:gbegin() 74 | winSize = win.window.frameSize:totable() 75 | win.painter:setcolor(1,1,1) 76 | win.painter:setfontsize(30) 77 | win.painter:moveto(20,40+winSize.height/2) 78 | win.painter:show('saliency') 79 | win.painter:moveto(20,40+winSize.height/4*3) 80 | win.painter:show('saliency fine grain') 81 | win.painter:moveto(20,40+winSize.height/4) 82 | win.painter:show('heat-map') 83 | win.painter:setcolor(0,0,0) 84 | win.painter:moveto(20,40) 85 | win.painter:show('Camera input') 86 | win.painter:gend() 87 | else 88 | win.painter:gbegin() 89 | winSize = win.window.frameSize:totable() 90 | win.painter:setcolor(1,1,1) 91 | win.painter:setfontsize(30) 92 | win.painter:moveto(20,40+winSize.height/2) 93 | win.painter:show('saliency') 94 | win.painter:moveto(20+winSize.width/2,40+winSize.height/2) 95 | win.painter:show('saliency fine grain') 96 | win.painter:moveto(20+winSize.width/2,40) 97 | win.painter:show('heat-map') 98 | win.painter:setcolor(0,0,0) 99 | win.painter:moveto(20,40) 100 | win.painter:show('Camera input') 101 | win.painter:gend() 102 | end 103 | end 104 | 105 | run = function () 106 | 107 | img = cam:forward():cuda() 108 | model:forward(img:cuda()) 109 | 110 | -- Estimating pseudo-probability 111 | psProb = nn.SpatialSoftMax(model.output) 112 | 113 | colorMap = image.y2jet(psProb[2]:float()*255 + 1) 114 | 115 | def = psProb:clone():zero() 116 | def[2] = psProb[2]:clone():mul(-1) 117 | gradInput = model:updateGradInput(img:cuda(), def) 118 | -- image.display{image = gradInput:float(), zoom = 0.5} 119 | win = image.display{ 120 | zoom = 1, nrow = layout, scaleeach = true, win = win, min = 0, max = 1, 121 | legend = '"person" pseudo probability and top-down saliency map', 122 | image = { 123 | img:float(), 124 | image.fit(colorMap), 125 | mask(img:float(), gradManipulation(gradInput, 0), .1), 126 | --gradManipulation(gradInput, 0), 127 | gradManipulation(gradInput, 0), 128 | } 129 | } 130 | 131 | printLabels() 132 | 133 | return win.window.visible 134 | 135 | end 136 | 137 | -- Main program ---------------------------------------------------------------- 138 | running = true 139 | while running do running = run() end 140 | -------------------------------------------------------------------------------- /dev/profile-wide.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | -- Profiling bigger-than-eye algorithm 3 | -------------------------------------------------------------------------------- 4 | -- Alfredo Canziani, Feb 15 5 | -------------------------------------------------------------------------------- 6 | 7 | require 'image' 8 | require 'getSpatialModel' 9 | require 'ext' 10 | 11 | -- Profiling ------------------------------------------------------------------- 12 | 13 | dash = '--------------------------------------------------------------------------------\n' 14 | 15 | -------------------------------------------------------------------------------- 16 | -------------------------------------------------------------------------------- 17 | 18 | -- Eye-size, no SoftMax(), spatial 19 | print(dash..'Profiling eye-size, no SoftMax(), spatial\n') 20 | img = top10.phone.image[1] 21 | count = 1000 22 | collectgarbage() 23 | timer = torch.Timer() 24 | for i = 1, count do 25 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 26 | end 27 | cutorch.synchronize() 28 | forwardTime = timer:time().real / count 29 | collectgarbage() 30 | timer:reset() 31 | for i = 1, count do 32 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 33 | gradInput = model:updateGradInput(img:cuda(), gradLoss) 34 | end 35 | cutorch.synchronize() 36 | forwardAndGradTime = timer:time().real / count 37 | print('Forward time / sample [ms]: ', 1e3 * forwardTime) 38 | print('Forward and grad time / sample [ms]: ', 1e3 * forwardAndGradTime) 39 | print('Just grad time / sample [ms]: ', 1e3 * (forwardAndGradTime - forwardTime)) 40 | 41 | -- 1067x1600, no SoftMax(), spatial 42 | print(dash..'Profiling 1067x1600, no SoftMax(), spatial\n') 43 | img = image.load('../imgs/peep01.jpg') 44 | count = 100 45 | collectgarbage() 46 | timer:reset() 47 | for i = 1, count do 48 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 49 | end 50 | cutorch.synchronize() 51 | forwardTime = timer:time().real / count 52 | collectgarbage() 53 | timer:reset() 54 | for i = 1, count do 55 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 56 | gradInput = model:updateGradInput(img:cuda(), gradLoss) 57 | end 58 | cutorch.synchronize() 59 | forwardAndGradTime = timer:time().real / count 60 | print('Forward time / sample [ms]: ', 1e3 * forwardTime) 61 | print('Forward and grad time / sample [ms]: ', 1e3 * forwardAndGradTime) 62 | print('Just grad time / sample [ms]: ', 1e3 * (forwardAndGradTime - forwardTime)) 63 | 64 | -------------------------------------------------------------------------------- 65 | -------------------------------------------------------------------------------- 66 | 67 | -- Eye-size, spatial, bottom-up 68 | print(dash..'Profiling eye-size, spatial, bottom-up\n') 69 | img = top10.phone.image[1] 70 | model:forward(img:cuda()) 71 | psProb = torch.CudaTensor(#model.output) 72 | count = 1000 73 | collectgarbage() 74 | timer:reset() 75 | for i = 1, count do 76 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 77 | for i = 1, model.output:size(2) do 78 | for j = 1, model.output:size(3) do 79 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 80 | end 81 | end 82 | end 83 | cutorch.synchronize() 84 | forwardTime = timer:time().real / count 85 | collectgarbage() 86 | timer:reset() 87 | for i = 1, count do 88 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 89 | for i = 1, model.output:size(2) do 90 | for j = 1, model.output:size(3) do 91 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 92 | end 93 | end 94 | gradInput = model:updateGradInput(img:cuda(), psProb) -- bottom-up 95 | end 96 | cutorch.synchronize() 97 | forwardAndGradTime = timer:time().real / count 98 | print('Forward time / sample [ms]: ', 1e3 * forwardTime) 99 | print('Forward and grad time / sample [ms]: ', 1e3 * forwardAndGradTime) 100 | print('Just grad time / sample [ms]: ', 1e3 * (forwardAndGradTime - forwardTime)) 101 | 102 | -- 1067x1600, spatial, bottom-up 103 | print(dash..'Profiling 1067x1600, spatial, bottom-up\n') 104 | img = image.load('../imgs/peep01.jpg') 105 | model:forward(img:cuda()) 106 | psProb = torch.CudaTensor(#model.output) 107 | count = 100 108 | collectgarbage() 109 | timer:reset() 110 | for i = 1, count do 111 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 112 | for i = 1, model.output:size(2) do 113 | for j = 1, model.output:size(3) do 114 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 115 | end 116 | end 117 | end 118 | cutorch.synchronize() 119 | forwardTime = timer:time().real / count 120 | collectgarbage() 121 | timer:reset() 122 | for i = 1, count do 123 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 124 | for i = 1, model.output:size(2) do 125 | for j = 1, model.output:size(3) do 126 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 127 | end 128 | end 129 | gradInput = model:updateGradInput(img:cuda(), psProb) -- bottom-up 130 | end 131 | cutorch.synchronize() 132 | forwardAndGradTime = timer:time().real / count 133 | print('forward time / sample [ms]: ', 1e3 * forwardTime) 134 | print('forward and grad time / sample [ms]: ', 1e3 * forwardAndGradTime) 135 | print('just grad time / sample [ms]: ', 1e3 * (forwardAndGradTime - forwardTime)) 136 | 137 | -- 1067x1600, spatial, bottom-up, SpatialSoftMax 138 | print(dash..'Profiling 1067x1600, spatial, bottom-up, SpatialSoftMax\n') 139 | img = image.load('../imgs/peep01.jpg') 140 | model:forward(img:cuda()) 141 | count = 100 142 | collectgarbage() 143 | timer:reset() 144 | for i = 1, count do 145 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 146 | psProb = nn.SpatialSoftMax(model.output) 147 | end 148 | cutorch.synchronize() 149 | forwardTime = timer:time().real / count 150 | collectgarbage() 151 | timer:reset() 152 | for i = 1, count do 153 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 154 | psProb = nn.SpatialSoftMax(model.output) 155 | gradInput = model:updateGradInput(img:cuda(), psProb) -- bottom-up 156 | end 157 | cutorch.synchronize() 158 | forwardAndGradTime = timer:time().real / count 159 | print('Forward time / sample [ms]: ', 1e3 * forwardTime) 160 | print('Forward and grad time / sample [ms]: ', 1e3 * forwardAndGradTime) 161 | print('Just grad time / sample [ms]: ', 1e3 * (forwardAndGradTime - forwardTime)) 162 | 163 | -------------------------------------------------------------------------------- 164 | -------------------------------------------------------------------------------- 165 | --[[ 166 | -- Eye-size, spatial, top-down 167 | print('profiling eye-size, spatial, top-down') 168 | img = top10.phone.image[1] 169 | model:forward(img:cuda()) 170 | psProb = torch.CudaTensor(#model.output) 171 | target = psProb:clone():zero() 172 | count = 1000 173 | collectgarbage() 174 | timer:reset() 175 | for i = 1, count do 176 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 177 | for i = 1, model.output:size(2) do 178 | for j = 1, model.output:size(3) do 179 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 180 | end 181 | end 182 | end 183 | cutorch.synchronize() 184 | forwardTime = timer:time().real / count 185 | collectgarbage() 186 | timer:reset() 187 | for i = 1, count do 188 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 189 | for i = 1, model.output:size(2) do 190 | for j = 1, model.output:size(3) do 191 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 192 | end 193 | end 194 | target[2] = psProb[2]:clone():mul(-1) -- top-down 195 | gradInput = model:updateGradInput(img:cuda(), target) 196 | end 197 | cutorch.synchronize() 198 | forwardAndGradTime = timer:time().real / count 199 | print('Forward time / sample [ms]: ', 1e3 * forwardTime) 200 | print('Forward and grad time / sample [ms]: ', 1e3 * forwardAndGradTime) 201 | print('Just grad time / sample [ms]: ', 1e3 * (forwardAndGradTime - forwardTime)) 202 | 203 | -- 1067x1600, spatial, top-down 204 | print('profiling 1067x1600, spatial, top-down') 205 | img = image.load('../imgs/peep01.jpg') 206 | model:forward(img:cuda()) 207 | psProb = torch.CudaTensor(#model.output) 208 | target = psProb:clone():zero() 209 | count = 100 210 | collectgarbage() 211 | timer:reset() 212 | for i = 1, count do 213 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 214 | for i = 1, model.output:size(2) do 215 | for j = 1, model.output:size(3) do 216 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 217 | end 218 | end 219 | end 220 | cutorch.synchronize() 221 | forwardTime = timer:time().real / count 222 | collectgarbage() 223 | timer:reset() 224 | for i = 1, count do 225 | gradLoss = model:forward(img:cuda()) -- if getspatialmodel 226 | for i = 1, model.output:size(2) do 227 | for j = 1, model.output:size(3) do 228 | psProb[{ {},i,j }] = SM:forward(model.output[{ {},i,j }]) 229 | end 230 | end 231 | target[2] = psProb[2]:clone():mul(-1) -- top-down 232 | gradInput = model:updateGradInput(img:cuda(), target) 233 | end 234 | cutorch.synchronize() 235 | forwardAndGradTime = timer:time().real / count 236 | print('Forward time / sample [ms]: ', 1e3 * forwardTime) 237 | print('Forward and grad time / sample [ms]: ', 1e3 * forwardAndGradTime) 238 | print('Just grad time / sample [ms]: ', 1e3 * (forwardAndGradTime - forwardTime)) 239 | 240 | --]] 241 | --------------------------------------------------------------------------------