├── LICENSE
├── README.md
├── components_coding_phase_info_figure.m
├── effect_of_architecture_figure.m
├── effect_of_lrn_nobackprop.m
├── effect_of_phase_deconvnet.m
├── fig_spatial_selectivity_noarcheffects.m
├── fig_splash_all_images.m
├── fourier_phase.m
├── generate_all_figures.m
├── hand_specified_neuron_viz_fn.m
├── resizencrop.m
├── saliency
    ├── HackyConv.m
    ├── HackyLRN.m
    ├── HackyReLU.m
    ├── baseline3.m
    ├── exp_ksseg_production.m
    ├── exp_seg_eval.m
    ├── exp_seg_unpack.m
    ├── ksresize.m
    ├── kssaliency_fcn_proper.m
    ├── ksseg.m
    ├── resizencrop.m
    ├── run_2.m
    ├── salseg.m
    └── seg_eval.m
├── segmentation_qualitative_results_figure.m
├── setup_eccv2016code.m
├── supplementary
    ├── effect_of_phase_sup.m
    ├── fig_spatial_selectivity_noarcheffects_sup.m
    ├── fig_splash_all_images_sup.m
    └── generate_all_figures_sup.m
├── vl_imsc_am.m
└── vl_simplenn.m


/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2016-17 Aravindh Mahendran, Andrea Vedaldi
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms are permitted
 5 | provided that the above copyright notice and this paragraph are
 6 | duplicated in all such forms and that any documentation,
 7 | advertising materials, and other materials related to such
 8 | distribution and use acknowledge that the software was developed
 9 | by the <organization>. The name of the
10 | <organization> may not be used to endorse or promote products derived
11 | from this software without specific prior written permission.
12 | THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
13 | IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
14 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
15 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # deconvnet_analysis
 2 | Code for "Salient Deconvolutional Networks, Aravindh Mahendran, Andrea Vedaldi, ECCV 2016"
 3 | 
 4 | ### Parts of this code
 5 | 
 6 | 1. Generate figures in the paper
 7 | 2. Segmentation code for table 1 and .mat files required to generate figure 8.
 8 | 3. Supplementary material figures.
 9 | 
10 | Parts 1 and 2 are complete and documented. Part 3 is mainly an extension of part 1 with more images
11 | 
12 | ### How to run this code
13 | Follow these steps. ROOT refers to the main directory containing the file generate_all_figures.m
14 | 
15 | 1. Copy and install matconvnet into ROOT/matconvnet . Delete ROOT/matconvnet/matlab/simplenn/vl_simplenn.m
16 | 2. Copy vlfeat into ROOT/vlfeat
17 | 3. Download the alexnet model - imagenet-caffe-alex from http://www.vlfeat.org/matconvnet/pretrained/ into ROOT/models
18 | 4. Similarly download the vgg-verydeep-16 model - imagenet-vgg-verydeep-16 into ROOT/models
19 | 5. Download the imagenet validation dataset into ROOT/imagenet12-val .
20 | 6. Download and install gsc-1.2 from https://www.robots.ox.ac.uk/~vgg/software/iseg/ into ROOT/saliency/gsc
21 | 7. Download gtsegs_ijcv.mat from http://groups.inf.ed.ac.uk/calvin/proj-imagenet/data/ into saliency/data/ferrari
22 | 8. Start matlab and change directory to ROOT/saliency. Run exp_seg_unpack() - this will unpack gtsegs_ijcv.mat and compute the imdb.mat for the Ferrari dataset [4].
23 | 9. Explore ROOT/saliency/run_2.m to run different segmentation experiments to get numbers in Table 1 of the paper.
24 | 10. Explore and run ROOT/generate_all_figures.m to get figures from the paper. They will be saved into the ROOT/genfigs folder.
25 | 
26 | Note that step 9 assumes all the experiments for segmentation in ROOT/saliency/run_2.m were run. This assumption applies only when generating figure 8. You can also uncomment segmentation_qualitative_results_figure to skip it.
27 | 
28 | ### Research code
29 | If anything doesn't work then please post the issue and I'll try to fix it. This is research code and comes with no WARRANTY or GUARANTY of any sort.
30 | 
31 | ### References
32 | 1. Guillaumin, M., Küttel, D., Ferrari, V.: Imagenet auto-annotation with segmentation propagation. In: IJCV (2014)
33 | 2. Noh, H., Hong, S., Han, B.: Learning deconvolution network for semantic segmentation. In: Proc. ICCV (2015)
34 | 3. Simonyan, K., Vedaldi, A., Zisserman, A.: Deep inside convolutional networks: Visualising image classification models and saliency maps. In: ICLR (2014)
35 | 4. Springenberg, J.T., Dosovitskiy, A., Brox, T., Riedmiller, M.: Striving for simplicity: The all convolutional net. In: ICLR Workshop (2015)
36 | 5. Zeiler, M.D., Fergus, R.: Visualizing and understanding convolutional networks. In: Proc. ECCV (2014)
37 | 6. V. Gulshan, C. Rother, A. Criminisi, A. Blake and A. Zisserman.: Geodesic star convexity for interactive image segmentation. In: Proc. CVPR (2010)
38 | 


--------------------------------------------------------------------------------
/components_coding_phase_info_figure.m:
--------------------------------------------------------------------------------
 1 | % generates fig. 3 of the paper
 2 | 
 3 | FIGS_PATH = 'genfigs/';
 4 | 
 5 | %IMAGE_IDS = [77, 170];%, 98, 234,289,295,298,299,306];
 6 | IMAGE_IDS = [299];
 7 | IMAGE_NAMES = {sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(1))};
 8 | 
 9 | MODEL_NAMES = {'models/imagenet-vgg-verydeep-16.mat'};
10 | 
11 | LAYERS = {[31]};
12 | 
13 | % Relu mask (T/F), Pooling Switches (T/F), Relu Backwards (T/F)
14 | METHODS = {'TTT', 'TFT', 'FFT', 'FTT', 'TTF', 'TFF', 'FFF', 'FTF'};
15 | 
16 | viz_images = cell(1,1,1,8);
17 | 
18 | for img_no = 1
19 |     opts = struct();
20 |     opts.randomizeWeights = false;
21 |     opts.gpu = false;
22 |     
23 |     opts.relus_to_change = 1:100;
24 |     opts.pools_to_change = 1:5;
25 |     opts.convs_to_change = 1:100;
26 |     
27 |     opts.imagePath = IMAGE_NAMES{img_no};
28 |     
29 |     for model_no = 1
30 |         opts.modelPath = MODEL_NAMES{model_no};
31 |         
32 |         for layer_no = 1
33 |             opts.layer = LAYERS{model_no}(layer_no);
34 |             
35 |             for method_no = 1:numel(METHODS)
36 |                 opts.algorithm = METHODS{method_no};
37 |                 
38 |                 opts.neuron_I = inf;
39 |                 opts.neuron_J = inf;
40 |                 opts.neuron_channel = inf; 
41 |                 
42 |                 [~, viz_images{img_no, model_no, layer_no, method_no}, ~, ~] ...
43 |                      = hand_specified_neuron_viz_fn(opts);
44 |                  
45 |                 viz_images{img_no, model_no, layer_no, method_no} = ...
46 |                      padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no}), [1, 1], 1, 'both');
47 |             end
48 |             
49 |         end
50 |     end
51 | end
52 | 
53 | final_image_relubackwards = [viz_images{1,1,1,1}, viz_images{1,1,1,4};
54 |                viz_images{1,1,1,2}, viz_images{1,1,1,3}];
55 |            
56 | final_image_norelubackwards = [viz_images{1,1,1,5}, viz_images{1,1,1,6};
57 |                viz_images{1,1,1,8}, viz_images{1,1,1,7}];
58 | 
59 | imwrite(final_image_relubackwards, [FIGS_PATH, 'phasesources_relubackwards.jpg']);
60 | imwrite(final_image_norelubackwards, [FIGS_PATH, 'phasesources_norelubackwards.jpg']);
61 | 
62 | imwrite(viz_images{1,1,1,1}, [FIGS_PATH, 'relubackward_relumask_mpoolswitches.jpg']);
63 | imwrite(viz_images{1,1,1,2}, [FIGS_PATH, 'relubackward_relumask_mpoolcenter.jpg']);
64 | imwrite(viz_images{1,1,1,3}, [FIGS_PATH, 'relubackward_norelumask_mpoolcenter.jpg']);
65 | imwrite(viz_images{1,1,1,4}, [FIGS_PATH, 'relubackward_norelumask_mpoolswitches.jpg']);
66 | 
67 | imwrite(viz_images{1,1,1,5}, [FIGS_PATH, 'norelubackward_relumask_mpoolswitches.jpg']);
68 | imwrite(viz_images{1,1,1,6}, [FIGS_PATH, 'norelubackward_relumask_mpoolcenter.jpg']);
69 | imwrite(viz_images{1,1,1,7}, [FIGS_PATH, 'norelubackward_norelumask_mpoolcenter.jpg']);
70 | imwrite(viz_images{1,1,1,8}, [FIGS_PATH, 'norelubackward_norelumask_mpoolswitches.jpg']);
71 | 


--------------------------------------------------------------------------------
/effect_of_architecture_figure.m:
--------------------------------------------------------------------------------
  1 | % Generates fig. 5 of the paper
  2 | 
  3 | FIGS_PATH = 'genfigs/';
  4 | 
  5 | %IMAGE_IDS = [77, 170];%, 98, 234,289,295,298,299,306];
  6 | IMAGE_IDS = [77];
  7 | IMAGE_NAMES = {sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(1))};%, ...
  8 | 
  9 | MODEL_NAMES = {'models/imagenet-vgg-verydeep-16.mat',...
 10 |     'models/imagenet-caffe-alex.mat'};
 11 | 
 12 | %LAYERS = {[37,39], [15,17]};
 13 | LAYERS = {[31,36], [15,20]};
 14 | viz_images = cell(1,1,2,7);
 15 | 
 16 | for img_no = 1
 17 |     opts = struct();
 18 |     opts.randomizeWeights = false;
 19 |     opts.gpu = false;
 20 |     
 21 |     opts.relus_to_change = 1:100;
 22 |     opts.pools_to_change = 1:5;
 23 |     opts.convs_to_change = 1:100;
 24 |     
 25 |     opts.imagePath = IMAGE_NAMES{img_no};
 26 |     
 27 |     for model_no = 1
 28 |         opts.modelPath = MODEL_NAMES{model_no};
 29 |         
 30 |         for layer_no = 1:2
 31 |             opts.randomizeWeights = false;
 32 |             opts.gpu = false;
 33 |             
 34 |             opts.layer = LAYERS{model_no}(layer_no);
 35 |             
 36 |             % The the deconvnet
 37 |             opts.algorithm = 'deconvnet';
 38 |             
 39 |             opts.neuron_I = 1/2;
 40 |             opts.neuron_J = 1/2;
 41 |             opts.neuron_channel = inf;
 42 |             
 43 |             [~, t, ~, ~] = hand_specified_neuron_viz_fn(opts);
 44 |             
 45 |             viz_images{img_no, model_no, layer_no, 1} = ...
 46 |                 padarray(vl_imsc_am(t), [1, 1], 1, 'both');
 47 |             
 48 |             % But all the pixels are active
 49 |             viz_images{img_no, model_no, layer_no, 2} = ...
 50 |                 padarray(vl_imsc_am(sign(t).*abs(t).^0.1), [1,1], 1, 'both');
 51 |             
 52 |             % First our algorithm
 53 |             opts.algorithm = 'TTT';
 54 |             
 55 |             opts.neuron_I = 1/2;
 56 |             opts.neuron_J = 1/2;
 57 |             opts.neuron_channel = inf;
 58 |             
 59 |             [~, t, ~, ~] = hand_specified_neuron_viz_fn(opts);
 60 |             
 61 |             viz_images{img_no, model_no, layer_no, 3} = ...
 62 |                 padarray(vl_imsc_am(t), [1, 1], 1, 'both');
 63 |             
 64 |             % But all the pixels are active
 65 |             viz_images{img_no, model_no, layer_no, 4} = ...
 66 |                 padarray(vl_imsc_am(sign(t).*abs(t).^0.1), [1,1], 1, 'both');
 67 |             
 68 |             % What if we remove everything - we still see the foveation
 69 |             opts.algorithm = 'FFF';
 70 |             [~, viz_images{img_no, model_no, layer_no, 5}, opts_new, ~] ...
 71 |                 = hand_specified_neuron_viz_fn(opts);
 72 |             
 73 |             viz_images{img_no, model_no, layer_no, 5} = ...
 74 |                 padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, 5}), [1, 1], 1, 'both');
 75 |             
 76 |             % This has nothing to do with the weights. See it is there even
 77 |             % if we randomize the weights.
 78 |             opts.randomizeWeights = true;
 79 |             opts.algorithm = 'FFF';
 80 |             opts.neuron_I = opts_new.neuron_I; % After randomizing the weights the max response
 81 |             % is going to be at a different location. So this is to ensure
 82 |             % that we use the same neuron as above.
 83 |             opts.neuron_J = opts_new.neuron_J;
 84 |             opts.neuron_channel = opts_new.neuron_channel;
 85 |             [~, viz_images{img_no, model_no, layer_no, 6}, ~, ~] ...
 86 |                 = hand_specified_neuron_viz_fn(opts);
 87 |             
 88 |             viz_images{img_no, model_no, layer_no, 6} = ...
 89 |                 padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, 6}), [1, 1], 1, 'both');
 90 | 
 91 |             % And it's like this for AlexNet.
 92 |             opts_alex = opts;
 93 |             opts_alex.modelPath = MODEL_NAMES{2};
 94 |             opts_alex.randomizeWeights = true;
 95 |             opts_alex.algorithm = 'FFF';
 96 |             opts_alex.neuron_I = inf;
 97 |             opts_alex.neuron_J = inf;
 98 |             opts_alex.neuron_channel = inf;
 99 |             opts_alex.layer = LAYERS{2}(layer_no);
100 |             
101 |             [~, viz_images{img_no, model_no, layer_no, 7}, ~, ~] ...
102 |                 = hand_specified_neuron_viz_fn(opts_alex);
103 |             
104 |             viz_images{img_no, model_no, layer_no, 7} = ...
105 |                 padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, 7}), [1, 1], 1, 'both');
106 |             
107 |             
108 |         end
109 |     end
110 | end
111 | 
112 | img = imread(IMAGE_NAMES{1});
113 | img_pp = resizencrop(img, size(viz_images{1,1,1,1}));
114 | final_image = [cat(2, viz_images{1,1,1,[3,4,5,6]})];%...
115 |     %cat(2, viz_images{1,1,2,1:6})];
116 | 
117 | %final_image_alexnet = [viz_images{1,1,1,7}, viz_images{1,1,2,7}];
118 | 
119 | %figure; subplot(1,2,1); imshow(final_image); subplot(1,2,2); imshow(final_image_alexnet);
120 | imwrite(final_image, [FIGS_PATH, 'architectureeffect_concise.jpg']);
121 | %imwrite(final_image_alexnet, [FIGS_PATH, 'fig6_architectureeffect_alexnet.jpg']);


--------------------------------------------------------------------------------
/effect_of_lrn_nobackprop.m:
--------------------------------------------------------------------------------
 1 | % Generates the figure in supplementary material that shows that LRN^{BP}
 2 | % vs identity operations in the reversed architecture are not very
 3 | % different
 4 | 
 5 | FIGS_PATH = 'genfigs/';
 6 | 
 7 | % The effect of hacking the local contrast normalization layers
 8 | 
 9 | % we are using 1 image only and comparing our approach with local response
10 | % normalization and without local response normalization. The latter is
11 | % closer to deconvnet the former is closer to saliency.
12 | 
13 | % As VGG-19 doesn't have LRN layers, this figure only involves alexnet.
14 | 
15 | % We'll do it for one shallow and one deep layer
16 | 
17 | %IMAGE_IDS = [43];
18 | IMAGE_NAMES = {'stock_abstract.jpg'};%, ...
19 |    %sprintf([IMAGENET12_VAL_PATH, '/ILSVRC2012_val_%08d.JPEG'], IMAGE_IDS(1)) };
20 | 
21 | MODEL_NAMES = {'models/imagenet-caffe-alex.mat'};
22 | 
23 | LAYERS = {[15,20]}; % pool5 and relu7
24 | 
25 | METHOD_NAMES = {'TTT', 'TTTF'};
26 | 
27 | viz_images = cell(2,1,2,2);
28 | 
29 | for img_no = 1:1
30 |     opts = struct();
31 |     opts.randomizeWeights = false;
32 |     opts.gpu = false;
33 |     
34 |     opts.relus_to_change = 1:100;
35 |     opts.pools_to_change = 1:5;
36 |     opts.convs_to_change = 1:100;
37 |     
38 |     opts.imagePath = IMAGE_NAMES{img_no};
39 |     
40 |     for model_no = 1
41 |         opts.modelPath = MODEL_NAMES{model_no};
42 |         
43 |         for layer_no = 1:2
44 |             opts.layer = LAYERS{model_no}(layer_no);
45 |             
46 |             for method_no = 1:2
47 |                 opts.algorithm = METHOD_NAMES{method_no};
48 |             
49 |                 opts.neuron_I = inf;
50 |                 opts.neuron_J = inf;
51 |                 opts.neuron_channel = inf;
52 |             
53 |                 [~, t, ~, ~] = hand_specified_neuron_viz_fn(opts);
54 |             
55 |                 viz_images{img_no, model_no, layer_no, method_no} = ...
56 |                     padarray(vl_imsc_am(t), [1, 1], 1, 'both');
57 |             
58 |             end
59 |         end
60 |     end
61 | end
62 | 
63 | NET = vl_simplenn_tidy(load(MODEL_NAMES{1}));
64 | img = im2single(imread(IMAGE_NAMES{1}));
65 | img_pp = padarray(resizencrop(img, NET.meta.normalization.imageSize), [1,1], 1, 'both');
66 | 
67 | viz_final = [img_pp, viz_images{1,1,1,1}, viz_images{1,1,1,2}, viz_images{1,1,2,1}, viz_images{1,1,2,2}];
68 | %viz_images{2,1,1,1}, viz_images{2,1,1,2}, viz_images{2,1,2,1}, viz_images{2,1,2,2}];
69 | 
70 | imwrite(viz_final, [FIGS_PATH, 'effectof_lrnnobackprop.jpg']);   


--------------------------------------------------------------------------------
/effect_of_phase_deconvnet.m:
--------------------------------------------------------------------------------
  1 | % generates fig. 4 of the paper
  2 | 
  3 | FIGS_PATH = 'genfigs';
  4 | 
  5 | IMAGE_IDS = [234,289];
  6 | IMAGE_NAMES = {sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(1)), ...
  7 |     sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(2))};
  8 | 
  9 | MODEL_NAMES = {'models/imagenet-vgg-verydeep-16.mat',...
 10 |     'models/imagenet-caffe-alex.mat'};
 11 | 
 12 | LAYERS = {[24, 36], [12, 20]};
 13 | 
 14 | METHODS = {'deconvnet'}; % other options are 'TTT', 'saliency' 
 15 | 
 16 | viz_images = cell(2,2,2,1,3);
 17 | 
 18 | for img_no = 1:2
 19 |     opts = struct();
 20 |     opts.randomizeWeights = false;
 21 |     opts.gpu = false;
 22 |     
 23 |     opts.relus_to_change = 1:100;
 24 |     opts.pools_to_change = 1:5;
 25 |     opts.convs_to_change = 1:100;
 26 |     
 27 |     opts.imagePath = IMAGE_NAMES{img_no};
 28 |     
 29 |     for model_no = 1:2
 30 |         opts.modelPath = MODEL_NAMES{model_no};
 31 |         
 32 |         for layer_no = 1:2
 33 |             opts.layer = LAYERS{model_no}(layer_no);
 34 |             
 35 |             for method_no = 1:1
 36 |                 
 37 |                 cituation_no = 1;
 38 |                 
 39 |                 opts.neuron_I = 1/2;
 40 |                 opts.neuron_J = 1/2;
 41 |                 opts.neuron_channel = inf;
 42 |                 
 43 |                 opts.algorithm = METHODS{method_no};
 44 |                 
 45 |                 [~, viz_images{img_no, model_no, layer_no, method_no, cituation_no}, opts_new, ~] ...
 46 |                     = hand_specified_neuron_viz_fn(opts);
 47 |                 
 48 |                 viz_images{img_no, model_no, layer_no, method_no, cituation_no} = ...
 49 |                     padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no, cituation_no}), [1, 1], 1, 'both');
 50 |                 
 51 |                 cituation_no = 2;
 52 |                 
 53 |                 opts.neuron_I = opts_new.neuron_I;
 54 |                 opts.neuron_J = opts_new.neuron_J;
 55 |                 opts.neuron_channel = -1;
 56 |                 
 57 |                 [~, viz_images{img_no, model_no, layer_no, method_no, cituation_no}, opts_new2, NET] ...
 58 |                     = hand_specified_neuron_viz_fn(opts);
 59 |                 
 60 |                 if(opts_new2.neuron_channel == opts_new.neuron_channel)
 61 |                     error('Oops I randomly picked the maximally excited neuron');
 62 |                 end
 63 |                 
 64 |                 viz_images{img_no, model_no, layer_no, method_no, cituation_no} = ...
 65 |                     padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no, cituation_no}), [1,1], 1, 'both');
 66 |                 
 67 |                 cituation_no = 3;
 68 |                 
 69 |                 img = imread(IMAGE_NAMES{img_no});
 70 |                 img_pp = opts_new.normalize(img);
 71 |                 NET_info = vl_simplenn_display(NET, 'inputSize', [NET.meta.normalization.imageSize(1:2), 3, 1]);
 72 |                 dzdy = zeros(NET_info.dataSize(:, end)', 'single');
 73 |                 dzdy(opts_new.neuron_I, opts_new.neuron_J, :) = abs(randn([1,1,size(dzdy,3)], 'single'));
 74 |                 %dzdy = abs(randn(NET_info.dataSize(:, end)', 'single'));
 75 |                 res = vl_simplenn(NET, img_pp, dzdy, [], 'conserveMemory', true);
 76 |                 viz_images{img_no, model_no, layer_no, method_no, cituation_no} = ...
 77 |                     padarray(vl_imsc_am(res(1).dzdx), [1,1], 1, 'both');
 78 |                 
 79 |             end
 80 |         end
 81 |     end
 82 | end
 83 | 
 84 | % final_image_vgg16 = [viz_images{1,1,1,1}, viz_images{1,1,2,1};
 85 | %     viz_images{1,1,1,2}, viz_images{1,1,2,2};
 86 | %     viz_images{1,1,1,3}, viz_images{1,1,2,3};
 87 | %     viz_images{2,1,1,1}, viz_images{2,1,2,1};
 88 | %     viz_images{2,1,1,2}, viz_images{2,1,2,2};
 89 | %     viz_images{2,1,1,3}, viz_images{2,1,2,3}];
 90 | % 
 91 | % final_image_alexnet = [viz_images{1,2,1,1}, viz_images{1,2,2,1};
 92 | %     viz_images{1,2,1,2}, viz_images{1,2,2,2};
 93 | %     viz_images{1,2,1,3}, viz_images{1,2,2,3};
 94 | %     viz_images{2,2,1,1}, viz_images{2,2,2,1};
 95 | %     viz_images{2,2,1,2}, viz_images{2,2,2,2};
 96 | %     viz_images{2,2,1,3}, viz_images{2,2,2,3}];
 97 | for t = 1:2
 98 | %final_image_vgg16_TTT = [viz_images{t,1,1,1,1}, viz_images{t,1,1,1,2}, viz_images{t,1,1,1,3};
 99 | %    viz_images{t,1,2,1,1}, viz_images{t,1,2,1,2}, viz_images{t,1,2,1,3}];
100 | final_image_vgg16_TTT = [viz_images{t,1,1,1,1}, viz_images{t,1,2,1,1};
101 |                          viz_images{t,1,1,1,2}, viz_images{t,1,2,1,2};
102 |                          viz_images{t,1,1,1,3}, viz_images{t,1,2,1,2}];
103 |     
104 | % final_image_vgg16_TTT = [viz_images{1,1,1,2,1}, viz_images{1,1,1,2,2}, viz_images{1,1,1,2,3};
105 | %     viz_images{1,1,2,2,1}, viz_images{1,1,2,2,2}, viz_images{1,1,2,2,3}];
106 | 
107 | % final_image_vgg16_saliency = [viz_images{1,1,1,3,1}, viz_images{1,1,1,3,2}, viz_images{1,1,1,3,3};
108 | %     viz_images{1,1,2,3,1}, viz_images{1,1,2,3,2}, viz_images{1,1,2,3,3}];
109 | 
110 | %final_image_alexnet_TTT = [viz_images{t,2,1,1,1}, viz_images{t,2,1,1,2}, viz_images{t,2,1,1,3};
111 | %    viz_images{t,2,2,1,1}, viz_images{t,2,2,1,2}, viz_images{t,2,2,1,3}];
112 | 
113 | final_image_alexnet_TTT = [viz_images{t,2,1,1,1}, viz_images{t,2,2,1,1};
114 |                            viz_images{t,2,1,1,2}, viz_images{t,2,2,1,2};
115 |                            viz_images{t,2,1,1,3}, viz_images{t,2,2,1,3}];
116 | 
117 | % final_image_alexnet_TTT = [viz_images{1,2,1,2,1}, viz_images{1,2,1,2,2}, viz_images{1,2,1,2,3};
118 | %     viz_images{1,2,2,2,1}, viz_images{1,2,2,2,2}, viz_images{1,2,2,2,3}];
119 | 
120 | % final_image_alexnet_saliency = [viz_images{1,2,1,3,1}, viz_images{1,2,1,3,2}, viz_images{1,2,1,3,3};
121 | %     viz_images{1,2,2,3,1}, viz_images{1,2,2,3,2}, viz_images{1,2,2,3,3}];
122 | 
123 | % imwrite(final_image_alexnet_TTT, '../mahendran16cvpr/figs/fig2_alexnet_saliency_img234.png');
124 | % imwrite(final_image_vgg16_TTT, '../mahendran16cvpr/figs/fig2_vgg16_saliency_img234.png');
125 | 
126 | imwrite(final_image_alexnet_TTT, sprintf([FIGS_PATH, '/neuronselectivity_alexnet_%s_img%d_vert.png'], METHODS{1}, IMAGE_IDS(t)));
127 | imwrite(final_image_vgg16_TTT, sprintf([FIGS_PATH, '/neuronselectivity_vgg16_%s_img%d_vert.png'], METHODS{1}, IMAGE_IDS(t)));
128 | end


--------------------------------------------------------------------------------
/fig_spatial_selectivity_noarcheffects.m:
--------------------------------------------------------------------------------
  1 | % generates fig 6 in the paper
  2 | 
  3 | FIGS_PATH = 'genfigs/';
  4 | 
  5 | IMAGE_IDS = [2, 56, 94, 112, 131];
  6 | 
  7 | IMAGE_NAMES = {sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(1)), ...
  8 |      sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(2)),...
  9 |      sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(3)),...
 10 |      sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(4)),...
 11 |      sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(5))...
 12 |      };
 13 | 
 14 | MODEL_NAMES = {'models/imagenet-vgg-verydeep-16.mat',...
 15 |     'models/imagenet-caffe-alex.mat'};
 16 | 
 17 | LAYERS = {[31, 36], [15, 20]};
 18 | 
 19 | METHODS = {'deconvnet', 'saliency', 'TTT'};
 20 | 
 21 | viz_images = cell(5,2,2,4);
 22 | 
 23 | for img_no = 1:5
 24 |     opts = struct();
 25 |     
 26 |     opts.gpu = false;
 27 | 
 28 |     opts.relus_to_change = 1:100;
 29 |     opts.pools_to_change = 1:5;
 30 |     opts.convs_to_change = 1:100;
 31 | 
 32 |     opts.neuron_I = 1/2;
 33 |     opts.neuron_J = 1/2;
 34 |     opts.neuron_channel = inf;
 35 |     
 36 |     opts.imagePath = IMAGE_NAMES{img_no};
 37 |     
 38 |     for model_no = 1:2
 39 |         opts.modelPath = MODEL_NAMES{model_no};
 40 |         
 41 |         for layer_no = 1:2
 42 |             opts.layer = LAYERS{model_no}(layer_no);
 43 |             
 44 |             for method_no = 1:numel(METHODS)
 45 |                 opts.randomizeWeights = false;
 46 |                 opts.algorithm = METHODS{method_no};
 47 |                 
 48 |                 [~, viz_images{img_no, model_no, layer_no, method_no}, ~] ...
 49 |                     = hand_specified_neuron_viz_fn(opts);
 50 |                 viz_images{img_no, model_no, layer_no, method_no} = ...
 51 |                     padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no}), [1,1], 1, 'both');
 52 |                 
 53 |                 
 54 |             end
 55 |             
 56 |             %opts.algorithm = 'FFF';
 57 |             %opts.randomizeWeights = true;
 58 |             %
 59 |             %[~, viz_images{img_no, model_no, layer_no, 5}, ~] ...
 60 |             %        = hand_specified_neuron_viz_fn(opts);
 61 |             %viz_images{img_no, model_no, layer_no, 5} = ...
 62 |             %        padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, 5}), [1,1], 1, 'both');
 63 |             
 64 |         end
 65 |         
 66 |         img = imread(IMAGE_NAMES{img_no});
 67 |         NET = vl_simplenn_tidy(load(opts.modelPath));
 68 |         viz_images{img_no, model_no, 1, 4} = padarray(im2single(resizencrop(img, NET.meta.normalization.imageSize(1:2))), [1,1], 1, 'both');
 69 |         clear NET;
 70 |         clear img;
 71 |         
 72 |     end
 73 | 
 74 | end
 75 | 
 76 | final_image_vgg16 = ...
 77 | [cat(1, viz_images{:,1,1,4}), cat(1, viz_images{:,1,1,1}), cat(1, viz_images{:,1,1,2}), cat(1, viz_images{:,1,1,3}),...
 78 |  cat(1, viz_images{:,1,2,1}), cat(1, viz_images{:,1,2,2}), cat(1, viz_images{:,1,2,3})];
 79 | 
 80 | final_image_alexnet = ...
 81 | [cat(1, viz_images{:,2,1,4}), cat(1, viz_images{:,2,1,1}), cat(1, viz_images{:,2,1,2}), cat(1, viz_images{:,2,1,3}),...
 82 |  cat(1, viz_images{:,2,2,1}), cat(1, viz_images{:,2,2,2}), cat(1, viz_images{:,2,2,3})];
 83 | 
 84 | imwrite(final_image_vgg16, [FIGS_PATH, 'fig_spatial_selectivity_noarcheffects_vgg16.jpg']);
 85 | imwrite(final_image_alexnet, [FIGS_PATH, 'fig_spatial_selectivity_noarcheffects_alexnet.jpg']);
 86 | 
 87 | %final_image_vgg16_img170 = [viz_images{1,1,1,1}, viz_images{1,1,1,2}, viz_images{1,1,1,3}, viz_images{1,1,1,4}, viz_images{1,1,1,5};
 88 | %               viz_images{1,1,2,1}, viz_images{1,1,2,2}, viz_images{1,1,2,3}, viz_images{1,1,2,4}, viz_images{1,1,2,5}];
 89 |            
 90 | %final_image_vgg16_img98 = [viz_images{2,1,1,1}, viz_images{2,1,1,2}, viz_images{2,1,1,3}, viz_images{2,1,1,4}, viz_images{2,1,1,5};
 91 | %               viz_images{2,1,2,1}, viz_images{2,1,2,2}, viz_images{2,1,2,3}, viz_images{2,1,2,4}, viz_images{2,1,2,5}];
 92 | 
 93 | %final_image_alexnet_img170 = [viz_images{1,2,1,1}, viz_images{1,2,1,2}, viz_images{1,2,1,3}, viz_images{1,2,1,4}, viz_images{1,2,1,5};
 94 | %               viz_images{1,2,2,1}, viz_images{1,2,2,2}, viz_images{1,2,2,3}, viz_images{1,2,2,4}, viz_images{1,2,2,5}];
 95 |            
 96 | %final_image_alexnet_img98 = [viz_images{2,2,1,1}, viz_images{2,2,1,2}, viz_images{2,2,1,3}, viz_images{2,2,1,4}, viz_images{2,2,1,5};
 97 | %               viz_images{2,2,2,1}, viz_images{2,2,2,2}, viz_images{2,2,2,3}, viz_images{2,2,2,4}, viz_images{2,2,2,5}];
 98 | 
 99 | 
100 | %imwrite(final_image_alexnet_img170, '../mahendran16cvpr/figs/fig_spatial_selectivity_alexnet_img170.jpg');
101 | %imwrite(final_image_alexnet_img98, '../mahendran16cvpr/figs/fig_spatial_selectivity_alexnet_img98.jpg');
102 | 
103 | %imwrite(final_image_vgg16_img170, '../mahendran16cvpr/figs/fig_spatial_selectivity_vgg16_img170.jpg');
104 | %imwrite(final_image_vgg16_img98, '../mahendran16cvpr/figs/fig_spatial_selectivity_vgg16_img98.jpg');


--------------------------------------------------------------------------------
/fig_splash_all_images.m:
--------------------------------------------------------------------------------
 1 | % Generates fig 1 in the paper
 2 | FIGS_PATH = 'genfigs/';
 3 | 
 4 | IMAGE_IDS = [170, 98, 234,289,299,27,77];
 5 | IMAGE_NAMES = cell(numel(IMAGE_IDS),1);
 6 | for i=1:numel(IMAGE_IDS)
 7 |     IMAGE_NAMES{i} = sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(i));
 8 | end
 9 |     
10 | 
11 | MODEL_NAMES = {'models/imagenet-vgg-verydeep-16.mat',...
12 |     'models/imagenet-caffe-alex.mat'};
13 | 
14 | %LAYERS = {[28, 42], [12, 20]};
15 | LAYERS = {[36], [20]};
16 | 
17 | METHODS = {'deconvnet', 'saliency', 'TTT'};
18 | 
19 | viz_images = cell(numel(IMAGE_IDS),2,1,4);
20 | 
21 | for img_no = 1:numel(IMAGE_IDS)
22 |     opts = struct();
23 |     opts.randomizeWeights = false;
24 |     opts.gpu = false;
25 | 
26 |     opts.relus_to_change = 1:100;
27 |     opts.pools_to_change = 1:5;
28 |     opts.convs_to_change = 1:100;
29 | 
30 |     opts.neuron_I = inf;
31 |     opts.neuron_J = inf;
32 |     opts.neuron_channel = inf;
33 |     
34 |     opts.imagePath = IMAGE_NAMES{img_no};
35 |     
36 |     for model_no = 1:2
37 |         opts.modelPath = MODEL_NAMES{model_no};
38 |         
39 |         for layer_no = 1:1
40 |             opts.layer = LAYERS{model_no}(layer_no);
41 |             
42 |             for method_no = 1:numel(METHODS)
43 |                 opts.algorithm = METHODS{method_no};
44 |                 
45 |                 [~, viz_images{img_no, model_no, layer_no, method_no}, opts_new, ~] ...
46 |                     = hand_specified_neuron_viz_fn(opts);
47 |                 viz_images{img_no, model_no, layer_no, method_no} = ...
48 |                     padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no}), [1,1], 1, 'both');
49 |                 
50 |                 
51 |             end
52 |         end
53 |         img = imread(IMAGE_NAMES{img_no});
54 |         NET = vl_simplenn_tidy(load(opts.modelPath));
55 |         viz_images{img_no, model_no, 1, 4} = padarray(im2single(resizencrop(img, NET.meta.normalization.imageSize(1:2))), [1,1], 1, 'both');    
56 |         clear NET;
57 |         clear img;
58 |     end
59 |     
60 |     
61 | end
62 | 
63 | t = 1;
64 | final_image_vgg16 = [...
65 |                cat(2, viz_images{:,t,1,4});...
66 |                cat(2, viz_images{:,t,1,1});...
67 |                cat(2, viz_images{:,t,1,2});...
68 |                cat(2, viz_images{:,t,1,3})];
69 |            
70 | t = 2;
71 | final_image_alexnet = [...
72 |                cat(2, viz_images{:,t,1,4});...
73 |                cat(2, viz_images{:,t,1,1});...
74 |                cat(2, viz_images{:,t,1,2});...
75 |                cat(2, viz_images{:,t,1,3})];
76 |            
77 | imwrite(final_image_alexnet, [FIGS_PATH, 'fig_splash_alexnet.jpg']);
78 | imwrite(final_image_vgg16, [FIGS_PATH, 'fig_splash_vgg16.jpg']);
79 | 


--------------------------------------------------------------------------------
/fourier_phase.m:
--------------------------------------------------------------------------------
  1 | % generates fig. 7 in the paper
  2 | IMAGE_IDS = [177, 249];
  3 | 
  4 | FIGS_PATH = 'genfigs/';
  5 | 
  6 | IMAGE_NAMES = {sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(1)), ...
  7 |     sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(2))};
  8 |      
  9 | im = imread(IMAGE_NAMES{1}) ;
 10 | im = rgb2gray(im2single(im)) ;
 11 | im = resizencrop(im, [224,224]) ;
 12 | 
 13 | im_2 = imread(IMAGE_NAMES{2}) ;
 14 | im_2 = rgb2gray(im2single(im_2)) ;
 15 | im_2 = resizencrop(im_2, [224,224]) ;
 16 | 
 17 | % for real images one muust have
 18 | % Fm1 == circshift(flipud(fliplr(Fm1)),[1 1]) !
 19 | 
 20 | F = fft2(im) ;
 21 | Fa = angle(F) ;
 22 | Fm = abs(F) ;
 23 | Fm1 = randn(size(Fm)) ;
 24 | %Fm1 = Fm1 + circshift(flipud(fliplr(Fm1)),[1 1]) ;
 25 | %Fm1 = Fm1*0 + 1 ;
 26 | Fm2 = abs(Fm1) ;
 27 | 
 28 | %F_other = fft2(im_2);
 29 | %Fm2 = abs(F_other); % Get the magnitude from another image.
 30 | 
 31 | im0 = real(ifft2(Fm .* exp(1i*Fa))) ;
 32 | im1 = real(ifft2(Fm1 .* exp(1i*Fa))) ;
 33 | im2 = real(ifft2(Fm2 .* exp(1i*Fa))) ;
 34 | 
 35 | figure ; clf;
 36 | subplot(1,3,1) ; imagesc(im0) ; axis image ; axis off; title('Original Amplitude');
 37 | subplot(1,3,2) ; imagesc(im1) ; axis image ; axis off; title('Randomized Amplitude');
 38 | subplot(1,3,3) ; imagesc(im2) ; axis image ; axis off; title('Positive Randomized Amplitude') ;
 39 | 
 40 | im0_viz = padarray(im0, [1,1], 1, 'both');
 41 | im1_viz = padarray(im1, [1,1], 1, 'both');
 42 | im2_viz = padarray(vl_imsc_am(im2), [1,1], 1, 'both');
 43 | 
 44 | % Now to visualize VGG-16 fc8 for this image.
 45 | opts = struct();
 46 | opts.randomizeWeights = false;
 47 | opts.gpu = false;
 48 | 
 49 | opts.relus_to_change = 1:100;
 50 | opts.pools_to_change = 1:5;
 51 | opts.convs_to_change = 1:100;
 52 | 
 53 | opts.neuron_I = 1;
 54 | opts.neuron_J = 1;
 55 | opts.neuron_channel = inf;
 56 | 
 57 | opts.imagePath = IMAGE_NAMES{1};
 58 | opts.modelPath = 'models/imagenet-vgg-verydeep-16.mat';
 59 | opts.layer = 36;
 60 | opts.algorithm = 'TTT';
 61 | [~, TTT_viz, opts_new, NET, ~] = hand_specified_neuron_viz_fn(opts);
 62 | TTT_viz = mean(padarray(vl_imsc_am(TTT_viz), [1,1], 1, 'both'), 3);
 63 | 
 64 | %im_2_pp = opts_new.normalize(rgb2gray(imread(IMAGE_NAMES{2})));
 65 | %res = vl_simplenn(NET, im_2_pp);
 66 | %Fm2 = res(end).x;
 67 | 
 68 | im_1_pp = opts_new.normalize(rgb2gray(imread(IMAGE_NAMES{1})));
 69 | res = vl_simplenn(NET, im_1_pp);
 70 | Fm1 = res(end).x;
 71 | 
 72 | res2 = vl_simplenn(NET, im_1_pp, abs(randn(size(Fm1), 'single')));
 73 | TTT2_viz = mean(padarray(vl_imsc_am(res2(1).dzdx), [1, 1], 1, 'both'), 3);
 74 | 
 75 | clear NET;
 76 | 
 77 | % Now to visualize VGG-16 fc8 for this image using deconvnet instead of TTT.
 78 | opts = struct();
 79 | opts.randomizeWeights = false;
 80 | opts.gpu = false;
 81 | 
 82 | opts.relus_to_change = 1:100;
 83 | opts.pools_to_change = 1:5;
 84 | opts.convs_to_change = 1:100;
 85 | 
 86 | opts.neuron_I = 1;
 87 | opts.neuron_J = 1;
 88 | opts.neuron_channel = inf;
 89 | 
 90 | opts.imagePath = IMAGE_NAMES{1};
 91 | opts.modelPath = [MATCONVNET_PATH, 'data/models/imagenet-vgg-verydeep-16.mat'];
 92 | opts.layer = 36;
 93 | opts.algorithm = 'deconvnet';
 94 | [~, deconvnet_viz, opts_new, NET, ~] = hand_specified_neuron_viz_fn(opts);
 95 | deconvnet_viz = mean(padarray(vl_imsc_am(deconvnet_viz), [1,1], 1, 'both'), 3);
 96 | 
 97 | %im_2_pp = opts_new.normalize(rgb2gray(imread(IMAGE_NAMES{2})));
 98 | %res = vl_simplenn(NET, im_2_pp);
 99 | %Fm2 = res(end).x;
100 | 
101 | res2 = vl_simplenn(NET, im_1_pp, abs(randn(size(Fm1), 'single')));
102 | deconvnet2_viz = mean(padarray(vl_imsc_am(res2(1).dzdx), [1, 1], 1, 'both'), 3);
103 | 
104 | 
105 | 
106 | %opts.algorithm = 'deconvnet';
107 | %[~, DeConvNet_viz, ~, ~, ~] = hand_specified_neuron_viz_fn(opts);
108 | %DeConvNet_viz = mean(padarray(vl_imsc_am(DeConvNet_viz), [1,1], 1, 'both'), 3);
109 | 
110 | %opts.algorithm = 'saliency';
111 | %[~, Saliency_viz, ~, ~, ~] = hand_specified_neuron_viz_fn(opts);
112 | %Saliency_viz = mean(padarray(vl_imsc_am(Saliency_viz), [1,1], 1, 'both'), 3);
113 | 
114 | %final_image = [padarray(im,[1,1],0,'both'), im0_viz, im1_viz, im2_viz;...
115 | %              zeros(size(im) + 2), DeConvNet_viz, Saliency_viz, TTT_viz];
116 | final_image = [padarray(im,[1,1],1,'both'), im0_viz, im2_viz, TTT2_viz, deconvnet2_viz];
117 | 
118 | imwrite(final_image, [FIGS_PATH, 'fourier.png']);


--------------------------------------------------------------------------------
/generate_all_figures.m:
--------------------------------------------------------------------------------
 1 | % This script calls all the other scripts to generate all the images in the
 2 | % paper
 3 | % Author: Aravindh Mahendran (Copyright 2016-17)
 4 | % University of Oxford
 5 | 
 6 | clear all; setup_eccv2016code;
 7 | 'fig splash'
 8 | fig_splash_all_images; % fig. 1
 9 | 
10 | 
11 | clear all; setup_eccv2016code;
12 | 'Spatial selectivity no arch effects'
13 | fig_spatial_selectivity_noarcheffects; % fig. 6
14 | 
15 | 
16 | clear all; setup_eccv2016code;
17 | 'effect of phase deconvnet'
18 | effect_of_phase_deconvnet; % fig. 4
19 | 
20 | 
21 | clear all; setup_eccv2016code;
22 | 'components_coding_phase_info_figure'
23 | components_coding_phase_info_figure; % fig. 3
24 | 
25 | 
26 | clear all; setup_eccv2016code;
27 | 'segmentation_qualitative_results_figure'
28 | segmentation_qualitative_results_figure; % fig. 8
29 | 
30 | 
31 | clear all; setup_eccv2016code;
32 | 'effect_of_architecture_figure'
33 | effect_of_architecture_figure; % fig. 5
34 | 
35 | 
36 | clear all; setup_eccv2016code;
37 | 'effect_of_lrn_nobackprop'
38 | effect_of_lrn_nobackprop; % supplementary material fig. 1
39 | 
40 | 
41 | clear all; setup_eccv2016code;
42 | 'fourier proper'
43 | fourier_phase; % fig. 7


--------------------------------------------------------------------------------
/hand_specified_neuron_viz_fn.m:
--------------------------------------------------------------------------------
  1 | function [viz, template, opts, NET, img] = hand_specified_neuron_viz_fn(opts)
  2 | % This is the main engine - it reads many different options and creates a
  3 | % reversed network and visualizes some neuron with that reversed network
  4 | % 
  5 | % opts can be like below :-
  6 | %
  7 | % randomizeWeights: 0 - whether or not to randomize network weights
  8 | %
  9 | % gpu: 0 - boolean - whether or not to use the gpu
 10 | %
 11 | % relus_to_change: [1x100 double] - Which relu's in the network do you want
 12 | % to change when creating the reversed architecture. if set to [1] it will
 13 | % change the first relu in the network, [1,2] will change the first 2.
 14 | % Similarly, [1,3] will change the first and the third relu but skip the
 15 | % second one. If left unchanged they do BP in the reversed architecture
 16 | %
 17 | % pools_to_change: [1 2 3 4 5] - same as above but for pooling layers
 18 | %
 19 | % convs_to_change: [1x100 double] - same as above but for convs. 
 20 | %
 21 | % neuron_I: Inf - This is the spatial location of which neuron to visualize
 22 | % Inf for the maximally active neuron
 23 | % -1 for the last row of neurons
 24 | % 1/2 or some fraction between 0 and 1 to pick a relative location in the
 25 | % spatial field of view
 26 | % or an integer to directly specify the neuron location/
 27 | %
 28 | % neuron_J: Inf 
 29 | %
 30 | % neuron_channel: Inf
 31 | %
 32 | % Please be very careful when using the special cases (inf, fraction, -1
 33 | % etc) ... read the code and debug to check which neuron got picked.
 34 | %
 35 | % imagePath: 'imagenet12-val/ILSVRC2012_val_00000170.JPEG' - which image to
 36 | % visualize over.
 37 | %
 38 | % modelPath: 'models/imagenet-vgg-verydeep-16.mat' - which forward network
 39 | % to visualize
 40 | % 
 41 | % layer: 36 - which layer in the forward network to visualize
 42 | %
 43 | % algorithm: 'deconvnet' - which algorithm to use. There are many many
 44 | % options. Please have a look at the switch case below. 
 45 | % Prominently - 'deconvnet', 'saliency', 'TTT'(for DeSaliNet) can be used.
 46 | %
 47 | % Opts can also be a more complete version like
 48 | % randomizeWeights: 0
 49 | % gpu: 0
 50 | % relus_to_change: [1x100 double]
 51 | % pools_to_change: [1 2 3 4 5]
 52 | % convs_to_change: [1x100 double]
 53 | % neuron_I: 1
 54 | % neuron_J: 1
 55 | % neuron_channel: 159
 56 | % imagePath: 'imagenet12-val/ILSVRC2012_val_00000170.JPEG'
 57 | % modelPath: 'models/imagenet-vgg-verydeep-16.mat'
 58 | % layer: 36
 59 | % algorithm: 'deconvnet'
 60 | % use_relu_mask: 0
 61 | % use_pooling_switches: 1
 62 | % relu_backward: 1
 63 | % lrn_nobackprop: 1
 64 | % conv_exciteonly: 0
 65 | % normalize: [function_handle]
 66 | % denormalize: @(x)bsxfun(@plus,x,NET.meta.normalization.averageImage)
 67 | %
 68 | % Again, since there are many options it is best to run an image and step
 69 | % through the code to see what is happening to make sure it is constructing
 70 | % the right network.
 71 | %
 72 | % Author: Aravindh Mahendran (Copyright 2016-17)
 73 | % University of Oxford
 74 | 
 75 | if(~isfield(opts, 'gpu'))
 76 |     opts.gpu = false;
 77 | end
 78 | 
 79 | %% Settings based on the chosen algorithm
 80 | switch opts.algorithm
 81 |     case {'deconvnet_noisy'}
 82 |         opts.use_noisy_relu = true;
 83 |         opts.lrn_nobackprop = true;
 84 |     case {'deconvnet', 'FTT'}
 85 |         opts.use_relu_mask = false;
 86 |         opts.use_pooling_switches = true;
 87 |         opts.relu_backward = true;
 88 |         opts.lrn_nobackprop = true;        
 89 |     case {'hybrid', 'TTT'}
 90 |         opts.use_relu_mask = true;
 91 |         opts.use_pooling_switches = true;
 92 |         opts.relu_backward = true;
 93 |         opts.lrn_nobackprop = true;
 94 |     case 'TTTF'
 95 |         opts.use_relu_mask = true;
 96 |         opts.use_pooling_switches = true;
 97 |         opts.relu_backward = true;
 98 |         opts.lrn_nobackprop = false;
 99 |     case 'TTTFT'
100 |         opts.use_relu_mask = true;
101 |         opts.use_pooling_switches = true;
102 |         opts.relu_backward = true;
103 |         opts.lrn_nobackprop = false;
104 |         opts.conv_exciteonly = true;
105 |     case {'saliency'}
106 |         opts.use_relu_mask = true;
107 |         opts.use_pooling_switches = true;
108 |         opts.relu_backward = false;
109 |         opts.lrn_nobackprop = false;
110 |     case {'TTF'}
111 |         opts.use_relu_mask = true;
112 |         opts.use_pooling_switches = true;
113 |         opts.relu_backward = false;
114 |         opts.lrn_nobackprop = true; % This is the only bit that is different from 'Saliency'
115 |     case {'TTFF'}
116 |         opts.use_relu_mask = true;
117 |         opts.use_pooling_switches = true;
118 |         opts.relu_backward = false;
119 |         opts.lrn_nobackprop = false; % we want to do the conservative and non conservative backprop together
120 |     case {'deconvnet_unpooltocenter', 'FFT'}
121 |         opts.use_relu_mask = false;
122 |         opts.use_pooling_switches = false;
123 |         opts.relu_backward = true;
124 |         opts.lrn_nobackprop = true;
125 |     case 'TFT'
126 |         opts.use_relu_mask = true;
127 |         opts.use_pooling_switches = false;
128 |         opts.relu_backward = true;
129 |         opts.lrn_nobackprop = true;
130 |     case 'TFF'
131 |         opts.use_relu_mask = true;
132 |         opts.use_pooling_switches = false;
133 |         opts.relu_backward = false;
134 |         opts.lrn_nobackprop = true;
135 |     case 'FFF'
136 |         opts.use_relu_mask = false;
137 |         opts.use_pooling_switches = false;
138 |         opts.relu_backward = false;
139 |         opts.lrn_nobackprop = true;
140 |     case 'FTF'
141 |         opts.use_relu_mask = false;
142 |         opts.use_pooling_switches = true;
143 |         opts.relu_backward = false;
144 |         opts.lrn_nobackprop = true;
145 |     otherwise
146 |         error('Unknown algorithm %s\n', opts.algorithm);
147 | end
148 | 
149 | if(strcmp(opts.algorithm, 'TTTFT'))
150 |     opts.conv_exciteonly = true;
151 | else
152 |     opts.conv_exciteonly = false;
153 | end
154 | 
155 | %% Load the network and prune down the layers
156 | 
157 | NET = load(opts.modelPath);
158 | NET = vl_simplenn_tidy(NET);
159 | NET.layers = NET.layers(1:opts.layer); 
160 | 
161 | %% Randomize the layer weights
162 | if(opts.randomizeWeights)
163 |     for i=1:numel(NET.layers)
164 |         if(strcmp(NET.layers{i}.type, 'conv'))
165 |             type = 'single';
166 |             sz = size(NET.layers{i}.weights{1});
167 |             h = sz(1); w = sz(2); in = sz(3); out = sz(4);
168 |             %sc = sqrt(2/(h*w*out)) ;
169 |             %NET.layers{i}.weights{1} = randn(h, w, in, out, type)*sc ;
170 |             %NET.layers{i}.weights{2} = zeros(out, 1, type);
171 |             sc = 0.01;
172 |             NET.layers{i}.weights{1} = randn(h, w, in, out, type)*sc;
173 |             NET.layers{i}.weights{2} = zeros(out, 1, type);
174 |         end
175 |     end
176 | end
177 | 
178 | %% Create a network without the local response normalization layers. 
179 | % This is used to find receptive field sizes for the hybrid and deconvnet method
180 | 
181 | NET_nolrn = NET;
182 | relu_layer.type = 'relu';
183 | relu_layer.leak = 0;
184 | for i=1:numel(NET_nolrn.layers)
185 |   if strcmp(NET_nolrn.layers{i}.type, 'normalize')
186 |     NET_nolrn.layers{i} = relu_layer;
187 |   end
188 |   if strcmp(NET_nolrn.layers{i}.type, 'lrn')
189 |     NET_nolrn.layers{i} = relu_layer;
190 |   end
191 |   if strcmp(NET_nolrn.layers{i}.type, 'conv')
192 |     NET_nolrn.layers{i}.weights{1} = ones(size(NET.layers{i}.weights{1}), 'single');
193 |     NET_nolrn.layers{i}.weights{2} = ones(size(NET.layers{i}.weights{2}), 'single');
194 |   end
195 | end
196 | NET_nolrn = vl_simplenn_tidy(NET_nolrn);
197 | 
198 | %% Collect network information
199 | % This is also mostly for receptive field measurement
200 | 
201 | NET_info = vl_simplenn_display(NET, 'inputSize', [NET.meta.normalization.imageSize(1:3), 1]);
202 | NET_nolrn_info = vl_simplenn_display(NET_nolrn, 'inputSize', [NET.meta.normalization.imageSize(1:3), 1]);
203 | NUM_CHANNELS = NET_info.dataSize(3, end);
204 | 
205 | %% Change the RELUs based on the opts
206 | 
207 | if(~isempty(opts.relus_to_change))
208 | 
209 |     
210 | counter = 0;
211 | for i=1:numel(NET.layers)
212 |     if (strcmp(NET.layers{i}.type, 'relu'))
213 |         counter = counter + 1;
214 |         if(find(opts.relus_to_change == counter))
215 |             if(isfield('opts', 'use_noisy_relu') && opts.use_noisy_relu)
216 |                 NET.layers{i}.type = 'relu_noisy';
217 |             elseif(opts.use_relu_mask && opts.relu_backward)
218 |                 NET.layers{i}.type = 'relu_eccv16';
219 |             elseif (~opts.use_relu_mask && opts.relu_backward)
220 |                 NET.layers{i}.type = 'relu_deconvnet';
221 |             elseif (~opts.use_relu_mask && ~opts.relu_backward)
222 |                 NET.layers{i}.type = 'relu_nobackprop';
223 |             end
224 |         end
225 |     end
226 | end
227 | 
228 | end
229 | 
230 | 
231 | if(opts.conv_exciteonly && ~isempty(opts.convs_to_change))
232 | 
233 |     
234 | counter = 0;
235 | for i=1:numel(NET.layers)
236 |     if (strcmp(NET.layers{i}.type, 'conv'))
237 |         counter = counter + 1;
238 |         if(find(opts.convs_to_change == counter))
239 |            NET.layers{i}.type = 'conv_exciteonly';
240 |         end
241 |     end
242 | end
243 | 
244 | end
245 | 
246 | if(~opts.use_pooling_switches && ~isempty(opts.pools_to_change))
247 | 
248 | counter = 0;
249 | for i=1:numel(NET.layers)
250 |     if (strcmp(NET.layers{i}.type, 'pool'))
251 |         counter = counter + 1;
252 |         if(find(opts.pools_to_change == counter))
253 |            NET.layers{i}.type = 'pool_center';
254 |         end
255 |     end
256 | end
257 | 
258 | end
259 | 
260 | if(opts.lrn_nobackprop)
261 |     for i=1:numel(NET.layers)
262 |         if strcmp(NET.layers{i}.type, 'normalize')
263 |             NET.layers{i}.type = 'normalize_nobackprop';
264 |         end
265 |         if strcmp(NET.layers{i}.type, 'lrn')
266 |             NET.layers{i}.type = 'lrn_nobackprop';
267 |         end
268 |     end
269 | end
270 | 
271 | %% Move NET to GPU
272 | if(opts.gpu)
273 |     NET_GPU = vl_simplenn_move(NET, 'gpu');
274 | else
275 |     NET_GPU = NET;
276 | end
277 | 
278 | %% Setup the variables and functions for operating the network
279 | 
280 | opts.normalize = @(x) bsxfun(@minus, single(resizencrop(x, NET.meta.normalization.imageSize(1:2))), NET.meta.normalization.averageImage);
281 | opts.denormalize = @(x) bsxfun(@plus, x, NET.meta.normalization.averageImage);
282 | 
283 | %% Run over the neurons and generate the visuals.
284 | 
285 | % read the image and evaluate the network on it
286 | img = imread(opts.imagePath);
287 | if(opts.gpu)
288 |     img_pp = gpuArray(opts.normalize(img)); %, NET.meta.normalization.averageImage);
289 | else
290 |     img_pp = opts.normalize(img);
291 | end
292 | 
293 | sz = NET_info.dataSize(:, end)';
294 | 
295 | % set the top derivative to be 1 at the position of maximal response
296 | if(opts.gpu)
297 |     dzdy = zeros(sz, 'single', 'gpuArray');
298 | else
299 |     dzdy = zeros(sz, 'single');
300 | end
301 | 
302 | % Pick the neuron baesd on the parameters in opts
303 | 
304 | if(opts.neuron_I < 1 && opts.neuron_I ~= -1)
305 |   opts.neuron_I = max( round(size(dzdy, 1) * opts.neuron_I), 1);
306 | elseif(opts.neuron_I == -1)
307 |   opts.neuron_I = size(dzdy, 1);
308 | end
309 | 
310 | if(opts.neuron_J < 1 && opts.neuron_J ~= -1)
311 |   opts.neuron_J = max(round(size(dzdy, 2) * opts.neuron_J), 1);
312 | elseif(opts.neuron_J == -1)
313 |   opts.neuron_J = size(dzdy, 2);
314 | end
315 | 
316 | if(isinf(opts.neuron_I)) % find the maximally firing neuron
317 |    res = vl_simplenn(NET_GPU, img_pp);
318 |    if(isinf(opts.neuron_channel))
319 |      [~, argmax] = max(res(end).x(:));
320 |      [opts.neuron_I, opts.neuron_J, opts.neuron_channel] = ...
321 |         ind2sub(size(res(end).x), argmax);
322 |    else
323 |      [~, argmax] = max( ...
324 |          reshape( res(end).x(:, :, opts.neuron_channel), [], 1) );
325 |      [opts.neuron_I, opts.neuron_J, ~] = ind2sub(size(res(end).x), argmax);
326 |    end
327 | else
328 |     res = vl_simplenn(NET_GPU, img_pp);
329 |     if(isinf(opts.neuron_channel))
330 |         [~, opts.neuron_channel] = max( ...
331 |             res(end).x(opts.neuron_I, opts.neuron_J, :), [], 3);
332 |     end
333 | end
334 | 
335 | 
336 | if (opts.neuron_channel ~= -1)
337 | 
338 |   dzdy(opts.neuron_I, opts.neuron_J, opts.neuron_channel) = 1;
339 |   res2 = vl_simplenn(NET_GPU, img_pp, dzdy);
340 |   template = gather(res2(1).dzdx);
341 | 
342 | else
343 |   while(true)
344 |       neuron_channel = randi(sz(3), 1);
345 |       dzdy(opts.neuron_I, opts.neuron_J, neuron_channel) = 1;
346 |       res2 = vl_simplenn(NET_GPU, img_pp, dzdy);
347 |       template = gather(res2(1).dzdx);
348 |       pos = find(template ~= 0, 1);
349 |       if(isempty(pos))
350 |           dzdy(opts.neuron_I, opts.neuron_J, neuron_channel) = 0;
351 |       else
352 |           break;
353 |       end
354 |   end
355 |   opts.neuron_channel = neuron_channel;
356 | end
357 | 
358 | %rf_start_pos = [opts.neuron_I - 1; opts.neuron_J - 1] .* ...
359 | %    NET_nolrn_info.receptiveFieldStride(:, end) + ...
360 | %    NET_nolrn_info.receptiveFieldOffset(:, end) - ...
361 | %    ceil(NET_nolrn_info.receptiveFieldSize(:, end) / 2) + 1;
362 | %rf_start_pos = max(rf_start_pos, 1);
363 | %
364 | %rf_end_pos = min(rf_start_pos + NET_nolrn_info.receptiveFieldSize(:, end) - 1, ...
365 | %    NET.meta.normalization.imageSize(1:2)') ;
366 | %
367 | %viz = vl_imsc(template(rf_start_pos(1):rf_end_pos(1), ...
368 | %                       rf_start_pos(2):rf_end_pos(2), :));
369 | 
370 | img = resizencrop(img, NET.meta.normalization.imageSize(1:2));
371 | 
372 | rf_start_pos = [opts.neuron_I - 1; opts.neuron_J - 1] .* ...
373 |     NET_nolrn_info.receptiveFieldStride(:, end) + ...
374 |     NET_nolrn_info.receptiveFieldOffset(:, end) - ...
375 |     ceil(NET_nolrn_info.receptiveFieldSize(:, end) / 2) + 1;
376 | 
377 | rf_start_pos =ceil(rf_start_pos);
378 | 
379 | rf_end_pos = rf_start_pos + NET_nolrn_info.receptiveFieldSize(:, end) - 1;
380 | 
381 | TOP = max(1 - rf_start_pos(1), 0);
382 | LEFT = max(1 - rf_start_pos(2), 0);
383 | template_t = padarray(template, [TOP, LEFT], 0, 'pre');
384 | img = padarray(img, [TOP, LEFT, 0], 0, 'pre');
385 | rf_start_pos = rf_start_pos + [TOP; LEFT];
386 | rf_end_pos = rf_end_pos + [TOP; LEFT];
387 | 
388 | BOTTOM = max(rf_end_pos(1) - NET.meta.normalization.imageSize(1), 0);
389 | RIGHT = max(rf_end_pos(2) - NET.meta.normalization.imageSize(2), 0);
390 | template_t = padarray(template_t, [BOTTOM, RIGHT], 0, 'post');
391 | img = padarray(img, [BOTTOM, RIGHT, 0], 0, 'post');
392 | 
393 | viz = vl_imsc(template_t(rf_start_pos(1):rf_end_pos(1), ...
394 |                        rf_start_pos(2):rf_end_pos(2), :));                   
395 | img = uint8(img(rf_start_pos(1):rf_end_pos(1), ...
396 |                        rf_start_pos(2):rf_end_pos(2), :));
397 | 
398 | end
399 | 


--------------------------------------------------------------------------------
/resizencrop.m:
--------------------------------------------------------------------------------
 1 | function IMG_edited = resizencrop(IMG, imsize)
 2 | % IMG_edited = resizencrop(IMG, imsize) - resize and crop the image
 3 | %    This helper function transforms the image so as to preserve the aspect
 4 | % ratio before cropping off the extra bits.
 5 | 
 6 | % Resize preserving aspect ratio
 7 | if(size(IMG, 1) / size(IMG,2) > imsize(1) / imsize(2))
 8 |   % Image is taller than required
 9 |   IMG = imresize(IMG, [nan, imsize(2)]);
10 | else
11 |   % Image is broader than required
12 |   IMG = imresize(IMG, [imsize(1), nan]);
13 | end
14 | 
15 | % Crop off the sides so as to get it right
16 | offset_i = floor((size(IMG, 1) - imsize(1))/2) + 1;
17 | offset_j = floor((size(IMG, 2) - imsize(2))/2) + 1;
18 | 
19 | IMG_edited = IMG(offset_i:offset_i + imsize(1) - 1, ...
20 |     offset_j : offset_j + imsize(2) - 1, :);
21 | 
22 | end
23 | 


--------------------------------------------------------------------------------
/saliency/HackyConv.m:
--------------------------------------------------------------------------------
 1 | classdef HackyConv < dagnn.Filter
 2 |   properties
 3 |     size = [0 0 0 0]
 4 |     hasBias = true
 5 |     opts = {'cuDNN'}
 6 |     scale = 1
 7 |   end
 8 | 
 9 |   methods
10 |     function outputs = forward(obj, inputs, params)
11 |       if ~obj.hasBias, params{2} = [] ; end
12 |       outputs{1} = vl_nnconv(...
13 |         inputs{1}, params{1}, params{2}, ...
14 |         'pad', obj.pad, ...
15 |         'stride', obj.stride, ...
16 |         obj.opts{:}) ;
17 |     end
18 | 
19 |     function [derInputs, derParams] = backward(obj, inputs, params, derOutputs)
20 |       if ~obj.hasBias
21 |          params{2} = [] ; 
22 |          derParams{2} = [];
23 |          derIntermediate = derOutputs{1};
24 |       else
25 |          % First the bias correction
26 |          theeye = reshape(ones(size(params{1}, 4), 1, 'single'), ...
27 |                      [1, 1, 1, size(params{1}, 4)]);
28 |          if(isa(params{1}, 'gpuArray'))
29 |            theeye = gpuArray(theeye);
30 |          end
31 |          derIntermediate = vl_nnconv(derOutputs{1},...
32 |            theeye, -params{2}, 'pad', 0, 'stride', 1, obj.opts{:});
33 |          derParams{2} = [];
34 |       end
35 | 
36 |       % Next the actual deconvolution
37 |       [derInputs{1}, derParams{1}, ~] = vl_nnconv(...
38 |         inputs{1}, params{1}, [], derIntermediate, ...
39 |         'pad', obj.pad, ...
40 |         'stride', obj.stride, ...
41 |         obj.opts{:}) ;
42 | 
43 |       % scale correction
44 |       derInputs{1} = derInputs{1} / obj.scale;
45 |     end
46 | 
47 |     function kernelSize = getKernelSize(obj)
48 |       kernelSize = obj.size(1:2) ;
49 |     end
50 | 
51 |     function outputSizes = getOutputSizes(obj, inputSizes)
52 |       outputSizes = getOutputSizes@dagnn.Filter(obj, inputSizes) ;
53 |       outputSizes{1}(3) = obj.size(4) ;
54 |     end
55 | 
56 |     function params = initParams(obj)
57 |       sc = sqrt(2 / prod(obj.size(1:3))) ;
58 |       params{1} = randn(obj.size,'single') * sc ;
59 |       if obj.hasBias
60 |         params{2} = zeros(obj.size(4),1,'single') * sc ;
61 |       end
62 |     end
63 | 
64 |     function set.size(obj, ksize)
65 |       % make sure that ksize has 4 dimensions
66 |       ksize = [ksize(:)' 1 1 1 1] ;
67 |       obj.size = ksize(1:4) ;
68 |     end
69 | 
70 |     function obj = Conv(varargin)
71 |       obj.load(varargin) ;
72 |       % normalize field by implicitly calling setters defined in
73 |       % dagnn.Filter and here
74 |       obj.size = obj.size ;
75 |       obj.stride = obj.stride ;
76 |       obj.pad = obj.pad ;
77 |     end
78 |   end
79 | end
80 | 


--------------------------------------------------------------------------------
/saliency/HackyLRN.m:
--------------------------------------------------------------------------------
 1 | classdef HackyLRN < dagnn.ElementWise
 2 |   properties
 3 |     param = [5 1 0.0001/5 0.75]
 4 |   end
 5 | 
 6 |   methods
 7 |     function outputs = forward(obj, inputs, params)
 8 |       outputs{1} = vl_nnnormalize(inputs{1}, obj.param) ;
 9 |     end
10 | 
11 |     function [derInputs, derParams] = backward(obj, inputs, param, derOutputs)
12 |       derInputs{1} = derOutputs{1};
13 |       derParams = {} ;
14 |     end
15 | 
16 |     function obj = LRN(varargin)
17 |       obj.load(varargin) ;
18 |     end
19 |   end
20 | end
21 | 


--------------------------------------------------------------------------------
/saliency/HackyReLU.m:
--------------------------------------------------------------------------------
 1 | classdef HackyReLU < dagnn.ElementWise
 2 |   properties
 3 |     leak = 0
 4 |     opts = {}
 5 |     hackType = 'none'
 6 |   end
 7 | 
 8 |   methods
 9 |     function outputs = forward(obj, inputs, params)
10 |       outputs{1} = vl_nnrelu(inputs{1}, [], ...
11 |                              'leak', obj.leak, obj.opts{:}) ;
12 |     end
13 | 
14 |     function [derInputs, derParams] = backward(obj, inputs, params, derOutputs)
15 |       derInputs{1} = vl_nnrelu(inputs{1}, derOutputs{1}, ...
16 |                                'leak', obj.leak, ...
17 |                                obj.opts{:}) ;
18 |       derParams = {} ;
19 |       switch lower(obj.hackType)
20 |         case 'us'
21 |           derInputs{1} = max(derInputs{1}, 0) ;
22 |         case 'deconvnet'
23 |           derInputs{1} = max(derOutputs{1}, 0) ;
24 |         case 'none'
25 |         otherwise
26 |           assert(false) ;
27 |       end
28 |     end
29 | 
30 |     function obj = HackyReLU(varargin)
31 |       obj.load(varargin) ;
32 |     end
33 |   end
34 | end
35 | 


--------------------------------------------------------------------------------
/saliency/baseline3.m:
--------------------------------------------------------------------------------
 1 | function [mask, mask_signed] = baseline3(im0)
 2 | 
 3 | sz = size(im0);
 4 | mask = zeros(sz(1:2), 'single');
 5 | 
 6 | f = fspecial('gaussian', [125, 125], 24);
 7 | t = 62;
 8 | mask(end/2 - t:end/2 + t, end/2 - t:end/2 + t) = ...
 9 |   mask(end/2 - t:end/2 + t, end/2 - t:end/2 + t) + f;
10 | 
11 | t = 63;
12 | mask(1:t,1:t) = -f(t:end, t:end);
13 | mask(end-t+1:end,1:t) = -f(1:t, t:end);
14 | mask(end-t+1:end,end-t+1:end) = -f(1:t,1:t);
15 | mask(1:t,end-t+1:end) = -f(t:end,1:t);
16 | 
17 | mask = vl_imsc(mask);
18 | mask_signed = mask;
19 | 
20 | end % end mask = baseline3(im0);
21 | 


--------------------------------------------------------------------------------
/saliency/exp_ksseg_production.m:
--------------------------------------------------------------------------------
  1 | function exps = exp_ksseg_production(hackLRN, prefix, method, network)
  2 | % Code to perform segmentation
  3 | % hackLRN is a boolean (True/False) on whether or not to use Identity
  4 | % instead of LRN^{BP}. If True then Identity is used in the reversed
  5 | % architecture 
  6 | % prefix - results will be dumped in the folder data/ferrari/[prefix]
  7 | % method - 'am' for DeSaliNet, 'ks' for Karen's Saliency and baseline3 for
  8 | % the gaussian baseline, 'dc' for DeconvNets
  9 | % network - 'alex' for alexnet and 'vgg-vd-16' for VGG-verydeep-16
 10 | % 
 11 | % The return value is not useful. It is simply the list of experiments -
 12 | % one for each image.
 13 | %
 14 | % Pre-requisites
 15 | % data/ferrari should contain imdb.mat and images/ - the imdb and images
 16 | % ../models/imagenet-caffe-alex.mat should contain the alexnet model
 17 | % ../models/imagenet-vgg-verydeep-16.mat should contain the VGG-verydeep-16
 18 | % model
 19 | %
 20 | % Author: Aravindh Mahendran (Copyright 2016-17)
 21 | % University of Oxford
 22 | 
 23 | subset = [];
 24 | opts.parallel = true;
 25 | 
 26 | opts.prefix = prefix;
 27 | 
 28 | opts.method = method;
 29 | 
 30 | opts.expDir = fullfile('data/ferrari/', opts.prefix) ;
 31 | 
 32 | imdb = load('data/ferrari/imdb.mat') ;
 33 | mkdir(opts.expDir) ;
 34 | 
 35 | exps = {} ;
 36 | for i = 1:numel(imdb.images.name)
 37 |   name = imdb.images.name{i} ;
 38 |   exp.imagePath = sprintf(imdb.paths.image, name) ;
 39 |   exp.resPath = sprintf('%s/%s.mat', opts.expDir, name) ;
 40 |   exp.fig1Path = sprintf('%s/%s-1.pdf', opts.expDir, name) ;
 41 |   exp.fig2Path = sprintf('%s/%s-2.pdf', opts.expDir, name) ;
 42 |   exp.fig3Path = sprintf('%s/%s-3.pdf', opts.expDir, name) ;
 43 | 
 44 | 
 45 |   exp.network = network;
 46 | 
 47 |   exp.alreadyDone = exist(exp.resPath,'file') ;
 48 |   exps{end+1} = exp ;
 49 | end
 50 | 
 51 | if isempty(subset)
 52 |   subset = 1:numel(imdb.images.name) ;
 53 | end
 54 | numel(subset)
 55 | 
 56 | switch exp.network
 57 |   case 'alex'
 58 |     net = dagnn.DagNN.fromSimpleNN(...
 59 |       load('../models/imagenet-caffe-alex.mat'), ...
 60 |       'canonicalNames', true) ;
 61 | 
 62 |   case 'vgg-vd-16'
 63 |     net = dagnn.DagNN.fromSimpleNN(...
 64 |       load('../models/imagenet-vgg-verydeep-16.mat'), ...
 65 |       'canonicalNames', true) ;
 66 | end
 67 | 
 68 | switch opts.method
 69 |     case 'ks'
 70 |         pred0val = 1;
 71 | 
 72 |     case 'am'
 73 |         pred0val = 1;
 74 |         opts.hackType = 'us';
 75 |         layerNames = {net.layers.name} ;
 76 |         for l = 1:numel(layerNames)
 77 |             li = net.getLayerIndex(layerNames{l}) ;
 78 |             if isa(net.layers(li).block, 'dagnn.ReLU')
 79 |               net.addLayer([layerNames{l} 'hacky'], ...
 80 |                    HackyReLU('hackType', opts.hackType), ...
 81 |                    net.layers(li).inputs, ...
 82 |                    net.layers(li).outputs, ...
 83 |                    {}) ;
 84 |               net.removeLayer(layerNames{l}) ;
 85 |             elseif isa(net.layers(li).block, 'dagnn.LRN') && hackLRN
 86 |               t = HackyLRN();
 87 |               t.param = net.layers(li).block.param;
 88 |               net.addLayer([layerNames{l} 'hacky'], t, ...
 89 |                    net.layers(li).inputs, ...
 90 |                    net.layers(li).outputs, ...
 91 |                    {}) ;
 92 |               net.removeLayer(layerNames{l}) ;
 93 |             end
 94 |         end
 95 | 
 96 |     case 'dc'
 97 |         pred0val = 1;
 98 |         opts.hackType = 'deconvnet';
 99 |         layerNames = {net.layers.name} ;
100 |         for l = 1:numel(layerNames)
101 |             li = net.getLayerIndex(layerNames{l}) ;
102 |             if isa(net.layers(li).block, 'dagnn.ReLU')
103 |               net.addLayer([layerNames{l} 'hacky'], ...
104 |                    HackyReLU('hackType', opts.hackType), ...
105 |                    net.layers(li).inputs, ...
106 |                    net.layers(li).outputs, ...
107 |                    {}) ;
108 |               net.removeLayer(layerNames{l}) ;
109 |             elseif isa(net.layers(li).block, 'dagnn.LRN') && hackLRN
110 |               t = HackyLRN();
111 |               t.param = net.layers(li).block.param;
112 |               net.addLayer([layerNames{l} 'hacky'], t, ...
113 |                    net.layers(li).inputs, ...
114 |                    net.layers(li).outputs, ...
115 |                    {}) ;
116 |               net.removeLayer(layerNames{l}) ;
117 |             end
118 |         end
119 |   
120 |     otherwise
121 |       pred0val = nan;
122 |       net =struct();
123 | end
124 | 
125 | if(opts.parallel)
126 |  ' Using parallel for'
127 |   parfor i = subset
128 |     ts = tic;
129 |     im0 = imread(exps{i}.imagePath) ;
130 |     fprintf(1, '%s\n', exps{i}.imagePath);
131 |     [im0, scalingFactor] = ksresize(im0);
132 | 
133 |     switch opts.method
134 |       case {'ks', 'am', 'dc'}
135 |          [mask, mask_signed] = kssaliency_fcn_proper(net, im0, pred0val) ;
136 |       case 'baseline3'
137 |          [mask, mask_signed] = baseline3(im0);
138 |       otherwise
139 |          assert(false) ;
140 |     end
141 | 
142 |     res = salseg(im0/255, mask, opts.method) ;
143 |     res.mask_signed = mask_signed;
144 | 
145 |     dothesave(exps{i}, '-struct', res) ;
146 |     %print(1,'-dpdf', exps{i}.fig1Path) ;
147 |     %print(2,'-dpdf', exps{i}.fig2Path) ;
148 |     %print(101,'-dpdf', exps{i}.fig3Path) ;
149 | 
150 |     toc(ts);
151 |   end
152 | else
153 |   for i = subset
154 |     ts = tic;
155 |     im0 = imread(exps{i}.imagePath) ;
156 |     fprintf(1, '%d %s\n', i, exps{i}.imagePath);
157 |     [im0, scalingFactor] = ksresize(im0);
158 | 
159 |     switch opts.method
160 |       case {'ks','am', 'dc'}
161 |         [mask, mask_signed] = kssaliency_fcn_proper(net, im0, pred0val) ;
162 |       case {'baseline1'}
163 |         mask = baseline1(im0);
164 |       case {'baseline2'}
165 |         mask = baseline2(im0);
166 |       case 'baseline3'
167 |          [mask, mask_signed] = baseline3(im0);
168 |       otherwise
169 |         assert(false) ;
170 |     end
171 |     res = salseg(im0/255, mask, opts.method) ;
172 |     res.mask_signed = mask_signed;
173 | 
174 |     save(exps{i}.resPath, '-struct', 'res') ;
175 |     %print(1,'-dpdf', exps{i}.fig1Path) ;
176 |     print(2,'-dpdf', exps{i}.fig2Path) ;
177 |     print(101,'-dpdf', exps{i}.fig3Path) ;
178 | 
179 |     toc(ts);
180 |   end
181 | end
182 | 
183 | function dothesave(a,b,res)
184 |   save(a.resPath,b,'res');
185 |   print(2,'-dpdf',a.fig2Path);
186 |   print(101,'-dpdf',a.fig3Path);
187 | 


--------------------------------------------------------------------------------
/saliency/exp_seg_eval.m:
--------------------------------------------------------------------------------
 1 | function exp_seg_eval(prefix, subset, silent)
 2 | % evaluates segmentation results dumped in data/ferrari/[prefix] folder
 3 | % subset is useful to choose a subset of the examples for evaluation
 4 | % silent - true if you don't want to output anything into the terminal.
 5 | 
 6 | if(~exist('silent', 'var'))
 7 |   silent = false;
 8 | end
 9 | 
10 | opts.prefix = prefix; 
11 | 
12 | opts.expDir = fullfile('data/ferrari/', opts.prefix) ;
13 | 
14 | imdb = load('data/ferrari/imdb.mat') ;
15 | 
16 | if(nargin < 2 || isempty(subset))
17 |   subset = 1:numel(imdb.images.name);
18 | end
19 | 
20 | imdb.images.class = imdb.image.class ;
21 | K = numel(imdb.classes.name) ;
22 | confusion = zeros(K) ;
23 | 
24 | acc = zeros(1,K) ;
25 | iou = zeros(1,K) ;
26 | precision = zeros(1,K);
27 | mass = zeros(1,K) ;
28 | found = 0;
29 | 
30 | for idx = 1:numel(subset)
31 | 
32 |   i = subset(idx);
33 | 
34 |   name = imdb.images.name{i} ;
35 | 
36 |   exp.imagePath = sprintf(imdb.paths.image, name) ;
37 |   exp.resPath = sprintf('%s/%s.mat', opts.expDir, name) ;
38 | 
39 |   if exist(exp.resPath)
40 |     found= 1;
41 |     res = load(exp.resPath) ;
42 |     [acc_, iou_, precision_] = seg_eval(imdb, i, res.seg) ;
43 |     c = imdb.images.class(i) ;
44 |     acc(c) = (mass(c) * acc(c) + acc_) / (mass(c) + 1) ;
45 |     iou(c) = (mass(c) * iou(c) + iou_) / (mass(c) + 1) ;
46 |     precision(c) = (mass(c) * precision(c) + precision_) / (mass(c) + 1) ;
47 |     mass(c) = mass(c) + 1 ;
48 |   else
49 |     found = 0;
50 |     acc_ = 0 ;
51 |     iou_ = 0 ;
52 |     precision_ = 0;
53 |   end
54 | 
55 | if(~silent)
56 | if(found)
57 |   fprintf('%05d, %05d, %8.2f, %8.2f, %8.2f, %8.2f, %8.2f, %8.2f, %s %s\n', ...
58 |           i, numel(imdb.images.name), ...
59 |           mean(acc(mass>0)), acc_, mean(iou(mass>0)), iou_, ...
60 |           mean(precision(mass>0)), precision_, ...
61 |           name, ' ') ;
62 | else
63 |   fprintf('%05d, %05d, %8.2f, %8.2f, %8.2f, %8.2f, %8.2f, %8.2f, %s %s\n', ...
64 |           i, numel(imdb.images.name), ...
65 |           mean(acc(mass>0)), acc_, mean(iou(mass>0)), iou_, ...
66 |           mean(precision(mass>0)), precision_, ...
67 |           name, 'not found') ;
68 | end
69 | end
70 | 
71 | end
72 | 
73 | fprintf(1, '%s %8.2f, %8.2f\n', prefix, mean(acc(mass > 0)), mean(iou(mass>0)));
74 | 


--------------------------------------------------------------------------------
/saliency/exp_seg_unpack.m:
--------------------------------------------------------------------------------
 1 | function exp_seg_unpack()
 2 | 
 3 | load('data/ferrari/gtsegs_ijcv.mat','value') ;
 4 | mkdir('data/ferrari/images') ;
 5 | mkdir('data/ferrari/segs') ;
 6 | 
 7 | imdb.paths.image = 'data/ferrari/images/%s.jpg' ;
 8 | imdb.paths.seg = 'data/ferrari/segs/%s-v%d.png' ;
 9 | imdb.images.name = value.id ;
10 | imdb.images.numSegs = zeros(size(imdb.images.name)) ;
11 | [imdb.classes.name,~,imdb.images.class] = unique(value.target) ;
12 | 
13 | for i = 1:numel(value.img)
14 |   imagePath = sprintf(imdb.paths.image, imdb.images.name{i}) ;
15 |   imwrite(value.img{i}, imagePath) ;
16 |   imdb.images.numSegs(i) = numel(value.gt{i}) ;
17 | 
18 |   for j = 1:imdb.images.numSegs(i)
19 |     segPath = sprintf(imdb.paths.seg, imdb.images.name{i}, j) ;
20 |     imwrite(value.gt{i}{j}, segPath) ;
21 |   end
22 | end
23 | 
24 | save('data/ferrari/imdb.mat', '-struct', 'imdb') ;
25 | 


--------------------------------------------------------------------------------
/saliency/ksresize.m:
--------------------------------------------------------------------------------
1 | function [im, sc] = ksresize(im)
2 | sz = [size(im,1) size(im,2)] ;
3 | sc = 256/min(sz) ;
4 | sz = round(sc*sz) ;
5 | im = imresize(im, sz, 'bicubic') ;
6 | im = single(im) ;
7 | 


--------------------------------------------------------------------------------
/saliency/kssaliency_fcn_proper.m:
--------------------------------------------------------------------------------
 1 | function [mask, mask_signed] = kssaliency_fcn_proper(net, im0, pred0val)
 2 | 
 3 | im_mean = mean(mean(net.meta.normalization.averageImage,1),2) ;
 4 | im0 = bsxfun(@minus, im0, im_mean) ;
 5 | 
 6 | % get class
 7 | net.vars(net.getVarIndex('prediction')).precious = true ;
 8 | net.eval({'input', im0}) ;
 9 | prediction = net.vars(net.getVarIndex('prediction')).value;
10 | [~,label] = max(squeeze(max(max(prediction, [], 1), [], 2)));
11 | 
12 | pred0 = zeros(size(prediction),'single') ;
13 | idx = find(prediction(:,:,label) > 0);
14 | pred0_plate = zeros(size(prediction, 1), size(prediction, 2), 'single');
15 | pred0_plate(idx) = pred0val;
16 | pred0(:,:,label) = pred0_plate ;
17 | 
18 | net.eval({'input', im0}, {'prediction', pred0});
19 | dd = net.vars(net.getVarIndex('input')).der;
20 | mask = max(abs(dd), [], 3);
21 | mask_signed = dd;
22 | 


--------------------------------------------------------------------------------
/saliency/ksseg.m:
--------------------------------------------------------------------------------
  1 | function [seg, segBox, labels, labelsClamp] = ksseg(img, imgSal, varargin)
  2 | % I don't understand this code. Sorry :(
  3 | 
  4 | opts.bVis = true ;
  5 | opts.quantMax = 0.95;
  6 | opts.quantMin = 0.3;
  7 | opts.quantMaxClamp = 0.995;
  8 | opts.quantMinClamp = 0.005;
  9 | opts.ksCompatible = true ;
 10 | opts.getLargestCC = true ;
 11 | opts.gmmNmix_fg = 5;
 12 | opts.gmmNmix_bg = 5;
 13 | 
 14 | opts = vl_argparse(opts, varargin) ;
 15 | 
 16 | if opts.ksCompatible
 17 |   % karen's code assumes the saliency is a square (256 x256 with
 18 |   % accumulation?)
 19 |   imgSal = imgSal(...
 20 |     round((1:256)+(size(imgSal,1)-256)/2), ...
 21 |     round((1:256)+(size(imgSal,2)-256)/2), ...
 22 |     :) ;
 23 | end
 24 | 
 25 | % thresholds
 26 | TMax = quantile(imgSal(:), opts.quantMax);
 27 | TMin = quantile(imgSal(:), opts.quantMin);
 28 | 
 29 | useClamp = true;
 30 | if opts.quantMaxClamp <= 1
 31 |   TMaxClamp = quantile(imgSal(:), opts.quantMaxClamp);
 32 | else
 33 |   TMaxClamp = +inf ;
 34 | end
 35 | if opts.quantMinClamp >= 0
 36 |   TMinClamp = quantile(imgSal(:), opts.quantMinClamp);
 37 | else
 38 |   TMinClamp = -inf ;
 39 | end
 40 | 
 41 | w = size(img, 2);
 42 | h = size(img, 1);
 43 | assert(w == 256 || h == 256);
 44 | labels = zeros(h, w, 'uint8'); % by def = ignore
 45 | labelsClamp = zeros(h, w, 'uint8'); % by def = ignore
 46 | 
 47 | % find sub-region for which the saliency is computed
 48 | if opts.ksCompatible
 49 |   imgSalSize = 256 ;
 50 |   if w > h
 51 |     y0 = 1 + 0.5 * (256 - imgSalSize);
 52 |     x0 = 1 + 0.5 * (256 - imgSalSize) + round(0.5 * (w - 256));
 53 |   else
 54 |     x0 = 1 + 0.5 * (256 - imgSalSize);
 55 |     y0 = 1 + 0.5 * (256 - imgSalSize) + round(0.5 * (h - 256));
 56 |   end
 57 |   y1 = y0 + imgSalSize - 1;
 58 |   x1 = x0 + imgSalSize - 1;
 59 | 
 60 |   labelsCrop = labels(y0:y1, x0:x1);
 61 |   labelsCrop(imgSal >= TMax) = 1;
 62 |   labelsCrop(imgSal <= TMin) = 2;
 63 |   labels(y0:y1, x0:x1) = labelsCrop;
 64 | 
 65 |   if useClamp
 66 |     labelsCrop = labelsClamp(y0:y1, x0:x1);
 67 |     labelsCrop(imgSal >= TMaxClamp) = 1;
 68 |     labelsCrop(imgSal <= TMinClamp) = 2;
 69 |     labelsClamp(y0:y1, x0:x1) = labelsCrop;
 70 |   end
 71 | else
 72 |   % labels: 2=bg, 1=fg 0=ignore?
 73 |   labels(imgSal >= TMax) = 1;
 74 |   labels(imgSal <= TMin) = 2;
 75 |   if useClamp
 76 |     labelsClamp(imgSal >= TMaxClamp) = 1 ; % fg
 77 |     labelsClamp(imgSal <= TMinClamp) = 2 ; % bg
 78 |   end
 79 | end
 80 | 
 81 | % Make segmentation options
 82 | Params = bj.segOpts();
 83 | Params.gcGamma = 150;
 84 | Params.gmmNmix_fg = opts.gmmNmix_fg ;
 85 | Params.gmmNmix_bg = opts.gmmNmix_bg ;
 86 | Params.postProcess = true;
 87 | 
 88 | % Intialize the segmentation object
 89 | segH = bj.segEngine(0,Params);
 90 | 
 91 | % preProcess image
 92 | segH.preProcess(im2double(img)); % Only takes in double images
 93 | 
 94 | % Get the first segmentation
 95 | segH.start(labels, labelsClamp);
 96 | seg = segH.seg;
 97 | 
 98 | % largest connected component
 99 | seg0 = seg ;
100 | if opts.getLargestCC
101 |   cc = bwconncomp(seg) ;
102 |   numPixels = cellfun(@numel, cc.PixelIdxList);
103 |   [~, maxIdx] = max(numPixels);
104 |   seg(:) = 0;
105 |   seg(cc.PixelIdxList{maxIdx}) = 1 ;
106 | end
107 | 
108 | % box
109 | [i,j] = find(seg);
110 | segBox = [min(j(:)); min(i(:)); max(j(:)); max(i(:))];
111 | 
112 | if opts.bVis
113 |   figure(101) ; clf ;
114 |   subplot(2,3,1) ; imagesc(img) ; axis image ; hold on ;
115 |   rectangle('Position', [segBox(1:2); segBox(3:4) - segBox(1:2) + 1], 'EdgeColor', 'yellow', 'LineWidth', 2);
116 |   subplot(2,3,2) ; imagesc(imgSal) ; axis image ; colormap gray
117 |   subplot(2,3,3) ; imagesc(labels,[0 2]) ; axis image ; title('Soft labels') ;
118 |   subplot(2,3,4) ; imagesc(labelsClamp,[0 2]) ; axis image ; title('Hard labels') ;
119 |   subplot(2,3,5) ; imagesc(seg) ; axis image ;
120 |   subplot(2,3,6) ; imagesc(seg0) ; axis image ;
121 | end
122 | 
123 | % starts from 0, convert back to full-res
124 | %segBox = (segBox - 1) * scalingFactor ;
125 | 


--------------------------------------------------------------------------------
/saliency/resizencrop.m:
--------------------------------------------------------------------------------
 1 | function IMG_edited = resizencrop(IMG, imsize)
 2 | % IMG_edited = resizencrop(IMG, imsize) - resize and crop the image
 3 | %    This helper function transforms the image so as to preserve the aspect ratio before cropping off the extra bits.
 4 | 
 5 | % Resize preserving aspect ratio
 6 | if(size(IMG, 1) / size(IMG,2) > imsize(1) / imsize(2))
 7 |   % Image is taller than required
 8 |   IMG = imresize(IMG, [nan, imsize(2)]);
 9 | else
10 |   % Image is broader than required
11 |   IMG = imresize(IMG, [imsize(1), nan]);
12 | end
13 | 
14 | % Crop off the sides so as to get it right
15 | offset_i = floor((size(IMG, 1) - imsize(1))/2) + 1;
16 | offset_j = floor((size(IMG, 2) - imsize(2))/2) + 1;
17 | 
18 | IMG_edited = IMG(offset_i:offset_i + imsize(1) - 1, ...
19 |     offset_j : offset_j + imsize(2) - 1, :);
20 | 
21 | end
22 | 


--------------------------------------------------------------------------------
/saliency/run_2.m:
--------------------------------------------------------------------------------
 1 | % We need to call the following function with different inputs to complete
 2 | % all the experiments
 3 | % function exps = exp_ksseg_production(hackLRN, prefix, method, network)
 4 | % Uncomment different lines in the code below to get different parts of table 1.
 5 | %
 6 | % exp_ksseg_production runs the segmentation code and dumps the results
 7 | % into data/ferrari/prefix where prefix is one of the input arguments to
 8 | % this function
 9 | %
10 | % exp_seg_eval - evaluates the segmentation results dumped above and
11 | % outputs 
12 | % prefix Per-pixel-accuracy and 
13 | %
14 | % pre-requisites 
15 | % gsc contains the gsc-1.2 from - http://www.robots.ox.ac.uk/~vgg/research/iseg/
16 | % + pre-requisites in exp_ksseg_production.m
17 | % + pre-requisites in exp_seg_eval.m
18 | % 
19 | % Author: Aravindh Mahendran (Copyright 2016-17)
20 | % University of Oxford
21 | 
22 | 
23 | 
24 | 
25 | 
26 | 
27 | 
28 | run gsc/setup.m
29 | 
30 | %exps = exp_ksseg_production(true, 'sal-dc-101-vd'  , 'dc', 'vgg-vd-16');
31 | %exps = exp_ksseg_production(false, 'sal-ks-101-alex', 'ks', 'alex');
32 | %exps = exp_ksseg_production(false, 'sal-ks-101-vd'  , 'ks', 'vgg-vd-16');
33 | exps = exp_ksseg_production(false, 'sal-am-101-alex', 'am', 'alex');
34 | %exps = exp_ksseg_production(false, 'sal-am-101-vd'  , 'am', 'vgg-vd-16');
35 | %exps = exp_ksseg_production(false, 'sal-baseline3-101', 'baseline3', '');
36 | 
37 | %exp_seg_eval('sal-ks-101-alex', [], true);
38 | %exp_seg_eval('sal-ks-101-vd', [], true);
39 | 
40 | %exp_seg_eval('sal-am-101-alex', [], true);
41 | exp_seg_eval('sal-am-101-alex', [], true);
42 | %exp_seg_eval('sal-am-101-vd', [], true);
43 | 
44 | %exp_seg_eval('sal-baseline3-101', [], true);
45 | 


--------------------------------------------------------------------------------
/saliency/salseg.m:
--------------------------------------------------------------------------------
 1 | function res =  salseg(im0, mask, variant)
 2 | 
 3 | if nargin < 3, variant = 'sal' ; end
 4 | 
 5 | res.mask = mask ;
 6 | 
 7 | opts.bVis = true ;
 8 | switch lower(variant)
 9 |   case 'ks'
10 |     opts.quantMax = 0.95;
11 |     opts.quantMin = 0.3;
12 |     opts.quantMaxClamp = 0.995;
13 |     opts.quantMinClamp = 0.005;
14 |     opts.ksCompatible = false ;
15 | 
16 |   case {'sal'}
17 |     opts.quantMax = 0.90;
18 |     opts.quantMin = 0.30;
19 |     opts.quantMaxClamp = 0.995 ;
20 |     opts.quantMinClamp = 0.005 ;
21 |     opts.getLargestCC = true ;
22 |     opts.gmmNmix_fg = 5 ;
23 |     opts.ksCompatible = false ;
24 | 
25 |   case {'am', 'dc'}
26 |     opts.quantMax = 0.90;
27 |     opts.quantMin = 0.30;
28 |     opts.quantMaxClamp = 0.995 ;
29 |     opts.quantMinClamp = 0.005 ;
30 |     opts.getLargestCC = true ;
31 |     opts.gmmNmix_fg = 5 ;
32 |     opts.ksCompatible = false ;
33 | 
34 |   case {'baseline1'}
35 |     opts.quantMax = 0.99999;
36 |     opts.quantMin = 0.00001;
37 |     opts.quantMaxClamp = 0.99999;
38 |     opts.quantMinClamp = 0.00001;
39 |     opts.getLargestCC = true ;
40 |     opts.gmmNmix_fg = 5 ;
41 |     opts.ksCompatible = false ;
42 |     
43 |   case {'baseline2'}
44 |     idx_high = find(mask > 0.6);  
45 |     idx_low = find(mask < 0.4);
46 |     opts.quantMax = 1 - numel(idx_high) / numel(mask);
47 |     opts.quantMin = numel(idx_high) / numel(mask);
48 |     opts.quantMaxClamp = opts.quantMax;
49 |     opts.quantMinClamp = opts.quantMin;
50 |     opts.getLargestCC = true ;
51 |     opts.gmmNmix_fg = 5 ;
52 |     opts.ksCompatible = false ;
53 | 
54 |   case {'baseline3'}
55 |     opts.quantMax = 0.90;
56 |     opts.quantMin = 0.30;
57 |     opts.quantMaxClamp = 0.995 ;
58 |     opts.quantMinClamp = 0.005 ;
59 |     opts.getLargestCC = true ;
60 |     opts.gmmNmix_fg = 5 ;
61 |     opts.ksCompatible = false ;
62 | 
63 | end
64 | 
65 | [res.seg, res.segBox] = ksseg(im0, res.mask, opts) ;
66 | 
67 | figure(2) ; clf ;
68 | subplot(2,2,1) ; imagesc(im0) ; axis image ;
69 | hold on ;
70 | rectangle('Position', [res.segBox(1:2); res.segBox(3:4) - res.segBox(1:2) + 1], ...
71 |           'EdgeColor', 'yellow', 'LineWidth', 2) ;
72 | subplot(2,2,2) ; imagesc(res.mask) ; axis image ;
73 | subplot(2,2,3) ; imagesc(res.seg) ; axis image ;
74 | subplot(2,2,4) ; imagesc(bsxfun(@times, single(res.seg), im0)) ; axis image ;
75 | 


--------------------------------------------------------------------------------
/saliency/seg_eval.m:
--------------------------------------------------------------------------------
 1 | function [acc, iou, precision] = seg_eval(imdb, i, seg, doplot)
 2 | 
 3 | if nargin < 4, doplot = false ; end
 4 | 
 5 | m = imdb.images.numSegs(i) ;
 6 | precision_ = zeros(1,m) ;
 7 | acc_ = zeros(1,m) ;
 8 | iou_ = zeros(1,m) ;
 9 | for j = 1:m
10 |   gt{j} = imread(sprintf(imdb.paths.seg, imdb.images.name{i}, j)) ;
11 |   gt{j} = uint8(gt{j}) ;
12 |   if j == 1
13 |     mass_ = numel(gt{j}) ;
14 |     est = imresize(seg, size(gt{j}), 'nearest') ;
15 |   end
16 |   acc_(j) = sum(gt{j}(:) == est(:)) / mass_ * 100 ;
17 |   iou_(j) = sum(gt{j}(:) > 0 & est(:) > 0) / sum(gt{j}(:) + est(:) > 0) * 100 ;
18 |   precision_(j) = sum(gt{j}(:) > 0 & est(:) > 0) / sum(est(:) > 0);
19 | end
20 | [acc,j] = max(acc_) ;
21 | [iou,j] = max(iou_) ;
22 | [precision,j] = max(precision_);
23 | 
24 | if doplot
25 |   figure(200) ; clf ;
26 |   imagesc([est > 0, gt{j} > 0]) ;
27 |   drawnow ;
28 | end
29 | 


--------------------------------------------------------------------------------
/segmentation_qualitative_results_figure.m:
--------------------------------------------------------------------------------
  1 | % generates fig . 8 of the paper
  2 | % This code assumes that the several segmentation experiments in the
  3 | % saliency folder have been completed and all the results are stored in
  4 | % saliency/data/ferrari/
  5 | 
  6 | FIGS_PATH = 'genfigs/';
  7 | 
  8 | % Segmentation results figure. 
  9 | % We'll use 4 images and for each of them show
 10 | % the resized image, the ground truth segmentation mask, our result, result
 11 | % using saliency, result using deconvnet, the center seed baseline.
 12 | 
 13 | addpath('saliency');
 14 | 
 15 | NUM_IMAGES = 5;
 16 | ROOT = 'saliency/';
 17 | SALIENCY = 'sal-ks-101-alex';
 18 | DECONVNET = 'sal-dc-101-alex';
 19 | HYBRID = 'sal-am-101-alex';
 20 | BASELINE = 'sal-baseline3-101';
 21 | 
 22 | opts = struct();
 23 | opts.randomizeWeights = false;
 24 | opts.gpu = false;
 25 | 
 26 | opts.relus_to_change = 1:100;
 27 | opts.pools_to_change = 1:5;
 28 | opts.convs_to_change = 1:100;
 29 | 
 30 | opts.neuron_I = inf;
 31 | opts.neuron_J = inf;
 32 | opts.neuron_channel = inf;
 33 | 
 34 | opts.modelPath = 'models/imagenet-caffe-alex.mat';
 35 | opts.layer = 20;
 36 | 
 37 | pad_fn = @(x) padarray(x, [1,1], 1, 'both');
 38 | 
 39 | imdb = load('saliency/data/ferrari/imdb.mat');
 40 | %image_ids = randi(numel(imdb.images.name), [1, NUM_IMAGES]); % Randomly pick 5 images
 41 | image_ids = [ 3241, 3178, 1678, 2803, 732]; % There were generated by a call to the above line
 42 | image_names = cell(NUM_IMAGES, 1);
 43 | seg_names = cell(NUM_IMAGES, 1);
 44 | result_names = cell(NUM_IMAGES, 4);
 45 | viz = cell(NUM_IMAGES,1);
 46 | for i=1:NUM_IMAGES
 47 |     cur_img_name = imdb.images.name{image_ids(i)};
 48 |     image_names{i} = [ROOT, sprintf(imdb.paths.image, cur_img_name)];
 49 |     seg_names{i} = [ROOT, sprintf(imdb.paths.seg, cur_img_name, 1)];
 50 |     result_names{i,1} = [ROOT, 'data/ferrari/', SALIENCY, '/', cur_img_name, '.mat'];
 51 |     result_names{i,2} = [ROOT, 'data/ferrari/', DECONVNET, '/', cur_img_name, '.mat'];
 52 |     result_names{i,3} = [ROOT, 'data/ferrari/', HYBRID, '/', cur_img_name, '.mat'];
 53 |     result_names{i,4} = [ROOT, 'data/ferrari/', BASELINE, '/', cur_img_name, '.mat'];
 54 |     
 55 |     opts.imagePath = image_names{i};
 56 |     
 57 |     % Original image
 58 |     img = ksresize(imread(image_names{i}));
 59 |     viz_img = im2single(uint8(img));
 60 |     
 61 |     % Ground truth segmentation
 62 |     sz = size(img);
 63 |     seg = imresize(imread(seg_names{i}), [sz(1), sz(2)], 'nearest');
 64 |     viz_gt =  im2single(uint8(bsxfun(@times, single(seg), img)));
 65 |     
 66 |     % saliency
 67 |     res = load(result_names{i,1}); 
 68 |     viz_saliency = vl_imsc(bsxfun(@times, single(res.seg), img));
 69 |     %opts.algorithm = 'saliency';
 70 |     %[~, mask_saliency, ~] ...
 71 |     %        = hand_specified_neuron_viz_fn(opts);
 72 |     %mask_saliency = vl_imsc_am(mask_saliency);
 73 |     mask_saliency = vl_imsc_am(res.mask_signed);
 74 |     
 75 |     % Deconvnet
 76 |     res = load(result_names{i,2});
 77 |     viz_deconvnet = vl_imsc(bsxfun(@times, single(res.seg), img));
 78 |     %opts.algorithm = 'deconvnet';
 79 |     %[~, mask_deconvnet, ~] ...
 80 |     %        = hand_specified_neuron_viz_fn(opts);
 81 |     %mask_deconvnet = vl_imsc_am(mask_deconvnet);
 82 |     mask_deconvnet = vl_imsc_am(res.mask_signed);
 83 |     
 84 |     % Hybrid
 85 |     res = load(result_names{i,3});
 86 |     viz_am = vl_imsc(bsxfun(@times, single(res.seg), img));
 87 |     %opts.algorithm = 'TTT';
 88 |     %[~, mask_am, ~] ...
 89 |     %        = hand_specified_neuron_viz_fn(opts);
 90 |     %mask_am = vl_imsc_am(mask_am);
 91 |     mask_am = vl_imsc_am(res.mask_signed);
 92 |     
 93 |     % Baseline 2
 94 |     res = load(result_names{i,4});
 95 |     viz_baseline = vl_imsc(bsxfun(@times, single(res.seg), img));
 96 |     %mask_baseline = vl_imsc(baseline2(img));
 97 |     mask_baseline = repmat(vl_imsc(res.mask), [1,1,3]);
 98 |     
 99 |     viz{i} = [cat(2, pad_fn(viz_gt), pad_fn(viz_am), pad_fn(viz_saliency), pad_fn(viz_deconvnet), pad_fn(viz_baseline));
100 |               cat(2, pad_fn(viz_img), pad_fn(mask_am), pad_fn(mask_saliency), pad_fn(mask_deconvnet), pad_fn(mask_baseline))];
101 | end
102 | 
103 | viz_sizes = cell2mat(cellfun(@(x) size(x), viz, 'UniformOutput', false));
104 | chosen_size = min(viz_sizes(:,2));
105 | 
106 | for i=1:NUM_IMAGES
107 |     viz{i} = imresize(viz{i}, [NaN, chosen_size], 'nearest');
108 | end
109 | 
110 | final_viz = cat(1, viz{:});
111 | 
112 | imwrite(final_viz, [FIGS_PATH, 'segmentation_101.png']);
113 | 


--------------------------------------------------------------------------------
/setup_eccv2016code.m:
--------------------------------------------------------------------------------
1 | % The setup file simply sets up the required toolboxes
2 | 
3 | run vlfeat/toolbox/vl_setup.m
4 | run matconvnet/matlab/vl_setupnn.m
5 | 
6 | addpath('matconvnet/examples');


--------------------------------------------------------------------------------
/supplementary/effect_of_phase_sup.m:
--------------------------------------------------------------------------------
  1 | FIGS_PATH = 'supfigs/';
  2 | 
  3 | IMAGE_IDS = 10000 + [3:5:50];
  4 | IMAGE_NAMES = cell(numel(IMAGE_IDS),1);
  5 | for i=1:numel(IMAGE_IDS)
  6 |     IMAGE_NAMES{i} = sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(i));
  7 | end
  8 | 
  9 | 
 10 | MODEL_NAMES = {'../models/imagenet-vgg-verydeep-16.mat',...
 11 |                '../models/imagenet-caffe-alex.mat'};
 12 | 
 13 | %LAYERS = {[28, 42], [12, 20]};
 14 | %LAYERS = {[24, 36], [12, 20]};
 15 | LAYERS = {[5, 10, 17, 24, 31, 33, 35, 36], [4, 8, 10, 12, 15, 17, 19, 20]};
 16 | 
 17 | METHODS = {'deconvnet', 'TTT', 'saliency'};
 18 | 
 19 | viz_images = cell(numel(IMAGE_IDS), numel(MODEL_NAMES), numel(LAYERS{1}), ...
 20 |     numel(METHODS), 4);
 21 | 
 22 | for img_no = 1:numel(IMAGE_IDS)
 23 |     opts = struct();
 24 |     opts.randomizeWeights = false;
 25 |     opts.gpu = false;
 26 |     
 27 |     opts.relus_to_change = 1:100;
 28 |     opts.pools_to_change = 1:5;
 29 |     opts.convs_to_change = 1:100;
 30 |     
 31 |     opts.imagePath = IMAGE_NAMES{img_no};
 32 |     
 33 |     for model_no = 1:numel(MODEL_NAMES)
 34 |         opts.modelPath = MODEL_NAMES{model_no};
 35 |         
 36 |         for layer_no = 1:numel(LAYERS{model_no})
 37 |             opts.layer = LAYERS{model_no}(layer_no);
 38 |             
 39 |             for method_no = 1:numel(METHODS)
 40 |                 
 41 |                 cituation_no = 1;
 42 |                 
 43 |                 opts.neuron_I = 1/2;
 44 |                 opts.neuron_J = 1/2;
 45 |                 opts.neuron_channel = inf;
 46 |                 
 47 |                 opts.algorithm = METHODS{method_no};
 48 |                 
 49 |                 [~, viz_images{img_no, model_no, layer_no, method_no, cituation_no}, opts_new, ~] ...
 50 |                     = hand_specified_neuron_viz_fn(opts);
 51 |                 
 52 |                 viz_images{img_no, model_no, layer_no, method_no, cituation_no} = ...
 53 |                     padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no, cituation_no}), [1, 1], 1, 'both');
 54 |                 
 55 |                 cituation_no = 2;
 56 |                 
 57 |                 opts.neuron_I = opts_new.neuron_I;
 58 |                 opts.neuron_J = opts_new.neuron_J;
 59 |                 opts.neuron_channel = -1;
 60 |                 
 61 |                 [~, viz_images{img_no, model_no, layer_no, method_no, cituation_no}, opts_new2, NET] ...
 62 |                     = hand_specified_neuron_viz_fn(opts);
 63 |                 
 64 |                 if(opts_new2.neuron_channel == opts_new.neuron_channel)
 65 |                     'Oops I randomly picked the maximally excited neuron'
 66 |                     keyboard;
 67 |                 end
 68 |                 
 69 |                 viz_images{img_no, model_no, layer_no, method_no, cituation_no} = ...
 70 |                     padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no, cituation_no}), [1,1], 1, 'both');
 71 |                 
 72 |                 cituation_no = 3;
 73 |                 
 74 |                 img = imread(IMAGE_NAMES{img_no});
 75 |                 img_pp = opts_new.normalize(img);
 76 |                 NET_info = vl_simplenn_display(NET, 'inputSize', [NET.meta.normalization.imageSize(1:2), 3, 1]);
 77 |                 dzdy = zeros(NET_info.dataSize(:, end)', 'single');
 78 |                 dzdy(opts_new.neuron_I, opts_new.neuron_J, :) = abs(randn([1,1,size(dzdy,3)], 'single'));
 79 |                 %dzdy = abs(randn(NET_info.dataSize(:, end)', 'single'));
 80 |                 res = vl_simplenn(NET, img_pp, dzdy, [], 'conserveMemory', true);
 81 |                 viz_images{img_no, model_no, layer_no, method_no, cituation_no} = ...
 82 |                     padarray(vl_imsc_am(res(1).dzdx), [1,1], 1, 'both');
 83 | 
 84 |                 clear NET NET_info dzdy res img img_pp opt_new opts_new2;
 85 |                 
 86 |             end
 87 |         end
 88 |     end
 89 | end
 90 | 
 91 | orig_images = cell(numel(IMAGE_IDS), numel(MODEL_NAMES));
 92 | 
 93 | for model_no = 1:numel(MODEL_NAMES)
 94 |     NET = vl_simplenn_tidy(load(MODEL_NAMES{model_no}));
 95 |     for img_no = 1:numel(IMAGE_IDS)
 96 |         img = imread(IMAGE_NAMES{img_no});
 97 |         orig_images{img_no, model_no} = padarray(im2single(resizencrop(img, NET.meta.normalization.imageSize(1:2))), [1,1], 1, 'both');
 98 | 
 99 |         if(strfind(MODEL_NAMES{model_no}, 'vgg-verydeep-16'))
100 |           model_str = 'vgg16';
101 |         elseif(strfind(MODEL_NAMES{model_no}, 'alex'))
102 |           model_str = 'alex';
103 |         end
104 |            
105 |         imwrite(orig_images{img_no, model_no}, sprintf([FIGS_PATH, 'effect_of_phase_center_%s_%d.png'], model_str, img_no));
106 | 
107 |         clear img;
108 |     end
109 |     clear NET;
110 | end
111 | 
112 | for method_no = 1:numel(METHODS)
113 | 
114 | final_image_vgg16 = [];
115 | final_image_alexnet = [];
116 | 
117 | for img_no = 1:5%numel(IMAGE_IDS)
118 | 
119 | final_image_vgg16 = cat(2, final_image_vgg16, [cat(1, viz_images{img_no,1,1:8,method_no,1}),...
120 |                                                cat(1, viz_images{img_no,1,1:8,method_no,2}),...
121 |                                                cat(1, viz_images{img_no,1,1:8,method_no,3})]);
122 | final_image_vgg16 = cat(2, final_image_vgg16, ones(size(final_image_vgg16,1),10,3,'single'));
123 | 
124 | final_image_alexnet = cat(2, final_image_alexnet, [cat(1, viz_images{img_no,2,1:8,method_no,1}),...
125 |                                                cat(1, viz_images{img_no,2,1:8,method_no,2}),...
126 |                                                cat(1, viz_images{img_no,2,1:8,method_no,3})]);
127 | final_image_alexnet = cat(2, final_image_alexnet, ones(size(final_image_alexnet,1),10,3,'single'));
128 | 
129 | 
130 | end
131 | 
132 | imwrite(final_image_alexnet, sprintf([FIGS_PATH, 'effect_of_phase_center_alexnet_%s_vert.png'], METHODS{method_no}));
133 | imwrite(final_image_vgg16, sprintf([FIGS_PATH, 'effect_of_phase_center_vgg16_%s_vert.png'], METHODS{method_no}));
134 | 
135 | end
136 | 


--------------------------------------------------------------------------------
/supplementary/fig_spatial_selectivity_noarcheffects_sup.m:
--------------------------------------------------------------------------------
 1 | FIGS_PATH = 'supfigs/';
 2 | 
 3 | IMAGE_IDS = 10000 + [4:5:50];
 4 | IMAGE_NAMES = cell(numel(IMAGE_IDS),1);
 5 | for i=1:numel(IMAGE_IDS)
 6 |     IMAGE_NAMES{i} = sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(i));
 7 | end
 8 | 
 9 | MODEL_NAMES = {'../models/imagenet-vgg-verydeep-16.mat',...
10 |                '../models/imagenet-caffe-alex.mat'};
11 | 
12 | %LAYERS = {[28, 42], [12, 20]};
13 | LAYERS = {[5, 10, 17, 24, 31, 33, 35, 36], [4, 8, 10, 12, 15, 17, 19, 20]};
14 | %LAYERS = {[31, 36], [15, 20]};
15 | 
16 | METHODS = {'deconvnet', 'saliency', 'TTT'};
17 | 
18 | viz_images = cell(5,2,8,4);
19 | 
20 | for img_no = 1:numel(IMAGE_IDS)
21 |     opts = struct();
22 |     
23 |     opts.gpu = false;
24 | 
25 |     opts.relus_to_change = 1:100;
26 |     opts.pools_to_change = 1:5;
27 |     opts.convs_to_change = 1:100;
28 | 
29 |     opts.neuron_I = 1/2;
30 |     opts.neuron_J = 1/2;
31 |     opts.neuron_channel = inf;
32 |     
33 |     opts.imagePath = IMAGE_NAMES{img_no};
34 |     
35 |     for model_no = 1:2
36 |         opts.modelPath = MODEL_NAMES{model_no};
37 |         
38 |         for layer_no = 1:numel(LAYERS{model_no})
39 |             opts.layer = LAYERS{model_no}(layer_no);
40 |             
41 |             for method_no = 1:numel(METHODS)
42 |                 opts.randomizeWeights = false;
43 |                 opts.algorithm = METHODS{method_no};
44 |                 
45 |                 [~, viz_images{img_no, model_no, layer_no, method_no}, ~] ...
46 |                     = hand_specified_neuron_viz_fn(opts);
47 |                 viz_images{img_no, model_no, layer_no, method_no} = ...
48 |                     padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no}), [1,1], 1, 'both');
49 |                 
50 |                 
51 |             end
52 |             
53 |             %opts.algorithm = 'FFF';
54 |             %opts.randomizeWeights = true;
55 |             %
56 |             %[~, viz_images{img_no, model_no, layer_no, 5}, ~] ...
57 |             %        = hand_specified_neuron_viz_fn(opts);
58 |             %viz_images{img_no, model_no, layer_no, 5} = ...
59 |             %        padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, 5}), [1,1], 1, 'both');
60 |             
61 |         end
62 |         
63 |         img = imread(IMAGE_NAMES{img_no});
64 |         NET = vl_simplenn_tidy(load(opts.modelPath));
65 |         viz_images{img_no, model_no, 1, 4} = padarray(im2single(resizencrop(img, NET.meta.normalization.imageSize(1:2))), [1,1], 1, 'both');
66 |         clear NET;
67 |         clear img;
68 |         
69 |     end
70 | 
71 | end
72 | 
73 | final_image_vgg16 = ...
74 | [cat(1, viz_images{:,1,1,4}), cat(1, viz_images{:,1,1,1}), cat(1, viz_images{:,1,1,2}), cat(1, viz_images{:,1,1,3}),...
75 |  cat(1, viz_images{:,1,2,1}), cat(1, viz_images{:,1,2,2}), cat(1, viz_images{:,1,2,3}),...
76 |  cat(1, viz_images{:,1,3,1}), cat(1, viz_images{:,1,3,2}), cat(1, viz_images{:,1,3,3}),...
77 |  cat(1, viz_images{:,1,4,1}), cat(1, viz_images{:,1,4,2}), cat(1, viz_images{:,1,4,3}),...
78 |  cat(1, viz_images{:,1,5,1}), cat(1, viz_images{:,1,5,2}), cat(1, viz_images{:,1,5,3}),...
79 |  cat(1, viz_images{:,1,6,1}), cat(1, viz_images{:,1,6,2}), cat(1, viz_images{:,1,6,3}),...
80 |  cat(1, viz_images{:,1,7,1}), cat(1, viz_images{:,1,7,2}), cat(1, viz_images{:,1,7,3}),...
81 |  cat(1, viz_images{:,1,8,1}), cat(1, viz_images{:,1,8,2}), cat(1, viz_images{:,1,8,3})];
82 | 
83 | final_image_alexnet = ...
84 | [cat(1, viz_images{:,2,1,4}), cat(1, viz_images{:,2,1,1}), cat(1, viz_images{:,2,1,2}), cat(1, viz_images{:,2,1,3}),...
85 |  cat(1, viz_images{:,2,2,1}), cat(1, viz_images{:,2,2,2}), cat(1, viz_images{:,2,2,3}),...
86 |  cat(1, viz_images{:,2,3,1}), cat(1, viz_images{:,2,3,2}), cat(1, viz_images{:,2,3,3}),...
87 |  cat(1, viz_images{:,2,4,1}), cat(1, viz_images{:,2,4,2}), cat(1, viz_images{:,2,4,3}),...
88 |  cat(1, viz_images{:,2,5,1}), cat(1, viz_images{:,2,5,2}), cat(1, viz_images{:,2,5,3}),...
89 |  cat(1, viz_images{:,2,6,1}), cat(1, viz_images{:,2,6,2}), cat(1, viz_images{:,2,6,3}),...
90 |  cat(1, viz_images{:,2,7,1}), cat(1, viz_images{:,2,7,2}), cat(1, viz_images{:,2,7,3}),...
91 |  cat(1, viz_images{:,2,8,1}), cat(1, viz_images{:,2,8,2}), cat(1, viz_images{:,2,8,3}) ];
92 | 
93 | imwrite(final_image_vgg16, [FIGS_PATH, 'fig_spatial_selectivity_noarcheffects_vgg16.png']);
94 | imwrite(final_image_alexnet, [FIGS_PATH, 'fig_spatial_selectivity_noarcheffects_alexnet.png']);
95 | 


--------------------------------------------------------------------------------
/supplementary/fig_splash_all_images_sup.m:
--------------------------------------------------------------------------------
 1 | FIGS_PATH = 'supfigs/';
 2 | 
 3 | %IMAGE_IDS = [77, 170];%, 98, 234,289,295,298,299,306];
 4 | IMAGE_IDS = 10000 + [5:5:50]; %[170, 98, 234,289,299,27,77,];%298,299,306];
 5 | IMAGE_NAMES = cell(numel(IMAGE_IDS),1);
 6 | for i=1:numel(IMAGE_IDS)
 7 |     IMAGE_NAMES{i} = sprintf('imagenet12-val/ILSVRC2012_val_%08d.JPEG', IMAGE_IDS(i));
 8 | end
 9 |     
10 | 
11 | MODEL_NAMES = {'../models/imagenet-vgg-verydeep-16.mat',...
12 |                '../models/imagenet-caffe-alex.mat'};
13 | 
14 | %LAYERS = {[28, 42], [12, 20]};
15 | LAYERS = {[36], [20]};
16 | 
17 | METHODS = {'deconvnet', 'saliency', 'TTT'};
18 | 
19 | viz_images = cell(numel(IMAGE_IDS),2,1,4);
20 | 
21 | for img_no = 1:numel(IMAGE_IDS)
22 |     opts = struct();
23 |     opts.randomizeWeights = false;
24 |     opts.gpu = false;
25 | 
26 |     opts.relus_to_change = 1:100;
27 |     opts.pools_to_change = 1:5;
28 |     opts.convs_to_change = 1:100;
29 | 
30 |     opts.neuron_I = inf;
31 |     opts.neuron_J = inf;
32 |     opts.neuron_channel = inf;
33 |     
34 |     opts.imagePath = IMAGE_NAMES{img_no};
35 |     
36 |     for model_no = 1:2
37 |         opts.modelPath = MODEL_NAMES{model_no};
38 |         
39 |         for layer_no = 1:1
40 |             opts.layer = LAYERS{model_no}(layer_no);
41 |             
42 |             for method_no = 1:numel(METHODS)
43 |                 opts.algorithm = METHODS{method_no};
44 |                 
45 |                 [~, viz_images{img_no, model_no, layer_no, method_no}, opts_new, ~] ...
46 |                     = hand_specified_neuron_viz_fn(opts);
47 |                 viz_images{img_no, model_no, layer_no, method_no} = ...
48 |                     padarray(vl_imsc_am(viz_images{img_no, model_no, layer_no, method_no}), [1,1], 1, 'both');
49 |                 
50 |                 
51 |             end
52 |         end
53 |         img = imread(IMAGE_NAMES{img_no});
54 |         NET = vl_simplenn_tidy(load(opts.modelPath));
55 |         viz_images{img_no, model_no, 1, 4} = padarray(im2single(resizencrop(img, NET.meta.normalization.imageSize(1:2))), [1,1], 1, 'both');    
56 |         clear NET;
57 |         clear img;
58 |     end
59 |     
60 |     
61 | end
62 | 
63 | t = 1;
64 | final_image_vgg16 = [...
65 |                cat(2, viz_images{:,t,1,4});...
66 |                cat(2, viz_images{:,t,1,1});...
67 |                cat(2, viz_images{:,t,1,2});...
68 |                cat(2, viz_images{:,t,1,3})];
69 |            
70 | t = 2;
71 | final_image_alexnet = [...
72 |                cat(2, viz_images{:,t,1,4});...
73 |                cat(2, viz_images{:,t,1,1});...
74 |                cat(2, viz_images{:,t,1,2});...
75 |                cat(2, viz_images{:,t,1,3})];
76 |            
77 | imwrite(final_image_alexnet, [FIGS_PATH, 'fig_splash_alexnet.png']);
78 | imwrite(final_image_vgg16, [FIGS_PATH, 'fig_splash_vgg16.png']);
79 | 


--------------------------------------------------------------------------------
/supplementary/generate_all_figures_sup.m:
--------------------------------------------------------------------------------
 1 | 
 2 | clear all; setup_eccv2016code;
 3 | 'fig splash'
 4 | fig_splash_all_images_sup;
 5 | 
 6 | 
 7 | clear all; setup_eccv2016code;
 8 | 'Spatial selectivity no arch effects'
 9 | fig_spatial_selectivity_noarcheffects_sup;
10 | 
11 | clear all; setup_eccv2016code;
12 | 'effect of phase'
13 | effect_of_phase_sup;
14 | 


--------------------------------------------------------------------------------
/vl_imsc_am.m:
--------------------------------------------------------------------------------
 1 | function img = vl_imsc_am(img_in)
 2 | % See footnote 4 in page 7.
 3 | 
 4 | a = quantile(img_in(:), 0.005);
 5 | if(abs(a) < 1e-5)
 6 |     a = min(img_in(:));
 7 | end
 8 | fprintf(1, 'Viz quantile is %f\n', a);
 9 | %img = 0.5 * (1 - img_in / a);
10 | img = 1./(1 + exp(-(img_in / a * -log(99))));
11 | end


--------------------------------------------------------------------------------
/vl_simplenn.m:
--------------------------------------------------------------------------------
  1 | function res = vl_simplenn(net, x, dzdy, res, varargin)
  2 | %VL_SIMPLENN  Evaluate a SimpleNN network.
  3 | %   RES = VL_SIMPLENN(NET, X) evaluates the convnet NET on data X.
  4 | %   RES = VL_SIMPLENN(NET, X, DZDY) evaluates the convnent NET and its
  5 | %   derivative on data X and output derivative DZDY (foward+bacwkard pass).
  6 | %   RES = VL_SIMPLENN(NET, X, [], RES) evaluates the NET on X reusing the
  7 | %   structure RES.
  8 | %   RES = VL_SIMPLENN(NET, X, DZDY, RES) evaluates the NET on X and its
  9 | %   derivatives reusing the structure RES.
 10 | %
 11 | %   This function process networks using the SimpleNN wrapper
 12 | %   format. Such networks are 'simple' in the sense that they consist
 13 | %   of a linear sequence of computational layers. You can use the
 14 | %   `dagnn.DagNN` wrapper for more complex topologies, or write your
 15 | %   own wrapper around MatConvNet computational blocks for even
 16 | %   greater flexibility.
 17 | %
 18 | %   The format of the network structure NET and of the result
 19 | %   structure RES are described in some detail below. Most networks
 20 | %   expect the input data X to be standardized, for example by
 21 | %   rescaling the input image(s) and subtracting a mean. Doing so is
 22 | %   left to the user, but information on how to do this is usually
 23 | %   contained in the `net.meta` field of the NET structure (see
 24 | %   below).
 25 | %
 26 | %   The NET structure needs to be updated as new features are
 27 | %   introduced in MatConvNet; use the `VL_SIMPLENN_TIDY()` function
 28 | %   to make an old network current, as well as to cleanup and check
 29 | %   the structure of an existing network.
 30 | %
 31 | %   Networks can run either on the CPU or GPU. Use VL_SIMPLENN_MOVE()
 32 | %   to move the network parameters between these devices.
 33 | %
 34 | %   To print or obtain summary of the network structure, use the
 35 | %   VL_SIMPLENN_DISPLAY() function.
 36 | %
 37 | %   VL_SIMPLENN(NET, X, DZDY, RES, 'OPT', VAL, ...) takes the following
 38 | %   options:
 39 | %
 40 | %   `Mode`:: `'normal'`
 41 | %      Specifies the mode of operation. It can be either `'normal'` or
 42 | %      `'test'`. In test mode, dropout and batch-normalization are
 43 | %      bypassed. Note that, when a network is deployed, it may be
 44 | %      preferable to *remove* such blocks altogether.
 45 | %
 46 | %   `ConserveMemory`:: `false`
 47 | %      Aggressively delete intermediate results. This in practice has
 48 | %      a very small performance hit and allows training much larger
 49 | %      models. However, it can be useful to disable it for
 50 | %      debugging. Keeps the values in `res(1)` (input) and `res(end)`
 51 | %      (output) with the outputs of `loss` and `softmaxloss` layers.
 52 | %      It is also possible to preserve individual layer outputs
 53 | %      by setting `net.layers{...}.precious` to `true`.
 54 | %      For back-propagation, keeps only the derivatives with respect to
 55 | %      weights.
 56 | %
 57 | %   `CuDNN`:: `true`
 58 | %      Use CuDNN when available.
 59 | %
 60 | %   `Accumulate`:: `false`
 61 | %      Accumulate gradients in back-propagation instead of rewriting
 62 | %      them. This is useful to break the computation in sub-batches.
 63 | %      The gradients are accumulated to the provided RES structure
 64 | %      (i.e. to call VL_SIMPLENN(NET, X, DZDY, RES, ...).
 65 | %
 66 | %   `BackPropDepth`:: `inf`
 67 | %      Limit the back-propagation to top-N layers.
 68 | %
 69 | %   `SkipForward`:: `false`
 70 | %      Reuse the output values from the provided RES structure and compute
 71 | %      only the derivatives (backward pass).
 72 | %
 73 | %   ## The result format
 74 | %
 75 | %   SimpleNN returns the result of its calculations in the RES
 76 | %   structure array. RES(1) contains the input to the network, while
 77 | %   RES(2), RES(3), ... contain the output of each layer, from first
 78 | %   to last. Each entry has the following fields:
 79 | %
 80 | %   - `res(i+1).x`: the output of layer `i`. Hence `res(1).x` is the
 81 | %     network input.
 82 | %
 83 | %   - `res(i+1).aux`: any auxiliary output data of layer i. For example,
 84 | %     dropout uses this field to store the dropout mask.
 85 | %
 86 | %   - `res(i+1).dzdx`: the derivative of the network output relative
 87 | %     to the output of layer `i`. In particular `res(1).dzdx` is the
 88 | %     derivative of the network output with respect to the network
 89 | %     input.
 90 | %
 91 | %   - `res(i+1).dzdw`: a cell array containing the derivatives of the
 92 | %     network output relative to the parameters of layer `i`. It can
 93 | %     be a cell array for multiple parameters.
 94 | %
 95 | %   ## The network format
 96 | %
 97 | %   The network is represented by the NET structure, which contains
 98 | %   two fields:
 99 | %
100 | %   - `net.layers` is a cell array with the CNN layers.
101 | %
102 | %   - `net.meta` is a grab-bag of auxiliary application-dependent
103 | %     information, including for example details on how to normalize
104 | %     input data, the class names for a classifiers, or details of
105 | %     the learning algorithm. The content of this field is ignored by
106 | %     VL_SIMPLENN().
107 | %
108 | %   SimpleNN is aware of the following layers:
109 | %
110 | %   Convolution layer::
111 | %     The convolution layer wraps VL_NNCONV(). It has fields:
112 | %
113 | %     - `layer.type` contains the string `'conv'`.
114 | %     - `layer.weights` is a cell array with filters and biases.
115 | %     - `layer.stride` is the sampling stride (e.g. 1).
116 | %     - `layer.pad` is the padding (e.g. 0).
117 | %     - `layer.dilate` is the dilation factor (e.g. 1).
118 | %
119 | %   Convolution transpose layer::
120 | %     The convolution transpose layer wraps VL_NNCONVT(). It has fields:
121 | %
122 | %     - `layer.type` contains the string `'convt'`.
123 | %     - `layer.weights` is a cell array with filters and biases.
124 | %     - `layer.upsample` is the upsampling factor (e.g. 1).
125 | %     - `layer.crop` is the amount of output cropping (e.g. 0).
126 | %
127 | %   Max pooling layer::
128 | %     The max pooling layer wraps VL_NNPOOL(). It has fields:
129 | %
130 | %     - `layer.type` contains the string `'pool'`.
131 | %     - `layer.method` is the pooling method (either 'max' or 'avg').
132 | %     - `layer.pool` is the pooling size (e.g. 3).
133 | %     - `layer.stride` is the sampling stride (usually 1).
134 | %     - `layer.pad` is the padding (usually 0).
135 | %
136 | %   Normalization (LRN) layer::
137 | %     The normalization layer wraps VL_NNNORMALIZE(). It has fields:
138 | %
139 | %     - `layer.type` contains the string `'normalize'` or `'lrn'`.
140 | %     - `layer.param` contains the normalization parameters (see VL_NNNORMALIZE()).
141 | %
142 | %   Spatial normalization layer::
143 | %     The spatial normalization layer wraps VL_NNSPNORM(). It has fields:
144 | %
145 | %     - `layer.type` contains the string `'spnorm'`.
146 | %     - `layer.param` contains the normalization parameters (see VL_NNSPNORM()).
147 | %
148 | %   Batch normalization layer::
149 | %     This layer wraps VL_NNBNORM(). It has fields:
150 | %
151 | %     - `layer.type` contains the string `'bnorm'`.
152 | %     - `layer.weights` contains is a cell-array with, multiplier and
153 | %       biases, and moments parameters
154 | %
155 | %     Note that moments are used only in `'test'` mode to bypass batch
156 | %     normalization.
157 | %
158 | %   ReLU and Sigmoid layers::
159 | %     The ReLU layer wraps VL_NNRELU(). It has fields:
160 | %
161 | %     - `layer.type` contains the string `'relu'`.
162 | %     - `layer.leak` is the leak factor (e.g. 0).
163 | %
164 | %     The sigmoid layer is the same, but for the sigmoid function,
165 | %     with `relu` replaced by `sigmoid` and no leak factor.
166 | %
167 | %   Dropout layer::
168 | %     The dropout layer wraps VL_NNDROPOUT(). It has fields:
169 | %
170 | %     - `layer.type` contains the string `'dropout'`.
171 | %     - `layer.rate` is the dropout rate (e.g. 0.5).
172 | %
173 | %     Note that the block is bypassed in `test` mode.
174 | %
175 | %   Softmax layer::
176 | %     The softmax layer wraps VL_NNSOFTMAX(). It has fields
177 | %
178 | %     - `layer.type` contains the string`'softmax'`.
179 | %
180 | %   Log-loss layer and softmax-log-loss::
181 | %     The log-loss layer wraps VL_NNLOSS(). It has fields:
182 | %
183 | %     - `layer.type` contains `'loss'`.
184 | %     - `layer.class` contains the ground-truth class labels.
185 | %
186 | %     The softmax-log-loss layer wraps VL_NNSOFTMAXLOSS() instead. it
187 | %     has the same parameters, but `type` contains the `'softmaxloss'`
188 | %     string.
189 | %
190 | %   P-dist layer::
191 | %     The p-dist layer wraps VL_NNPDIST(). It has fields:
192 | %
193 | %     - `layer.type` contains the string  `'pdist'`.
194 | %     - `layer.p` is the P parameter of the P-distance (e.g. 2).
195 | %     - `layer.noRoot` it tells whether to raise the distance to
196 | %     the P-th power (e.g. `false`).
197 | %     - `layer.epsilon` is the regularization parameter for the derivatives.
198 | %
199 | %   Custom layer::
200 | %     This can be used to specify custom layers.
201 | %
202 | %     - `layer.type` contains the string `'custom'`.
203 | %     - `layer.forward` is  a function handle computing the block.
204 | %     - `layer.backward` is a function handle computing the block derivative.
205 | %
206 | %     The first function is called as
207 | %
208 | %          res(i+1) = layer.forward(layer, res(i), res(i+1))
209 | %
210 | %     where RES is the structure array specified before. The second function is
211 | %     called as
212 | %
213 | %          res(i) = layer.backward(layer, res(i), res(i+1))
214 | %
215 | %     Note that the `layer` structure can contain additional custom
216 | %     fields if needed.
217 | %
218 | %   See also: dagnn.DagNN, VL_SIMPLENN_TIDY(),
219 | %   VL_SIMPLENN_DISPLAY(), VL_SIMPLENN_MOVE().
220 | 
221 | % Copyright (C) 2014-15 Andrea Vedaldi.
222 | % All rights reserved.
223 | %
224 | % This file is part of the VLFeat library and is made available under
225 | % the terms of the BSD license (see the COPYING file).
226 | 
227 | opts.conserveMemory = false ;
228 | opts.sync = false ;
229 | opts.mode = 'normal' ;
230 | opts.accumulate = false ;
231 | opts.cudnn = true ;
232 | opts.backPropDepth = +inf ;
233 | opts.skipForward = false ;
234 | opts.parameterServer = [] ;
235 | opts.holdOn = false ;
236 | opts = vl_argparse(opts, varargin);
237 | 
238 | n = numel(net.layers) ;
239 | assert(opts.backPropDepth > 0, 'Invalid `backPropDepth` value (!>0)');
240 | backPropLim = max(n - opts.backPropDepth + 1, 1);
241 | 
242 | if (nargin <= 2) || isempty(dzdy)
243 |   doder = false ;
244 |   if opts.skipForward
245 |     error('simplenn:skipForwardNoBackwPass', ...
246 |       '`skipForward` valid only when backward pass is computed.');
247 |   end
248 | else
249 |   doder = true ;
250 | end
251 | 
252 | if opts.cudnn
253 |   cudnn = {'CuDNN'} ;
254 |   bnormCudnn = {'NoCuDNN'} ; % ours seems slighty faster
255 | else
256 |   cudnn = {'NoCuDNN'} ;
257 |   bnormCudnn = {'NoCuDNN'} ;
258 | end
259 | 
260 | switch lower(opts.mode)
261 |   case 'normal'
262 |     testMode = false ;
263 |   case 'test'
264 |     testMode = true ;
265 |   otherwise
266 |     error('Unknown mode ''%s''.', opts. mode) ;
267 | end
268 | 
269 | gpuMode = isa(x, 'gpuArray') ;
270 | 
271 | if nargin <= 3 || isempty(res)
272 |   if opts.skipForward
273 |     error('simplenn:skipForwardEmptyRes', ...
274 |     'RES structure must be provided for `skipForward`.');
275 |   end
276 |   res = struct(...
277 |     'x', cell(1,n+1), ...
278 |     'dzdx', cell(1,n+1), ...
279 |     'dzdw', cell(1,n+1), ...
280 |     'aux', cell(1,n+1), ...
281 |     'stats', cell(1,n+1), ...
282 |     'time', num2cell(zeros(1,n+1)), ...
283 |     'backwardTime', num2cell(zeros(1,n+1))) ;
284 | end
285 | 
286 | if ~opts.skipForward
287 |   res(1).x = x ;
288 | end
289 | 
290 | % -------------------------------------------------------------------------
291 | %                                                              Forward pass
292 | % -------------------------------------------------------------------------
293 | 
294 | for i=1:n
295 |   if opts.skipForward, break; end;
296 |   l = net.layers{i} ;
297 |   res(i).time = tic ;
298 |   switch l.type
299 |     case 'conv'
300 |       res(i+1).x = vl_nnconv(res(i).x, l.weights{1}, l.weights{2}, ...
301 |         'pad', l.pad, ...
302 |         'stride', l.stride, ...
303 |         'dilate', l.dilate, ...
304 |         l.opts{:}, ...
305 |         cudnn{:}) ;
306 | 
307 |     case 'convt'
308 |       res(i+1).x = vl_nnconvt(res(i).x, l.weights{1}, l.weights{2}, ...
309 |         'crop', l.crop, ...
310 |         'upsample', l.upsample, ...
311 |         'numGroups', l.numGroups, ...
312 |         l.opts{:}, ...
313 |         cudnn{:}) ;
314 | 
315 |     case 'pool'
316 |       res(i+1).x = vl_nnpool(res(i).x, l.pool, ...
317 |         'pad', l.pad, 'stride', l.stride, ...
318 |         'method', l.method, ...
319 |         l.opts{:}, ...
320 |         cudnn{:}) ;
321 | 
322 |     case 'pool_center' % The forward pass doesn't change
323 |       res(i+1).x = vl_nnpool(res(i).x, l.pool, ...
324 |          'pad', l.pad, 'stride', l.stride, ...
325 |          'method', l.method, ...
326 |          l.opts{:},...
327 |          cudnn{:}) ;
328 | 
329 |     case {'normalize', 'lrn'}
330 |       res(i+1).x = vl_nnnormalize(res(i).x, l.param) ;
331 |    
332 |     case {'normalizelp'}
333 |       res(i+1).x = l.scale * ...
334 |                      vl_nnnormalizelp(res(i).x, [], ...
335 |                                       'p', l.p, ...
336 |                                       'epsilon', l.epsilon, ...
337 |                                       'spatial', l.spatial) ;
338 | 
339 |     case {'normalize_hacked', 'lrn_hacked', 'normalize_nobackprop', 'lrn_nobackprop'}
340 |       res(i+1).x = vl_nnnormalize(res(i).x, l.param) ;   
341 |       
342 |     case 'softmax'
343 |       res(i+1).x = vl_nnsoftmax(res(i).x) ;
344 | 
345 |     case 'loss'
346 |       res(i+1).x = vl_nnloss(res(i).x, l.class) ;
347 | 
348 |     case 'softmaxloss'
349 |       res(i+1).x = vl_nnsoftmaxloss(res(i).x, l.class) ;
350 | 
351 |     case 'relu'
352 |       if l.leak > 0, leak = {'leak', l.leak} ; else leak = {} ; end
353 |       res(i+1).x = vl_nnrelu(res(i).x,[],leak{:}) ;
354 | 
355 |     case {'relu_eccv16', 'relu_deconvnet', 'relu_nobackprop'}
356 |       if (isfield(l, 'leak') && l.leak ~= 0)
357 |           error('%s does not support leak', l.type) ;
358 |       else
359 |           leak = {} ;
360 |       end
361 |       res(i+1).x = vl_nnrelu(res(i).x,[],leak{:}) ;
362 |        
363 |     case 'tanh'
364 |       res(i+1).x = tanh(res(i).x);    
365 |       
366 |     case 'sigmoid'
367 |       res(i+1).x = vl_nnsigmoid(res(i).x) ;
368 | 
369 |     case 'noffset'
370 |       res(i+1).x = vl_nnnoffset(res(i).x, l.param) ;
371 | 
372 |     case 'spnorm'
373 |       res(i+1).x = vl_nnspnorm(res(i).x, l.param) ;
374 | 
375 |     case 'dropout'
376 |       if testMode
377 |         res(i+1).x = res(i).x ;
378 |       else
379 |         [res(i+1).x, res(i+1).aux] = vl_nndropout(res(i).x, 'rate', l.rate) ;
380 |       end
381 | 
382 |     case 'bnorm'
383 |       if testMode
384 |         res(i+1).x = vl_nnbnorm(res(i).x, l.weights{1}, l.weights{2}, ...
385 |                                 'moments', l.weights{3}, ...
386 |                                 'epsilon', l.epsilon, ...
387 |                                 bnormCudnn{:}) ;
388 |       else
389 |         res(i+1).x = vl_nnbnorm(res(i).x, l.weights{1}, l.weights{2}, ...
390 |                                 'epsilon', l.epsilon, ...
391 |                                 bnormCudnn{:}) ;
392 |       end
393 | 
394 |     case 'pdist'
395 |       res(i+1).x = vl_nnpdist(res(i).x, l.class, l.p, ...
396 |         'noRoot', l.noRoot, ...
397 |         'epsilon', l.epsilon, ...
398 |         'aggregate', l.aggregate, ...
399 |         'instanceWeights', l.instanceWeights) ;
400 | 
401 |     case 'custom'
402 |       res(i+1) = l.forward(l, res(i), res(i+1)) ;
403 | 
404 |     otherwise
405 |       error('Unknown layer type ''%s''.', l.type) ;
406 |   end
407 | 
408 |   % optionally forget intermediate results
409 |   needsBProp = doder && i >= backPropLim;
410 |   forget = opts.conserveMemory && ~needsBProp ;
411 |   if i > 1
412 |     lp = net.layers{i-1} ;
413 |     % forget RELU input, even for BPROP
414 |     forget = forget && (~needsBProp || (strcmp(l.type, 'relu') && ~lp.precious)) ;
415 |     forget = forget && ~(strcmp(lp.type, 'loss') || strcmp(lp.type, 'softmaxloss')) ;
416 |     forget = forget && ~lp.precious ;
417 |   end
418 |   if forget
419 |     res(i).x = [] ;
420 |   end
421 | 
422 |   if gpuMode && opts.sync
423 |     wait(gpuDevice) ;
424 |   end
425 |   res(i).time = toc(res(i).time) ;
426 | end
427 | 
428 | % -------------------------------------------------------------------------
429 | %                                                             Backward pass
430 | % -------------------------------------------------------------------------
431 | 
432 | if doder
433 |   res(n+1).dzdx = dzdy ;
434 |   for i=n:-1:backPropLim
435 |     l = net.layers{i} ;
436 |     res(i).backwardTime = tic ;
437 |     switch l.type
438 | 
439 |       case 'conv'
440 |         [res(i).dzdx, dzdw{1}, dzdw{2}] = ...
441 |           vl_nnconv(res(i).x, l.weights{1}, l.weights{2}, res(i+1).dzdx, ...
442 |           'pad', l.pad, ...
443 |           'stride', l.stride, ...
444 |           'dilate', l.dilate, ...
445 |           l.opts{:}, ...
446 |           cudnn{:}) ;
447 | 
448 |       case 'convt'
449 |         [res(i).dzdx, dzdw{1}, dzdw{2}] = ...
450 |           vl_nnconvt(res(i).x, l.weights{1}, l.weights{2}, res(i+1).dzdx, ...
451 |           'crop', l.crop, ...
452 |           'upsample', l.upsample, ...
453 |           'numGroups', l.numGroups, ...
454 |           l.opts{:}, ...
455 |           cudnn{:}) ;
456 | 
457 |       case 'pool'
458 |         res(i).dzdx = vl_nnpool(res(i).x, l.pool, res(i+1).dzdx, ...
459 |                                 'pad', l.pad, 'stride', l.stride, ...
460 |                                 'method', l.method, ...
461 |                                 l.opts{:}, ...
462 |                                 cudnn{:}) ;
463 | 
464 | 
465 |       case 'pool_center'
466 |         %Here's how it works
467 |         %1. Create a zeros tensor the same size as res(i).x
468 |         %2. Pad it with l.pad
469 |         %3. Set 1's at spacing l.stirde and starting from l.pool/2
470 |         %4. Do this in every channel
471 | 
472 |         if(isa(res(i+1).dzdx, 'gpuArray'))
473 |             hoax_input = zeros(size(res(i).x), 'single', 'gpuArray');
474 |         else
475 |             hoax_input = zeros(size(res(i).x), 'single');
476 |         end
477 |         hoax_input_padded_t = padarray(hoax_input, l.pad([1,3]), 0, 'pre');
478 |         hoax_input_padded = padarray(hoax_input_padded_t, l.pad([2,4]), 0, 'post');
479 | 
480 |         hoax_input_padded(ceil(l.pool(1)/2):l.stride(1):end, ceil(l.pool(2)/2):l.stride(2):end, :, :) = 1;
481 |         
482 |         res(i).dzdx = vl_nnpool(hoax_input_padded, l.pool, res(i+1).dzdx, ...
483 |                                 'stride', l.stride, 'method', l.method, ...
484 |                                 'Pad', [0,0,0,0], l.opts{:}, ...
485 |                                 cudnn{:}) ;
486 | 
487 |         res(i).dzdx = res(i).dzdx(l.pad(1) + 1 : end - l.pad(2), ...
488 |                                   l.pad(3) + 1 : end - l.pad(4), :, :);
489 |                             
490 |       case {'normalize', 'lrn'}
491 |         res(i).dzdx = vl_nnnormalize(res(i).x, l.param, res(i+1).dzdx) ;
492 | 
493 |       case {'normalize_hacked', 'lrn_hacked', 'normalize_nobackprop', 'lrn_nobackprop'}
494 |         res(i).dzdx = res(i+1).dzdx;
495 |         
496 | 
497 |       case {'normalizelp'}
498 |         res(i).dzdx = vl_nnnormalizelp(res(i).x, l.scale * res(i+1).dzdx, ...
499 |                                     'p', l.p, ...
500 |                                     'epsilon', l.epsilon, ...
501 |                                     'spatial', l.spatial) ;
502 | 
503 |       case 'softmax'
504 |         res(i).dzdx = vl_nnsoftmax(res(i).x, res(i+1).dzdx) ;
505 | 
506 |       case 'loss'
507 |         res(i).dzdx = vl_nnloss(res(i).x, l.class, res(i+1).dzdx) ;
508 | 
509 |       case 'softmaxloss'
510 |         res(i).dzdx = vl_nnsoftmaxloss(res(i).x, l.class, res(i+1).dzdx) ;
511 | 
512 |       case 'relu'
513 |         if l.leak > 0, leak = {'leak', l.leak} ; else leak = {} ; end
514 |         if ~isempty(res(i).x)
515 |           res(i).dzdx = vl_nnrelu(res(i).x, res(i+1).dzdx, leak{:}) ;
516 |         else
517 |           % if res(i).x is empty, it has been optimized away, so we use this
518 |           % hack (which works only for ReLU):
519 |           res(i).dzdx = vl_nnrelu(res(i+1).x, res(i+1).dzdx, leak{:}) ;
520 |         end
521 | 
522 |       case 'tanh'
523 |         res(i).dzdx = (1 - tanh(res(i).x).^2).*res(i+1).dzdx;
524 |         
525 |       case 'relu_eccv16'
526 |         if (isfield(l, 'leak') && l.leak ~= 0)
527 |             error('leak unsupported in relu_eccv16') ; else leak = {} ; end
528 |         if ~isempty(res(i).x)
529 |           res(i).dzdx = vl_nnrelu(res(i).x, res(i+1).dzdx, leak{:}) ;
530 |         else
531 |           % if res(i).x is empty, it has been optimized away, so we use this
532 |           % hack (which works only for ReLU):
533 |           res(i).dzdx = vl_nnrelu(res(i+1).x, res(i+1).dzdx, leak{:}) ;
534 |         end
535 |         res(i).dzdx = max(res(i).dzdx, 0);
536 |         
537 |       case 'relu_deconvnet'
538 |         if (isfield(l, 'leak') && l.leak ~= 0)
539 |             error('leak unsupported in relu_deconvnet') ; else leak = {} ; end
540 |         res(i).dzdx = max(res(i+1).dzdx, 0); 
541 |         
542 |       case 'relu_nobackprop'
543 |         if (isfield(l, 'leak') && l.leak ~= 0)
544 |             error('leak unsupported in relu_nobackprop') ; else leak = {} ; end
545 |         res(i).dzdx = res(i+1).dzdx;
546 |         
547 |       case 'sigmoid'
548 |         res(i).dzdx = vl_nnsigmoid(res(i).x, res(i+1).dzdx) ;
549 | 
550 |       case 'noffset'
551 |         res(i).dzdx = vl_nnnoffset(res(i).x, l.param, res(i+1).dzdx) ;
552 | 
553 |       case 'spnorm'
554 |         res(i).dzdx = vl_nnspnorm(res(i).x, l.param, res(i+1).dzdx) ;
555 | 
556 |       case 'dropout'
557 |         if testMode
558 |           res(i).dzdx = res(i+1).dzdx ;
559 |         else
560 |           res(i).dzdx = vl_nndropout(res(i).x, res(i+1).dzdx, ...
561 |                                      'mask', res(i+1).aux) ;
562 |         end
563 | 
564 |       case 'bnorm'
565 |         [res(i).dzdx, dzdw{1}, dzdw{2}, dzdw{3}] = ...
566 |           vl_nnbnorm(res(i).x, l.weights{1}, l.weights{2}, res(i+1).dzdx, ...
567 |                      'epsilon', l.epsilon, ...
568 |                      bnormCudnn{:}) ;
569 |         % multiply the moments update by the number of images in the batch
570 |         % this is required to make the update additive for subbatches
571 |         % and will eventually be normalized away
572 |         dzdw{3} = dzdw{3} * size(res(i).x,4) ;
573 | 
574 |       case 'pdist'
575 |         res(i).dzdx = vl_nnpdist(res(i).x, l.class, ...
576 |           l.p, res(i+1).dzdx, ...
577 |           'noRoot', l.noRoot, ...
578 |           'epsilon', l.epsilon, ...
579 |           'aggregate', l.aggregate, ...
580 |           'instanceWeights', l.instanceWeights) ;
581 | 
582 |       case 'custom'
583 |         res(i) = l.backward(l, res(i), res(i+1)) ;
584 | 
585 |     end % layers
586 | 
587 |     switch l.type
588 |       case {'conv', 'convt', 'bnorm'}
589 |         if ~opts.accumulate
590 |           res(i).dzdw = dzdw ;
591 |         else
592 |           for j=1:numel(dzdw)
593 |             res(i).dzdw{j} = res(i).dzdw{j} + dzdw{j} ;
594 |           end
595 |         end
596 |         dzdw = [] ;
597 |         if ~isempty(opts.parameterServer) && ~opts.holdOn
598 |           for j = 1:numel(res(i).dzdw)
599 |             opts.parameterServer.push(sprintf('l%d_%d',i,j),res(i).dzdw{j}) ;
600 |             res(i).dzdw{j} = [] ;
601 |           end
602 |         end
603 |     end
604 |     if opts.conserveMemory && ~net.layers{i}.precious && i ~= n
605 |       res(i+1).dzdx = [] ;
606 |       res(i+1).x = [] ;
607 |     end
608 |     if gpuMode && opts.sync
609 |       wait(gpuDevice) ;
610 |     end
611 |     res(i).backwardTime = toc(res(i).backwardTime) ;
612 |   end
613 |   if i > 1 && i == backPropLim && opts.conserveMemory && ~net.layers{i}.precious
614 |     res(i).dzdx = [] ;
615 |     res(i).x = [] ;
616 |   end
617 | end
618 | 


--------------------------------------------------------------------------------