├── COPYING
├── README.md
├── core
    └── invert_nn.m
├── dg_setup.m
├── experiments
    ├── data
    │   ├── hog
    │   │   └── img.png
    │   └── stock
    │   │   ├── stock_abstract.jpg
    │   │   └── stock_fish.jpg
    ├── experiment_cnn.m
    ├── experiment_get_dataset.m
    ├── experiment_init.m
    ├── experiment_run.m
    ├── experiment_setup.m
    ├── experiment_shallow.m
    ├── experiment_shallow_quantitative.m
    ├── networks
    │   ├── dsift_net.m
    │   └── hog_net.m
    └── x0_sigma.mat
└── helpers
    ├── cnn_denormalize.m
    ├── cnn_normalize.m
    ├── compute_features.m
    ├── get_cnn_denormalize.m
    └── get_cnn_normalize.m


/COPYING:
--------------------------------------------------------------------------------
 1 | Copyright (C) 2014-15, Aravindh Mahendran and Andrea Vedaldi
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without
 5 | modification, are permitted provided that the following conditions are
 6 | met:
 7 | 1. Redistributions of source code must retain the above copyright
 8 |    notice, this list of conditions and the following disclaimer.
 9 | 2. Redistributions in binary form must reproduce the above copyright
10 |    notice, this list of conditions and the following disclaimer in the
11 |    documentation and/or other materials provided with the
12 |    distribution.
13 | 
14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
15 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
16 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
17 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
18 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
20 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Directory Structure
 2 | -------------------
 3 | <pre>
 4 | .
 5 | +-- core
 6 | |   +-- invert_nn.m - The core optimization lies here
 7 | +-- helpers - Several auxiliary functions that may be useful in general
 8 | +-- experiments - All the code to replicate our experiments
 9 | |   +-- networks
10 | |   |   +-- hog_net.m - The hog and hogb networks are created using this
11 | |   |   +-- dsift_net.m - The dense sift neural network is here
12 | |   |   +-- Other networks used in our experiments can be downloaded from http://www.robots.ox.ac.uk/~aravindh/networks.html
13 | |   +-- data
14 | |   |   +-- hog/img.png - Image used for HOG and DSIFT qualitative results
15 | |   |   +-- stock/ - Contains some more figures for reproducing qualitative results.
16 | +-- ihog - either copy or soft link ihog from Vondrick et. al. This is required to run our experiments with hoggle.
17 | +-- matconvnet - either copy or soft link matconvnet code here. If this is not here, then the setup function will not work.
18 | +-- vlfeat - again either copy or soft copy. If this is not here, then the setup function will not work.
19 | </pre>
20 | 
21 | 
22 | Experiments from the paper
23 | --------------------------
24 | 
25 | To run the experiments used for our publication and replicate their results please follow the instructions below
26 | 
27 | Get the images
28 | 
29 | Download/soft link the imagenet validation images into experiments/data/imagenet12-val
30 | Download/soft link the stock abstrack images into experiments/data/stock
31 | 
32 | Compile ihog, vlfeat and matconvnet as per the instructions given at their respective webpages.
33 | 
34 | ihog: http://web.mit.edu/vondrick/ihog/
35 | 
36 | matconvnet: http://www.vlfeat.org/matconvnet/
37 | 
38 | vlfeat: http://www.vlfeat.org/
39 | 
40 | 
41 | I) CNN experiments - qualitative results
42 | 
43 |     cd experiments;
44 |     experiment_cnn;
45 |     
46 | This might run for several hours and generate a lot of matlab figures. Each figure contains the images used in the paper.
47 | 
48 | II) HOG, HOGle, DSIFT experiments - qualitative results
49 | 
50 |     cd experiments;
51 |     experiment_shallow;
52 |     
53 | Same as before, it will generate matlab figures with the required images.
54 | 
55 | III) HOG, HOGb, HOGgle, DSIFT - quantitative results
56 |     cd experiments;
57 |     experiment_shallow_quantitative.m
58 | 
59 | It will generate mean and std of the normalized reconstruction error across 100 images.
60 | For this it will compute pre-images for 100 images and this will take a very long time.
61 | 
62 | Setting up and running your own networks
63 | ----------------------------------------
64 | 
65 | 1. Create a network (net) that is compatible with matconvnet vl_simplenn function.
66 | 2. Run dg\_setup.m in matlab
67 | 3. Run the network forward to generate a target reference representation y0
68 | 4. Call res = invert\_nn(net, y0, \[options\]);
69 | 5. res.output\{end\} is the required reconstruction.
70 | 


--------------------------------------------------------------------------------
/core/invert_nn.m:
--------------------------------------------------------------------------------
  1 | function res = invert_nn(net, ref, varargin)
  2 | % INVERT  Invert a CNN representation
  3 | 
  4 | opts.learningRate = 0.001*[...
  5 |   ones(1,800), ...
  6 |   0.1 * ones(1,500), ...
  7 |   0.01 * ones(1,500), ...
  8 |   0.001 * ones(1,200), ...
  9 |   0.0001 * ones(1,100) ] ;
 10 | [opts, varargin] = vl_argparse(opts, varargin) ;
 11 | 
 12 | opts.maxNumIterations = numel(opts.learningRate) ;
 13 | opts.objective = 'l2' ; % The experiments in the paper use only l2
 14 | opts.lambdaTV = 10 ; % Coefficient of the TV^\beta regularizer
 15 | opts.lambdaL2 = 08e-10 ; % Coefficient of the L\beta regularizer on the reconstruction
 16 | opts.TVbeta = 2; % The power to which TV norm is raized.
 17 | opts.beta = 6 ; % The \beta of the L\beta regularizer
 18 | opts.momentum = 0.9 ; % Momentum used in the optimization
 19 | opts.numRepeats = 4 ; % Number of reconstructions to generate
 20 | opts.normalize = [] ; % A function handle that normalizes network input
 21 | opts.denormalize = [] ; % A function handle that denormalizes network input
 22 | opts.dropout = 0.5; % Dropout rate for any drop out layers.
 23 | % The L1 loss puts a drop out layer. We didn't experiment with this though.
 24 | opts.filterGroup = NaN ; % Helps select one or the other group of filters.
 25 | % This is used in selecting filters in conv 1 of alex net to see the difference
 26 | % in their properties.
 27 | opts.neigh = +inf ; % To select a small neighborhood of neurons. 
 28 | opts.optim_method = 'gradient-descent'; % Only 'gradient-descent' is currently used
 29 | 
 30 | opts.imgSize = [];
 31 | 
 32 | % Parse the input arguments to override the above defaults
 33 | opts = vl_argparse(opts, varargin) ;
 34 | 
 35 | % Update the number of iterations using the learning rate if required
 36 | if isinf(opts.maxNumIterations)
 37 |   opts.maxNumIterations = numel(opts.learningRate) ;
 38 | end
 39 | 
 40 | % The size of the image that we are trying to obtain
 41 | x0_size = cat(2, opts.imgSize(1:3), opts.numRepeats);
 42 | 
 43 | % x0_sigma is computed using a separate dataset.
 44 | % This is a useful normalization that helps scale the different terms in the
 45 | % optimization.
 46 | load('x0_sigma.mat', 'x0_sigma');
 47 | 
 48 | % Replicate the feature into a block. This is used for multiple inversions in parallel.
 49 | y0 = repmat(ref, [1 1 1 opts.numRepeats]);
 50 | 
 51 | % initial inversion image of size x0_size
 52 | x = randn(x0_size, 'single') ;
 53 | x = x / norm(x(:)) * x0_sigma  ; 
 54 | x_momentum = zeros(x0_size, 'single') ;
 55 | 
 56 | % allow reconstructing a subset of the representation by setting
 57 | % a suitable mask on the features y0
 58 | sf = 1:size(y0,3) ;
 59 | if opts.filterGroup == 1
 60 |   sf= vl_colsubset(sf, 0.5, 'beginning') ;
 61 | elseif opts.filterGroup == 2 ;
 62 |   sf= vl_colsubset(sf, 0.5, 'ending') ;
 63 | end
 64 | nx = min(opts.neigh, size(y0,2)) ;
 65 | ny = min(opts.neigh, size(y0,1)) ;
 66 | sx = (0:nx-1) + ceil((size(y0,2)-nx+1)/2) ;
 67 | sy = (0:ny-1) + ceil((size(y0,1)-ny+1)/2) ;
 68 | mask = zeros(size(y0), 'single') ;
 69 | mask(sy,sx,sf,:) = 1 ;
 70 | y0_sigma = norm(squeeze(y0(find(mask(:))))) ;
 71 | 
 72 | %% Tweak the network by adding a reconstruction loss at the end
 73 | 
 74 | layer_num = numel(net.layers) ; % The layer number which we are reconstructing
 75 | % This is saved here just for printing our progress as optimization proceeds
 76 | 
 77 | switch opts.objective
 78 |   case 'l2'
 79 |     % Add the l2 loss over the network
 80 |     ly.type = 'custom' ;
 81 |     ly.w = y0 ;
 82 |     ly.mask = mask ;
 83 |     ly.forward = @nndistance_forward ;
 84 |     ly.backward = @nndistance_backward ;
 85 |     net.layers{end+1} = ly ;
 86 |   case 'l1'
 87 |     % The L1 loss might want to use a dropout layer. 
 88 |     % This is just a guess and hasn't been tried.
 89 |     ly.type = 'dropout' ;
 90 |     ly.rate = opts.dropout ;
 91 |     net.layers{end+1} = ly ;
 92 |     ly.type = 'custom' ;
 93 |     ly.w = y0 ;
 94 |     ly.mask = mask ;
 95 |     ly.forward = @nndistance1_forward ;
 96 |     ly.backward = @nndistance1_backward ;
 97 |     net.layers{end+1} = ly ;
 98 |   case 'inner'
 99 |     % The inner product loss may be suitable for some networks
100 |     ly.type = 'custom' ;
101 |     ly.w = - y0 .* mask ;
102 |     ly.forward = @nninner_forward ;
103 |     ly.backward = @nninner_backward ;
104 |     net.layers{end+1} = ly ;
105 |   otherwise
106 |     error('unknown opts.objective') ;
107 | end
108 | 
109 | %% --------------------------------------------------------------------
110 | %%                                                 Perform optimisation
111 | %% --------------------------------------------------------------------
112 | 
113 | % Run forward propogation once on the modified network before we
114 | % begin backprop iterations - this is to exploit an optimization in
115 | % vl_simplenn
116 | res = vl_simplenn(net, x); % x is the random initialized image
117 | 
118 | % recored results
119 | output = {} ;
120 | prevlr = 0 ;
121 | 
122 | % Iterate until maxNumIterations to optimize the objective
123 | % and generate the reconstuction
124 | for t=1:opts.maxNumIterations
125 | 
126 |   % Effectively does both forward and backward passes
127 |   res = vl_simplenn(net, x, single(1)) ;
128 | 
129 |   y = res(end-1).x ; % The current best feature we could generate
130 | 
131 |   dr = zeros(size(x),'single'); % The derivative
132 | 
133 |   if opts.lambdaTV > 0 % Cost and derivative for TV\beta norm
134 |     [r_,dr_] = tv(x,opts.TVbeta) ;
135 |     E(2,t) = opts.lambdaTV/2 * r_ ;
136 |     dr = dr + opts.lambdaTV/2 * dr_ ;
137 |   else
138 |     E(2,t) = 0;
139 |   end
140 | 
141 |   if opts.lambdaL2 > 0 % Cost and derivative of L\beta norm
142 |     r_ = sum(x(:).^opts.beta) ;
143 |     dr_ = opts.beta * x.^(opts.beta-1) ;
144 |     E(3,t) = opts.lambdaL2/2 * r_ ;
145 |     dr = dr + opts.lambdaL2/2 * dr_ ;
146 |   else
147 |     E(3,t) = 0;
148 |   end
149 | 
150 |   % Rescale the different costs and add them up
151 |   E(1,t) = res(end).x/(y0_sigma^2);
152 |   E(2:3,t) = E(2:3,t) / (x0_sigma^2) ;
153 |   E(4,t) = sum(E(1:3,t)) ;
154 |   fprintf('iter:%05d sq. rec. err:%8.4g; obj:%8.4g;\n', t, E(1,end), E(4,end)) ;
155 | 
156 |   lr = opts.learningRate(min(t, numel(opts.learningRate))) ;
157 | 
158 |   % when the learning rate changes suddently, it is not
159 |   % possible for the gradient to crrect the momentum properly
160 |   % causing the algorithm to overshoot for several iterations
161 |   if lr ~= prevlr
162 |     fprintf('switching learning rate (%f to %f) and resetting momentum\n', ...
163 |       prevlr, lr) ;
164 |     x_momentum = 0 * x_momentum ;
165 |     prevlr = lr ;
166 |   end
167 | 
168 |   % x_momentum combines the current gradient and the previous gradients
169 |   % with decay (opts.momentum) 
170 |   x_momentum = opts.momentum * x_momentum ...
171 |       - lr * dr ...
172 |       - (lr * x0_sigma^2/y0_sigma^2) * res(1).dzdx;
173 | 
174 |   % This is the main update step (we are updating the the variable
175 |   % along the gradient
176 |   x = x + x_momentum ;
177 | 
178 |   %% -----------------------------------------------------------------------
179 |   %% Plots - Generate several plots to keep track of our progress
180 |   %% -----------------------------------------------------------------------
181 | 
182 |   if mod(t-1,25)==0
183 |     output{end+1} = opts.denormalize(x) ;
184 | 
185 |     figure(1) ; clf ;
186 | 
187 |     subplot(3,2,[1 3]) ;
188 |     if opts.numRepeats > 1
189 |       vl_imarraysc(output{end}) ;
190 |     else
191 |       imagesc(vl_imsc(output{end})) ;
192 |     end
193 |     axis image ; colormap gray ;
194 | 
195 |     subplot(3,2,2) ;
196 |     len = min(1000, numel(y0));
197 |     a = squeeze(y0(1:len)) ;
198 |     b = squeeze(y(1:len)) ;
199 |     plot(1:len,a,'b'); hold on ;
200 |     plot(len+1:2*len,abs(b-a), 'r');
201 |     legend('\Phi_0', '|\Phi-\Phi_0|') ;
202 |     title(sprintf('reconstructed layer %d %s', ...
203 |       layer_num, ...
204 |       net.layers{layer_num}.type)) ;
205 |     legend('ref', 'delta') ;
206 | 
207 |     subplot(3,2,4) ;
208 |     hist(x(:),100) ;
209 |     grid on ;
210 |     title('histogram of x') ;
211 | 
212 |     subplot(3,2,5) ;
213 |     plot(E') ;
214 |     h = legend('recon', 'tv_reg', 'l2_reg', 'tot') ;
215 |     set(h,'color','none') ; grid on ;
216 |     title(sprintf('iter:%d \\lambda_{tv}:%g \\lambda_{l2}:%g rate:%g obj:%s', ...
217 |                   t, opts.lambdaTV, opts.lambdaL2, lr, opts.objective)) ;
218 | 
219 |     subplot(3,2,6) ;
220 |     semilogy(E') ;
221 |     title('log scale') ;
222 |     grid on ;
223 |     drawnow ;
224 | 
225 |   end % end if(mod(t-1,25) == 0)
226 | end % end loop over maxNumIterations
227 | 
228 | 
229 | % Compute the features optained using feedforward on the computed inverse
230 | res_nn = vl_simplenn(net, x);
231 | 
232 | clear res;
233 | res.input = NaN;
234 | res.output = output ;
235 | res.energy = E ;
236 | res.y0 = y0 ;
237 | res.y = res_nn(end-1).x ;
238 | res.opts = opts ;
239 | res.err = res_nn(end).x / y0_sigma^2 ;
240 | 
241 | % --------------------------------------------------------------------
242 | function res_ = nndistance_forward(ly, res, res_)
243 | % --------------------------------------------------------------------
244 | res_.x = nndistance(res.x, ly.w, ly.mask) ;
245 | 
246 | % --------------------------------------------------------------------
247 | function res = nndistance_backward(ly, res, res_)
248 | % --------------------------------------------------------------------
249 | res.dzdx = nndistance(res.x, ly.w, ly.mask, res_.dzdx) ;
250 | 
251 | % --------------------------------------------------------------------
252 | function y = nndistance(x,w,mask,dzdy)
253 | % --------------------------------------------------------------------
254 | if nargin <= 3
255 |   d = x - w ;
256 |   y = sum(sum(sum(sum(d.*d.*mask)))) ;
257 | else
258 |   y = dzdy * 2 * (x - w) .* mask ;
259 | end
260 | 
261 | % --------------------------------------------------------------------
262 | function res_ = l1loss_forward(ly, res, res_)
263 | % --------------------------------------------------------------------
264 | res_.x = ly.w*sum(abs(res.x(:)));
265 | 
266 | 
267 | % --------------------------------------------------------------------
268 | function res = l1loss_backward(ly, res, res_)
269 | % --------------------------------------------------------------------
270 | res.dzdx = zeros(size(res.x), 'single');
271 | res.dzdx(res.x > 0) = single(ly.w)*res_.dzdx;
272 | res.dzdx(res.x < 0) = -single(ly.w)*res_.dzdx;
273 | res.dzdx(res.x == 0) = single(0);
274 | 
275 | % --------------------------------------------------------------------
276 | function res_ = nndistance1_forward(ly, res, res_)
277 | % --------------------------------------------------------------------
278 | res_.x = nndistance1(res.x, ly.w, ly.mask) ;
279 | 
280 | % --------------------------------------------------------------------
281 | function res = nndistance1_backward(ly, res, res_)
282 | % --------------------------------------------------------------------
283 | res.dzdx = nndistance1(res.x, ly.w, ly.mask, res_.dzdx) ;
284 | 
285 | % --------------------------------------------------------------------
286 | function y = nndistance1(x,w,mask,dzdy)
287 | % --------------------------------------------------------------------
288 | if nargin <= 3
289 |   d = x - w ;
290 |   y = sum(sum(sum(sum(abs(d).*mask)))) ;
291 | else
292 |   y = dzdy * sign(x - w) .* mask ;
293 | end
294 | 
295 | % --------------------------------------------------------------------
296 | function res_ = nninner_forward(ly, res, res_)
297 | % --------------------------------------------------------------------
298 | res_.x = nninner(res.x, ly.w) ;
299 | 
300 | % --------------------------------------------------------------------
301 | function res = nninner_backward(ly, res, res_)
302 | % --------------------------------------------------------------------
303 | res.dzdx = nninner(res.x, ly.w, res_.dzdx) ;
304 | 
305 | % --------------------------------------------------------------------
306 | function y = nninner(x,w,dzdy)
307 | % --------------------------------------------------------------------
308 | if nargin <= 2
309 |   y = sum(sum(sum(sum(w.*x)))) ;
310 | else
311 |   y = dzdy * w ;
312 | end
313 | 
314 | % --------------------------------------------------------------------
315 | function [e, dx] = tv(x,beta)
316 | % --------------------------------------------------------------------
317 | if(~exist('beta', 'var'))
318 |   beta = 1; % the power to which the TV norm is raized
319 | end
320 | d1 = x(:,[2:end end],:,:) - x ;
321 | d2 = x([2:end end],:,:,:) - x ;
322 | v = sqrt(d1.*d1 + d2.*d2).^beta ;
323 | e = sum(sum(sum(sum(v)))) ;
324 | if nargout > 1
325 |   d1_ = (max(v, 1e-5).^(2*(beta/2-1)/beta)) .* d1;
326 |   d2_ = (max(v, 1e-5).^(2*(beta/2-1)/beta)) .* d2;
327 |   d11 = d1_(:,[1 1:end-1],:,:) - d1_ ;
328 |   d22 = d2_([1 1:end-1],:,:,:) - d2_ ;
329 |   d11(:,1,:,:) = - d1_(:,1,:,:) ;
330 |   d22(1,:,:,:) = - d2_(1,:,:,:) ;
331 |   dx = beta*(d11 + d22);
332 |   if(any(isnan(dx)))
333 |   end
334 | end
335 | 
336 | % --------------------------------------------------------------------
337 | function test_tv()
338 | % --------------------------------------------------------------------
339 | x = randn(5,6,1,1) ;
340 | [e,dr] = tv(x,6) ;
341 | vl_testder(@(x) tv(x,6), x, 1, dr, 1e-3) ;
342 | 


--------------------------------------------------------------------------------
/dg_setup.m:
--------------------------------------------------------------------------------
1 | run vlfeat/toolbox/vl_setup ;
2 | run matconvnet/matlab/vl_setupnn ;
3 | addpath('core');
4 | addpath('helpers');
5 | 


--------------------------------------------------------------------------------
/experiments/data/hog/img.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aravindhm/deep-goggle/cc667f02dd079f060542594a3592d6b7feb1137c/experiments/data/hog/img.png


--------------------------------------------------------------------------------
/experiments/data/stock/stock_abstract.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aravindhm/deep-goggle/cc667f02dd079f060542594a3592d6b7feb1137c/experiments/data/stock/stock_abstract.jpg


--------------------------------------------------------------------------------
/experiments/data/stock/stock_fish.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aravindhm/deep-goggle/cc667f02dd079f060542594a3592d6b7feb1137c/experiments/data/stock/stock_fish.jpg


--------------------------------------------------------------------------------
/experiments/experiment_cnn.m:
--------------------------------------------------------------------------------
  1 | function experiment_cnn()
  2 | % Run some CNN experiments
  3 | 
  4 | experiment_setup;
  5 | 
  6 | %images = experiment_get_dataset('imnet');
  7 | images_for_progression = {'data/imagenet12-val/ILSVRC2012_val_00000013.JPEG'};
  8 | 
  9 | images_for_spiky = {'data/imagenet12-val/ILSVRC2012_val_00000043.JPEG'};
 10 | 
 11 | images_for_TVbeta = {'data/imagenet12-val/ILSVRC2012_val_00000043.JPEG'};
 12 | 
 13 | images_for_neigh = {'data/imagenet12-val/ILSVRC2012_val_00000013.JPEG'};
 14 | 
 15 | images_for_diversity = {'data/imagenet12-val/ILSVRC2012_val_00000011.JPEG',...
 16 |      'data/imagenet12-val/ILSVRC2012_val_00000014.JPEG',...
 17 |      'data/imagenet12-val/ILSVRC2012_val_00000018.JPEG',...
 18 |      'data/imagenet12-val/ILSVRC2012_val_00000023.JPEG',...
 19 |      'data/imagenet12-val/ILSVRC2012_val_00000033.JPEG'};
 20 | 
 21 | images_for_multiple = {'data/imagenet12-val/ILSVRC2012_val_00000043.JPEG',...
 22 |      'data/stock/stock_abstract.jpg'};
 23 | 
 24 | images_for_groups = {'data/stock/stock_abstract.jpg', ...
 25 |   'data/stock/stock_fish.jpg'};
 26 | 
 27 | images_for_teaser = {'data/imagenet12-val/ILSVRC2012_val_00000024.JPEG'};
 28 | 
 29 | % -------------------------------------------------------------------------
 30 | %                                                         Setup experiments
 31 | % -------------------------------------------------------------------------
 32 | 
 33 | exp = {} ;
 34 | ver = 'results' ;
 35 | opts.learningRate = 0.004 * [...
 36 |   ones(1,100), ...
 37 |   0.1 * ones(1,200), ...
 38 |   0.01 * ones(1,100)];
 39 | opts.objective = 'l2' ;
 40 | opts.beta = 6 ;
 41 | opts.lambdaTV = 1e2 ;
 42 | opts.lambdaL2 = 8e-10 ;
 43 | 
 44 | opts1 = opts;
 45 | opts1.lambdaTV = 1e0 ;
 46 | opts1.TVbeta = 2;
 47 | 
 48 | opts2 = opts ;
 49 | opts2.lambdaTV = 1e1 ;
 50 | opts2.TVbeta = 2;
 51 | 
 52 | opts3 = opts ;
 53 | opts3.lambdaTV = 1e2 ;
 54 | opts3.TVbeta = 2;
 55 | 
 56 | 
 57 | % -------------------------------------------------------------------------
 58 | %                                                           Run experiments
 59 | % -------------------------------------------------------------------------
 60 | 
 61 | if 1 % Figure 6
 62 |   for i = 1:numel(images_for_progression) 
 63 |     for layer = 1:6 % Different regularizer values for each of the layers
 64 |       exp{end+1} = experiment_init('caffe-ref', layer, images_for_progression{i}, ver, 'cnn', opts1) ;
 65 |     end
 66 |     for layer=7:12
 67 |       exp{end+1} = experiment_init('caffe-ref', layer, images_for_progression{i}, ver, 'cnn', opts2);
 68 |     end
 69 |     for layer=13:20
 70 |       exp{end+1} = experiment_init('caffe-ref', layer, images_for_progression{i}, ver, 'cnn', opts3);
 71 |     end
 72 |   end
 73 | end
 74 | 
 75 | if 1 % Figure 2
 76 |   opts4 = opts ;
 77 |   opts4.lambdaTV = 1e2 ;
 78 |   opts4.TVbeta = 1;
 79 | 
 80 |   opts5 = opts ;
 81 |   opts5.lambdaTV = 1e2 ;
 82 |   opts5.TVbeta = 2;
 83 |   for i = 1:numel(images_for_spiky)
 84 |     exp{end+1} = experiment_init('caffe-ref', 4, images_for_spiky{i}, ver, 'cnn_spiky1', opts4);
 85 |     exp{end+1} = experiment_init('caffe-ref', 4, images_for_spiky{i}, ver, 'cnn_spiky2', opts5);
 86 |   end
 87 | end
 88 | 
 89 | if 1 % Figure 8
 90 |   for i = 1:numel(images_for_TVbeta)
 91 |     exp{end+1} = experiment_init('caffe-ref', 20, images_for_TVbeta{i}, ver, 'cnn_tvbeta1', opts1);
 92 |     exp{end+1} = experiment_init('caffe-ref', 20, images_for_TVbeta{i}, ver, 'cnn_tvbeta2', opts2);
 93 |     exp{end+1} = experiment_init('caffe-ref', 20, images_for_TVbeta{i}, ver, 'cnn_tvbeta3', opts3);
 94 |   end
 95 | end
 96 | 
 97 | if 1 % Figure 9
 98 |   opts7 = opts3;
 99 |   opts7.learningRate = 0.004 * [...
100 |     0.1* ones(1,200), ...
101 |     0.01 * ones(1,200), ...
102 |     0.001 * ones(1,200)];
103 |   for i = 1:numel(images_for_neigh)
104 |     for l = 1:15
105 |       exp{end+1} = experiment_init('caffe-ref', l, images_for_neigh{i}, ver, 'cnn_neigh', opts7, 'neigh', 5);
106 |     end
107 |   end
108 | end
109 | 
110 | if 1 % Figure 11
111 |   for i = 1:numel(images_for_diversity)
112 |     exp{end+1} = experiment_init('caffe-ref', 15, images_for_diversity{i}, ver, 'cnn_diversity', opts3);
113 |   end
114 | end
115 | 
116 | if 1 % Figure 7
117 |   opts6_3 = opts3;
118 |   opts6_3.numRepeats = 6;
119 | 
120 |   opts6_2 = opts2;
121 |   opts6_2.numRepeats = 6;
122 | 
123 |   opts6_1 = opts1;
124 |   opts6_1.numRepeats = 6;
125 | 
126 |   for i = 1:numel(images_for_multiple)
127 |     for l=[15]
128 |       exp{end+1} = experiment_init('caffe-ref', l, images_for_multiple{i}, ver, 'cnn_multiple', opts6_1);
129 |     end
130 |     for l=[17]
131 |       exp{end+1} = experiment_init('caffe-ref', l, images_for_multiple{i}, ver, 'cnn_multiple', opts6_2);
132 |     end
133 |     for l=[19,20]
134 |       exp{end+1} = experiment_init('caffe-ref', l, images_for_multiple{i}, ver, 'cnn_multiple', opts6_3);
135 |     end
136 |   end
137 | end
138 | 
139 | if 1 % Figure 10
140 |   for i = 1:numel(images_for_groups)
141 |     for l=[1,4,8]
142 |       exp{end+1} = experiment_init('caffe-ref', l, images_for_groups{i}, ver, 'cnn_group1', opts1, 'filterGroup', 1);
143 |       exp{end+1} = experiment_init('caffe-ref', l, images_for_groups{i}, ver, 'cnn_group2', opts1, 'filterGroup', 2);
144 |     end
145 |   end
146 | end
147 | 
148 | if 1 % Figure 1
149 |   opts10 = opts3;
150 |   opts10.numRepeats = 6;
151 |   for i = 1:numel(images_for_teaser)
152 |     l = 20;
153 |     exp{end+1} = experiment_init('caffe-ref', l, images_for_teaser{i}, ver, 'cnn_teaser', opts10);
154 |   end
155 | end
156 | 
157 | experiment_run(exp) ;
158 | 
159 | 
160 | 
161 | % -------------------------------------------------------------------------
162 | %                                                        Accumulate results
163 | % -------------------------------------------------------------------------
164 | figure;
165 | subplot(1,2,1);
166 | imshow('data/results/cnn_spiky1/ILSVRC2012_val_00000043/l04-recon.png');
167 | title('Figure 2. Effect of \beta. \beta = 1');
168 | subplot(1,2,2);
169 | imshow('data/results/cnn_spiky2/ILSVRC2012_val_00000043/l04-recon.png');
170 | title('Figure 2. Effect of \beta. \beta = 2');
171 | 
172 | figure;
173 | subplot(1,3,1);
174 | imshow('data/results/cnn_tvbeta1/ILSVRC2012_val_00000043/l20-recon.png');
175 | subplot(1,3,2);
176 | imshow('data/results/cnn_tvbeta2/ILSVRC2012_val_00000043/l20-recon.png');
177 | title('Figure 8. Effect of V^\beta regularization on CNNs');
178 | subplot(1,3,3);
179 | imshow('data/results/cnn_tvbeta3/ILSVRC2012_val_00000043/l20-recon.png');
180 | 
181 | figure;
182 | for l=1:20
183 |   subplot(4,5,l);
184 |   imshow(sprintf('data/results/cnn/ILSVRC2012_val_00000013/l%02d-recon.png', l));
185 | end
186 | subplot(4,5,3);
187 | title('Figure 6. CNN reconstruction.');
188 | 
189 | figure;
190 | for l=1:15
191 |   subplot(3,5,l);
192 |   imshow(sprintf('data/results/cnn_neigh/ILSVRC2012_val_00000013/l%02d-recon.png', l));
193 | end
194 | subplot(3,5,3);
195 | title('Figure 9. CNN receptive fields.');
196 | 
197 | figure;
198 | subplot(1,5,1);
199 | imshow('data/results/cnn_diversity/ILSVRC2012_val_00000011/l15-recon.png');
200 | subplot(1,5,2);
201 | imshow('data/results/cnn_diversity/ILSVRC2012_val_00000014/l15-recon.png');
202 | subplot(1,5,3);
203 | imshow('data/results/cnn_diversity/ILSVRC2012_val_00000018/l15-recon.png');
204 | title('Figure 11. Diversity in the CNN model');
205 | subplot(1,5,4);
206 | imshow('data/results/cnn_diversity/ILSVRC2012_val_00000023/l15-recon.png');
207 | subplot(1,5,5);
208 | imshow('data/results/cnn_diversity/ILSVRC2012_val_00000033/l15-recon.png');
209 | 
210 | figure;
211 | subplot(2,4,1);
212 | imshow('data/results/cnn_multiple/ILSVRC2012_val_00000043/l15-recon.png');
213 | title('Figure 7: CNN invariances. Pool5');
214 | subplot(2,4,2);
215 | imshow('data/results/cnn_multiple/ILSVRC2012_val_00000043/l17-recon.png');
216 | title('Figure 7: CNN invariances. relu6');
217 | subplot(2,4,3);
218 | imshow('data/results/cnn_multiple/ILSVRC2012_val_00000043/l19-recon.png');
219 | title('Figure 7: CNN invariances. relu7');
220 | subplot(2,4,4);
221 | imshow('data/results/cnn_multiple/ILSVRC2012_val_00000043/l20-recon.png');
222 | title('Figure 7: CNN invariances. fc8');
223 | 
224 | subplot(2,4,5);
225 | imshow('data/results/cnn_multiple/stock_abstract/l15-recon.png');
226 | title('Figure 7: CNN invariances. Pool5');
227 | subplot(2,4,6);
228 | imshow('data/results/cnn_multiple/stock_abstract/l17-recon.png');
229 | title('Figure 7: CNN invariances. relu6');
230 | subplot(2,4,7);
231 | imshow('data/results/cnn_multiple/stock_abstract/l19-recon.png');
232 | title('Figure 7: CNN invariances. relu7');
233 | subplot(2,4,8);
234 | imshow('data/results/cnn_multiple/stock_abstract/l20-recon.png');
235 | title('Figure 7: CNN invariances. fc8');
236 | 
237 | figure;
238 | subplot(2,6,1);
239 | imshow('data/results/cnn_group1/stock_abstract/l01-recon.png');
240 | subplot(2,6,2);
241 | imshow('data/results/cnn_group1/stock_abstract/l04-recon.png');
242 | subplot(2,6,3);
243 | imshow('data/results/cnn_group1/stock_abstract/l08-recon.png');
244 | title('Figure 10: CNN neural streams');
245 | subplot(2,6,4);
246 | imshow('data/results/cnn_group2/stock_abstract/l01-recon.png');
247 | subplot(2,6,5);
248 | imshow('data/results/cnn_group2/stock_abstract/l04-recon.png');
249 | subplot(2,6,6);
250 | imshow('data/results/cnn_group2/stock_abstract/l08-recon.png');
251 | subplot(2,6,7);
252 | imshow('data/results/cnn_group1/stock_fish/l01-recon.png');
253 | subplot(2,6,8);
254 | imshow('data/results/cnn_group1/stock_fish/l04-recon.png');
255 | subplot(2,6,9);
256 | imshow('data/results/cnn_group1/stock_fish/l08-recon.png');
257 | subplot(2,6,10);
258 | imshow('data/results/cnn_group2/stock_fish/l01-recon.png');
259 | subplot(2,6,11);
260 | imshow('data/results/cnn_group2/stock_fish/l04-recon.png');
261 | subplot(2,6,12);
262 | imshow('data/results/cnn_group2/stock_fish/l08-recon.png');
263 | 
264 | figure;
265 | imshow('data/results/cnn_teaser/ILSVRC2012_val_00000024/l20-recon.png');
266 | title('Figure 1: What is encoded by a CNN?');
267 | 


--------------------------------------------------------------------------------
/experiments/experiment_get_dataset.m:
--------------------------------------------------------------------------------
 1 | function images = experiment_get_dataset(varargin)
 2 | % images = experiment_get_dataset(varargin) - Collect all the images
 3 | %   from one of more datasets
 4 | % Eg: images = experiment_get_dataset('stock') - to get the stock images
 5 | % Use any subset of 
 6 | %  'copydays', 'stock', 'imnet', 'imnet-lite', 'imnet-nogreen', 'imnet-large', 'hoggle'
 7 | 
 8 | if nargin == 0
 9 |   subsets = {'stock', 'imnet', 'hoggle'} ;
10 | else
11 |   subsets = varargin ;
12 | end
13 | 
14 | images = {} ;
15 | for i=1:numel(subsets)
16 | 
17 |   switch subsets{i}
18 | 
19 |     case 'copydays'
20 |       % copydays dataset
21 |       tmp = dir('/users/aravindh/work/datasets/copydays/*.jpg') ;
22 |       copydays = cellfun(@(x) fullfile('/users/aravindh/work/datasets/copydays/', x), {tmp(:).name}, 'uniform', false);
23 |       images = horzcat(images, copydays) ; 
24 | 
25 |     case 'stock'
26 |       % stock images
27 |       tmp = vertcat(...
28 |         dir('data/stock_images/*.jpg'), ...
29 |         dir('data/stock_images/*.png')) ;
30 |       stock = cellfun(@(x) fullfile('data/stock_images', x), {tmp(~[tmp.isdir]).name}, 'uniform', false);
31 |       images = horzcat(images, stock) ;
32 | 
33 |     case 'imnet'
34 |       % image net images. This is a large scale experiment.
35 |       tmp = dir('data/imagenet12-val/*.JPEG') ;
36 |       imnet = cellfun(@(x) fullfile('data/imagenet12-val/', x), {tmp(1:100).name}, 'uniform', false);
37 |       imnet_large = cellfun(@(x) fullfile('data/imagenet12-val/', x), {tmp(101:500).name}, 'uniform', false);
38 |       images = horzcat(images, imnet) ;
39 | 
40 |     case 'imnet-lite'
41 |       % image net images tiny subset
42 |       tmp = dir('data/imagenet12-val/*.JPEG') ;
43 |       imnet = cellfun(@(x) fullfile('data/imagenet12-val/', x), {tmp(1:100).name}, 'uniform', false);
44 |       imnet_large = cellfun(@(x) fullfile('data/imagenet12-val/', x), {tmp(101:500).name}, 'uniform', false);
45 |       images = horzcat(images, imnet([13 24 43 70])) ;% [1 2 5 8 17 18 30 34 48 56]
46 | 
47 |     case 'imnet-nogreen'
48 |       % image net images which are not green. To see if the greenish output 
49 |       % is a property of the prior or the network.
50 |       tmp = dir('data/imagenet12-val/ILSVRC2012_val_000000*.JPEG') ;
51 |       imnet = cellfun(@(x) fullfile('data/imagenet12-val/', x), {tmp(1:99).name}, 'uniform', false);
52 |       images = horzcat(images, imnet( [1 2 5 17 18 30 34 48 56 70] ));
53 | 
54 |     case 'imnet-large'
55 |       % image net images
56 |       tmp = dir('data/imagenet12-val/*.JPEG') ;
57 |       imnet_large = cellfun(@(x) fullfile('data/imagenet12-val/', x), {tmp(101:500).name}, 'uniform', false);
58 |       images = horzcat(images, imnet_large);
59 | 
60 |     case 'hoggle'
61 |       % hoggle images
62 |       hoggle{1} = 'data/hoggle/hoggle-orig-1.jpg' ;
63 |       hoggle{2} = 'data/hoggle/hoggle-orig-2.jpg' ;
64 |       images = horzcat(images, hoggle) ;
65 | 
66 |   end
67 | end
68 | 
69 | 


--------------------------------------------------------------------------------
/experiments/experiment_init.m:
--------------------------------------------------------------------------------
 1 | function exp = experiment_init(model, layer, imageName, prefix, suffix, varargin)
 2 | % Initialize an experiment
 3 | 
 4 | 
 5 | if nargin < 4
 6 |   prefix = '' ;
 7 | end
 8 | 
 9 | if nargin < 6
10 |   suffix = '' ;
11 | end
12 | 
13 | [imageDir,imageName,imageExt] = fileparts(imageName) ;
14 | if isempty(imageDir), imageDir = 'data/images' ; end
15 | 
16 | exp.expDir = fullfile('data', prefix, suffix) ;
17 | exp.model = model ;
18 | exp.layer = layer ;
19 | exp.name = imageName ;
20 | exp.useHoggle = false ;
21 | exp.path = fullfile(imageDir, [imageName, imageExt]) ;
22 | exp.opts.dropout = 0 ;
23 | exp.opts.neigh = +inf ;
24 | exp.opts.filterGroup = NaN ;
25 | exp.opts.objective = 'l2' ;
26 | exp.opts.learningRate = 0.1 * ones(1,100) ;
27 | exp.opts.maxNumIterations = +inf ;
28 | exp.opts.beta = 2 ;
29 | exp.opts.lambdaTV = 100 ;
30 | exp.opts.lambdaL2 = 0.1 ;
31 | exp.opts.TVbeta = 1;
32 | exp.opts.numRepeats = 1 ;
33 | exp.opts.optim_method = 'gradient-descent';
34 | 
35 | [exp,varargin] = vl_argparse(exp, varargin) ;
36 | exp.opts = vl_argparse(exp.opts, varargin) ;
37 | 
38 | 
39 | 


--------------------------------------------------------------------------------
/experiments/experiment_run.m:
--------------------------------------------------------------------------------
  1 | function experiment_run(exp)
  2 | 
  3 | if matlabpool('size') > 0
  4 |   parfor i=1:numel(exp) % Easily run lots of experiments on a cluster
  5 |     run_one(exp{i}) ;
  6 |   end
  7 | else
  8 |   for i=1:numel(exp)
  9 |     ts = tic;
 10 |     fprintf(1, 'Starting an expeirment');
 11 |     run_one(exp{i}) ;
 12 |     fprintf(1, 'done an expeirment');
 13 |     toc(ts);
 14 |   end
 15 | end
 16 | end
 17 | 
 18 | % -------------------------------------------------------------------------
 19 | function run_one(exp)
 20 | % -------------------------------------------------------------------------
 21 | 
 22 | expPath = fullfile(exp.expDir, exp.name) ;
 23 | expName = sprintf('l%02d', exp.layer) ;
 24 | if ~exist(expPath, 'dir'), mkdir(expPath) ; end
 25 | if exist(fullfile(expPath, [expName '.mat'])), return ; end
 26 | 
 27 | fprintf('running experiment %s\n', exp.name) ;
 28 | 
 29 | % read image
 30 | im = imread(exp.path) ;
 31 | if size(im,3) == 1, im = cat(3,im,im,im) ; end
 32 | 
 33 | if exp.useHoggle
 34 |   run_one_hoggle(exp, expPath, expName, im) ;
 35 |   return ;
 36 | end
 37 | 
 38 | switch exp.model
 39 |   case 'caffe-ref'
 40 |     net = load('networks/imagenet-caffe-ref.mat') ;
 41 |     net = vl_simplenn_tidy(net) ;
 42 |     exp.opts.normalize = get_cnn_normalize(net.meta.normalization) ;
 43 |     exp.opts.denormalize = get_cnn_denormalize(net.meta.normalization) ;
 44 |     exp.opts.imgSize = net.meta.normalization.imageSize;
 45 |   case 'caffe-mitplaces'
 46 |     net = load('networks/places-caffe-ref-upgraded.mat');
 47 |     net = vl_simplenn_tidy(net) ;
 48 |     exp.opts.normalize = get_cnn_normalize(net.meta.normalization) ;
 49 |     exp.opts.denormalize = get_cnn_denormalize(net.meta.normalization) ;
 50 |     exp.opts.imgSize = net.meta.normalization.imageSize;
 51 |   case 'caffe-alex'
 52 |     net = load('networks/imagenet-caffe-alex.mat') ;
 53 |     net = vl_simplenn_tidy(net) ;
 54 |     exp.opts.normalize = get_cnn_normalize(net.meta.normalization) ;
 55 |     exp.opts.denormalize = get_cnn_denormalize(net.meta.normalization) ;
 56 |     exp.opts.imgSize = net.meta.normalization.imageSize;
 57 |   case 'dsift'
 58 |     net = dsift_net(5) ;
 59 |     exp.opts.normalize = @(x) 255 * rgb2gray(im2single(x)) - 128 ;
 60 |     exp.opts.denormalize = @(x) cat(3,x,x,x) + 128 ;
 61 |     exp.opts.imgSize = [size(im, 1), size(im, 2), 1];
 62 |   case 'hog'
 63 |     net = hog_net(8) 
 64 |     exp.opts.normalize = @(x) 255 * rgb2gray(im2single(x)) - 128  ;
 65 |     exp.opts.denormalize = @(x) cat(3,x,x,x) + 128 ;
 66 |     exp.opts.imgSize = [size(im,1), size(im,2), 1];
 67 |   case 'hogb'
 68 |     net = hog_net(8, 'bilinearOrientations', true) ;
 69 |     exp.opts.normalize = @(x) 255 * rgb2gray(im2single(x)) - 128  ;
 70 |     exp.opts.denormalize = @(x) cat(3,x,x,x) + 128 ;
 71 |     exp.opts.imgSize = [size(im, 1), size(im, 2), 1];
 72 | end
 73 | 
 74 | net = vl_simplenn_tidy(net);
 75 | 
 76 | if isinf(exp.layer), exp.layer = numel(net.layers) ; end
 77 | net.layers = net.layers(1:exp.layer) ;
 78 | 
 79 | feats = compute_features(net, im, exp.opts);
 80 | input = im;
 81 | 
 82 | % run experiment
 83 | args = expandOpts(exp.opts) ;
 84 | if(strcmp(exp.opts.optim_method , 'gradient-descent'))
 85 |   res = invert_nn(net, feats, args{:}) ;
 86 | else
 87 |   fprintf(1, 'Unknown optimization method %s\n', exp.opts.optim_method);
 88 |   return;
 89 | end
 90 | 
 91 | if isfield(net.layers{end}, 'name')
 92 |   res.name = net.layers{end}.name ;
 93 | else
 94 |   res.name = sprintf('%s%d', net.layers{end}.type, exp.layer) ;
 95 | end
 96 | 
 97 | % save images
 98 | if size(res.output{end},4) > 1
 99 |   im = vl_imarraysc(res.output{end}) ;
100 | else
101 |   im = vl_imsc(res.output{end}) ;
102 | end
103 | vl_printsize(1) ;
104 | print('-dpdf',  fullfile(expPath, [expName '-opt.pdf'])) ;
105 | imwrite(input / 255, fullfile(expPath, [expName '-orig.png'])) ;
106 | imwrite(im, fullfile(expPath, [expName '-recon.png'])) ;
107 | 
108 | % save videos
109 | makeMovieFromCell(fullfile(expPath, [expName '-evolution']), res.output) ;
110 | makeMovieFromArray(fullfile(expPath, [expName '-recon']), res.output{end}) ;
111 | 
112 | % save results
113 | res.output = res.output{end} ; % too much disk space
114 | save(fullfile(expPath, [expName '.mat']), '-struct', 'res');
115 | end
116 | 
117 | % -------------------------------------------------------------------------
118 | function run_one_hoggle(exp, expPath, expName, im)
119 | % -------------------------------------------------------------------------
120 | 
121 | addpath ../ihog
122 | addpath ../ihog/internal
123 | addpath ../ihog/spams
124 | addpath ../ihog/spams/src_release
125 | addpath ../ihog/spams/build
126 | 
127 | % obtain reconstructions and corresponding HOGs
128 | im = rgb2gray(im2single(im)) ;
129 | hog = features(double(repmat(im,[1 1 3])),8) ;
130 | imrec = repmat(invertHOG(hog), [1 1 3]) ;
131 | hog_ = features(imrec,8) ;
132 | 
133 | % ignore as we do the texture components in the evaluation
134 | hog = hog(:,:,1:27) ;
135 | hog_ = hog_(:,:,1:27) ;
136 | del = hog_ - hog ;
137 | 
138 | res.input = 255* im ;
139 | res.output = imrec ;
140 | res.err = norm(del(:))^2 / norm(hog(:))^2 ;
141 | res.y = hog_;
142 | res.y0 = hog;
143 | res.name = 'hog' ;
144 | 
145 | imhog = imresize(showHOG(hog), [size(res.input,1), size(res.input,2)], 'bicubic') ;
146 | 
147 | % save images
148 | im = vl_imsc(imrec) ;
149 | vl_printsize(1) ;
150 | imwrite(res.input / 255, fullfile(expPath, [expName '-orig.png'])) ;
151 | imwrite(vl_imsc(imhog), fullfile(expPath, [expName '-hog.png'])) ;
152 | imwrite(im, fullfile(expPath, [expName '-recon.png'])) ;
153 | 
154 | % save results
155 | save(fullfile(expPath, [expName '.mat']), '-struct', 'res');
156 | 
157 | end
158 | 
159 | % -------------------------------------------------------------------------
160 | function args = expandOpts(opts)
161 | % -------------------------------------------------------------------------
162 | args = horzcat(fieldnames(opts), struct2cell(opts))' ;
163 | end
164 | 
165 | % -------------------------------------------------------------------------
166 | function makeMovieFromCell(moviePath, x)
167 | % -------------------------------------------------------------------------
168 | writerObj = VideoWriter(moviePath,'Motion JPEG AVI');
169 | open(writerObj) ;
170 | for k = 1:numel(x)
171 |   if size(x{k},4) > 1
172 |     im = vl_imarraysc(x{k}) ;
173 |   else
174 |     im = vl_imsc(x{k}) ;
175 |   end
176 |   writeVideo(writerObj,im2frame(double(im)));
177 | end
178 | close(writerObj);
179 | end
180 | 
181 | % -------------------------------------------------------------------------
182 | function makeMovieFromArray(moviePath, x)
183 | % -------------------------------------------------------------------------
184 | writerObj = VideoWriter(moviePath,'Motion JPEG AVI');
185 | open(writerObj) ;
186 | for k = 1:size(x,4)
187 |   im = vl_imsc(x(:,:,:,k)) ;
188 |   writeVideo(writerObj,im2frame(double(im)));
189 | end
190 | close(writerObj);
191 | end
192 | 


--------------------------------------------------------------------------------
/experiments/experiment_setup.m:
--------------------------------------------------------------------------------
 1 | addpath('../core');
 2 | addpath('../helpers');
 3 | 
 4 | run ../vlfeat/toolbox/vl_setup ;
 5 | run ../matconvnet/matlab/vl_setupnn ;
 6 | 
 7 | if(~exist('data', 'dir'))
 8 |   mkdir('data');
 9 | end
10 | 
11 | if(~exist('data/results', 'dir'))
12 |   mkdir('data/results');
13 | end
14 | 
15 | if(~exist('networks/imagenet-caffe-ref.mat', 'file'))
16 |   cmd='wget http://www.robots.ox.ac.uk/~aravindh/imagenet-caffe-ref.mat -O networks/imagenet-caffe-ref.mat';
17 |   system(cmd);
18 |   clear cmd;
19 | end
20 | 
21 | addpath('networks')
22 | 
23 | 


--------------------------------------------------------------------------------
/experiments/experiment_shallow.m:
--------------------------------------------------------------------------------
 1 | function experiment_shallow()
 2 | % Run some CNN experiments
 3 | 
 4 | experiment_setup;
 5 | 
 6 | images = {'data/hog/img.png'};
 7 | 
 8 | % -------------------------------------------------------------------------
 9 | %                                                         Setup experiments
10 | % -------------------------------------------------------------------------
11 | 
12 | exp = {} ;
13 | ver = 'results' ;
14 | opts.learningRate = 0.004 * [...
15 |   ones(1,200), ...
16 |   0.1 * ones(1,200), ...
17 |   0.01 * ones(1,200),...
18 |   0.001 * ones(1,200)];
19 | opts.objective = 'l2' ;
20 | opts.beta = 6 ;
21 | opts.lambdaTV = 1e2 ;
22 | opts.lambdaL2 = 8e-10 ;
23 | opts.numRepeats = 1;
24 | 
25 | opts1 = opts;
26 | opts1.lambdaTV = 1e0 ;
27 | opts1.TVbeta = 2;
28 | 
29 | opts2 = opts;
30 | opts2.lambdaTV = 1e1 ;
31 | opts2.TVbeta = 2;
32 | 
33 | opts3 = opts;
34 | opts3.lambdaTV = 1e2 ;
35 | opts3.TVbeta = 2;
36 | 
37 | % -------------------------------------------------------------------------
38 | %                                                           Run experiments
39 | % -------------------------------------------------------------------------
40 | 
41 | for i = 1:numel(images) 
42 |   exp{end+1} = experiment_init('hog', inf, images{i}, ver, 'hog', opts1);
43 |   exp{end+1} = experiment_init('hog', inf, images{i}, ver, 'hog2', opts2);
44 |   exp{end+1} = experiment_init('hog', inf, images{i}, ver, 'hog3', opts3);
45 | 
46 |   exp{end+1} = experiment_init('hog', inf, images{i}, ver, 'hoggle', opts1, 'useHoggle', true);
47 |   exp{end+1} = experiment_init('dsift', inf, images{i}, ver, 'dsift', opts1);
48 |   exp{end+1} = experiment_init('hogb', inf, images{i}, ver, 'hogb', opts1);
49 | end
50 | 
51 | experiment_run(exp) ;
52 | 
53 | 
54 | % -------------------------------------------------------------------------
55 | %                                                        Accumulate results
56 | % -------------------------------------------------------------------------
57 | img_invhog = imread('data/results/hog/img/lInf-recon.png');
58 | img_invhog2 = imread('data/results/hog2/img/lInf-recon.png');
59 | img_invhog3 = imread('data/results/hog3/img/lInf-recon.png');
60 | 
61 | img_invdsift = imread('data/results/dsift/img/lInf-recon.png');
62 | img_invhogb = imread('data/results/hogb/img/lInf-recon.png');
63 | img_hoggle = imread('data/results/hoggle/img/lInf-recon.png');
64 | 
65 | figure;
66 | subplot(2, 2, 1);
67 | imshow(img_invhog);
68 | title('HOG Pre-Image [Our approach]');
69 | subplot(2, 2, 2);
70 | imshow(img_invdsift);
71 | title('DSIFT Pre-Image [Our approach]');
72 | subplot(2, 2, 3);
73 | imshow(img_invhogb);
74 | title('HOGb Pre-Image [Our approach]');
75 | subplot(2, 2, 4);
76 | imshow(img_hoggle);
77 | title('HOG pre-image using HOGle');
78 | drawnow; 
79 | 
80 | figure;
81 | subplot(1,3,1);
82 | imshow(img_invhog);
83 | subplot(1,3,2);
84 | imshow(img_invhog2);
85 | title('Figure 4. Effect of v^{\beta} regularization.');
86 | subplot(1,3,3);
87 | imshow(img_invhog3);
88 | 


--------------------------------------------------------------------------------
/experiments/experiment_shallow_quantitative.m:
--------------------------------------------------------------------------------
 1 | function experiment_shallow_quantitative()
 2 | % Run some CNN experiments
 3 | 
 4 | experiment_setup;
 5 | 
 6 | images = experiment_get_dataset('imnet');
 7 | %images = {images{1:10}};
 8 | 
 9 | % -------------------------------------------------------------------------
10 | %                                                         Setup experiments
11 | % -------------------------------------------------------------------------
12 | 
13 | exp = {} ;
14 | res_ver = 'results' ;
15 | opts.learningRate = 0.004 * [...
16 |   ones(1,200), ...
17 |   0.1 * ones(1,200), ...
18 |   0.01 * ones(1,200),...
19 |   0.001 * ones(1,200)];
20 | opts.objective = 'l2' ;
21 | opts.beta = 6 ;
22 | opts.lambdaTV = 1e2 ;
23 | opts.lambdaL2 = 8e-10 ;
24 | opts.numRepeats = 1;
25 | 
26 | opts1 = opts;
27 | opts1.lambdaTV = 1e0 ;
28 | opts1.TVbeta = 2;
29 | 
30 | opts2 = opts;
31 | opts2.lambdaTV = 1e1 ;
32 | opts2.TVbeta = 2;
33 | 
34 | opts3 = opts;
35 | opts3.lambdaTV = 1e2 ;
36 | opts3.TVbeta = 2;
37 | 
38 | % -------------------------------------------------------------------------
39 | %                                                           Run experiments
40 | % -------------------------------------------------------------------------
41 | 
42 | for i = 1:numel(images) 
43 |   exp{end+1} = experiment_init('hog', inf, images{i}, res_ver, 'hog', opts1);
44 |   
45 |   exp{end+1} = experiment_init('hog', inf, images{i}, res_ver, 'hoggle', opts1, 'useHoggle', true);
46 |   exp{end+1} = experiment_init('dsift', inf, images{i}, res_ver, 'dsift', opts1);
47 |   exp{end+1} = experiment_init('hogb', inf, images{i}, res_ver, 'hogb', opts1);
48 | end
49 | 
50 | experiment_run(exp) ;
51 | 
52 | 
53 | % -------------------------------------------------------------------------
54 | %                                                        Accumulate results
55 | % -------------------------------------------------------------------------
56 | 
57 |     
58 | [hog_error_mean, hog_error_std] = eval_recons('hog', images, res_ver)
59 | [hogb_error_mean, hogb_error_std] = eval_recons('hogb', images, res_ver)
60 | [hoggle_error_mean, hoggle_error_std] = eval_recons('hoggle', images, res_ver)
61 | [dsift_error_mean, dsift_error_std] = eval_recons('dsift', images, res_ver)
62 | 
63 | end
64 | 
65 | function [error_mean, error_std] = eval_recons(prefix, images, res_ver) 
66 |   size_hog = 0;
67 |   for i=1:numel(images)
68 |     [~, image_filename, ~] = fileparts(images{i});
69 |     res_filename = fullfile('data', res_ver, prefix, image_filename, 'lInf.mat');
70 |     res = load(res_filename);
71 |     size_hog = max(size_hog, numel(res.y));
72 |   end
73 | 
74 |   y_hog = zeros(size_hog, numel(images));
75 |   y0_hog = zeros(size_hog, numel(images));
76 | 
77 |   for i=1:numel(images)
78 |     [~, image_filename, ~] = fileparts(images{i});
79 |     res_filename = fullfile('data', res_ver, prefix, image_filename, 'lInf.mat');
80 |     res = load(res_filename);
81 |     
82 |     y_hog(1:numel(res.y),i) = res.y(:);
83 |     y0_hog(1:numel(res.y),i) = res.y0(:);
84 |   end
85 |   
86 |   [error_mean, error_std] = my_meanstd(y_hog, y0_hog);
87 | end
88 | 
89 | function [error_mean, error_std] = my_meanstd(y, y0)
90 |   temp = pdist2(y0', y0');
91 |   normalization_factor = mean(temp(:).^2);
92 | 
93 |   temp = sum((y - y0).^2) / normalization_factor;
94 |   error_mean = mean(temp);
95 |   error_std = std(temp);
96 | end
97 | 


--------------------------------------------------------------------------------
/experiments/networks/dsift_net.m:
--------------------------------------------------------------------------------
  1 | function net = dsift_net(binSize, varargin)
  2 | % Define a CNN equivalent to dense SIFT
  3 | 
  4 | opts.numOrientations = 4 ;
  5 | opts = vl_argparse(opts, varargin) ;
  6 | opts.binSize = binSize ;
  7 | 
  8 | % Spatial derivatives along NO directions
  9 | NO = opts.numOrientations ;
 10 | dx = [0 0 0 ; -1 0 1 ; 0  0 0]/2 ;
 11 | dy = dx' ;
 12 | for i=0:2*NO-1
 13 |   t = (2*pi)/(2*NO)*i ;
 14 |   spatialDer{i+1} = cos(t)*dx+sin(t)*dy;
 15 | end
 16 | spatialDer = single(cat(4, spatialDer{:})) ;
 17 | 
 18 | % Spatial bilienar binning
 19 | a = 1 - abs(((1:2*opts.binSize) - (2*opts.binSize+1)/2)/opts.binSize);
 20 | bilinearFilter = repmat(single(a'*a), [1, 1, 1, 2*NO]) ;
 21 | 
 22 | % Stacking of SIFT cells into 4x4 blocks.
 23 | 
 24 | sigma = 1.5 ;
 25 | mask = {} ;
 26 | t = 0 ;
 27 | for i=1:4
 28 |   for j=1:4
 29 |     for o=1:2*NO
 30 |       t=t+1 ;
 31 |       mask{t} = zeros(4,4,2*NO) ;
 32 |       mask{t}(i,j,o) = exp(-0.5*((i-2.5).^2 + (j-2.5).^2) / sigma^2) ;
 33 |     end
 34 |   end
 35 | end
 36 | mask = single(cat(4, mask{:})) ;
 37 | 
 38 | net.layers = {} ;
 39 | net.layers{end+1} = struct('type','conv', ...
 40 |   'filters', spatialDer, ...
 41 |   'biases', zeros(size(spatialDer,4),1,'single'), ...
 42 |   'stride', 1, 'pad', 0) ;
 43 | if 0
 44 |   net.layers{end+1} = struct('type','noffset', 'param', [.5*cos(2*pi/8), .5]) ;
 45 |   net.layers{end+1} = struct('type','relu') ;
 46 | else
 47 |   net.layers{end+1} = get_hog_binning_layer(opts) ;
 48 | end
 49 | net.layers{end+1} = struct('type','conv', ...
 50 |   'filters', bilinearFilter, ...
 51 |   'biases', [], ...
 52 |   'stride', binSize, 'pad', 0) ;
 53 | net.layers{end+1} = struct('type','conv', ...
 54 |   'filters', mask, ...
 55 |   'biases', [], ...
 56 |   'stride', 3, 'pad', 0) ;
 57 | net.layers{end+1} = struct('type', 'normalize', 'param', [128*2, 0.000001, 1, .5]) ;
 58 | net.layers{end+1} = get_hog_clamp_layer(opts) ;
 59 | 
 60 | % -------------------------------------------------------------------------
 61 | function l = get_hog_clamp_layer(opts)
 62 | % -------------------------------------------------------------------------
 63 | l.type = 'custom' ;
 64 | l.forward = @hog_clamp_forward ;
 65 | l.backward = @hog_clamp_backward ;
 66 | 
 67 | function res_ = hog_clamp_forward(ly,res,res_)
 68 | res_.x = min(res.x,single(0.2)) ;
 69 | %res_.x = res.x ;
 70 | 
 71 | function res = hog_clamp_backward(ly,res,res_)
 72 | res.dzdx = res_.dzdx .* (res.x <= 0.2) ;
 73 | 
 74 | % -------------------------------------------------------------------------
 75 | function l = get_hog_binning_layer(opts)
 76 | % -------------------------------------------------------------------------
 77 | l.type = 'custom' ;
 78 | l.NO = opts.numOrientations ;
 79 | l.forward = @hog_binning_forward ;
 80 | l.backward = @hog_binning_backward ;
 81 | 
 82 | function res_ = hog_binning_forward(ly,res,res_)
 83 | x = res.x ;
 84 | n2 = sum(x.^2,3)/ly.NO ;
 85 | n = sqrt(n2) ;
 86 | cs = bsxfun(@rdivide, x, max(n, 1e-10)) ;
 87 | cs = max(min(cs,1),-1) ;
 88 | delta = 1 - (2*ly.NO)/(2*pi)*acos(cs) ;
 89 | w = max(0, delta) ;
 90 | res_.x = bsxfun(@times, n, w) ;
 91 | 
 92 | function res = hog_binning_backward(ly,res,res_)
 93 | % forward computations
 94 | x = res.x ;
 95 | n2 = sum(x.^2,3)/ly.NO ;
 96 | n = sqrt(n2) ;
 97 | cs = bsxfun(@rdivide, x, max(n, 1e-10)) ;
 98 | cs = max(min(cs,1),-1) ;
 99 | delta = 1 - (2*ly.NO)/(2*pi)*acos(cs) ;
100 | w = max(0, delta) ;
101 | 
102 | dn = bsxfun(@rdivide, x/ly.NO, max(n, 1e-10)) ;
103 | dwdcs = ((2*ly.NO)/(2*pi) ./ sqrt(1.0000001 - cs.^2)) .* (delta > 0) .* res_.dzdx ;
104 | dw = ...
105 |   + dwdcs .* repmat(1./n, [1 1 2*ly.NO]) ...
106 |   - bsxfun(@times, sum(bsxfun(@rdivide, dwdcs.*x/ly.NO, n.*n2),3), x) ;
107 | res.dzdx = bsxfun(@times, sum(res_.dzdx .* w, 3), dn) +  bsxfun(@times, n, dw) ;
108 | 


--------------------------------------------------------------------------------
/experiments/networks/hog_net.m:
--------------------------------------------------------------------------------
  1 | function net = hog_net(binSize, varargin)
  2 | % HOG_NET   CNN-HOG
  3 | %   NET = HOG_NET(BINSIZE) returns a CNN equivalent to the HOG feature
  4 | %   extractor for the specified bin size.
  5 | %
  6 | %   The implementation is numerically identical to VL_HOG(), which in turns
  7 | %   is nearly exactly the same as UoCTTI HOG implementation (DPM V5).
  8 | 
  9 | opts.bilinearOrientations = false ;
 10 | opts.numOrientations = 9 ;
 11 | opts = vl_argparse(opts, varargin) ;
 12 | opts.binSize = binSize ;
 13 | 
 14 | % Spatial derivatives along O directions
 15 | NO = opts.numOrientations ;
 16 | dx = [0 0 0 ; -1 0 1 ; 0  0 0]/2 ;
 17 | dy = dx' ;
 18 | for i=0:2*NO-1
 19 |   t = (2*pi)/(2*NO)*i ;
 20 |   spatialDer{i+1} = cos(t)*dx+sin(t)*dy;
 21 | end
 22 | spatialDer = single(cat(4, spatialDer{:})) ;
 23 | 
 24 | % Spatial bilienar binning
 25 | a = 1 - abs(((1:2*opts.binSize) - (2*opts.binSize+1)/2)/opts.binSize);
 26 | bilinearFilter = repmat(single(a'*a), [1, 1, 1, 2*NO]) ;
 27 | 
 28 | % Stacking of HOG cells into 2x2 blocks.
 29 | % A detail ritical for what comes later: cells are stacked in column-major
 30 | % order.
 31 | mask = {} ;
 32 | t = 0 ;
 33 | for j=1:2
 34 |   for i=1:2
 35 |     for o=1:2*NO
 36 |       t=t+1 ;
 37 |       mask{t} = zeros(2,2,2*NO) ;
 38 |       mask{t}(i,j,o) = 1 ;
 39 |     end
 40 |     for o=2*NO+(1:NO)
 41 |       t=t+1 ;
 42 |       mask{t} = zeros(2,2,2*NO) ;
 43 |       mask{t}(i,j,o-2*NO) = 1 ;
 44 |       mask{t}(i,j,o-NO) = 1 ;
 45 |     end
 46 |   end
 47 | end
 48 | mask = single(cat(4, mask{:})) ;
 49 | 
 50 | % Break down blocks to cells. This uses once more 2x2 filters.
 51 | % Note how these filters are related to the one that build the blocks
 52 | % above. The application of such a decompositon filter involves four
 53 | % blocks which share a central cell. This cell is the bottom-right
 54 | % one with respect to the first upper-left block. This affects which
 55 | % dimensions of the block must be extracted by the decomposition filter.
 56 | t = 0 ;
 57 | for o=1:3*NO
 58 |   t=t+1 ;
 59 |   dec{t} = zeros(2,2,3*NO*4) ;
 60 |   for i=1:2
 61 |     for j=1:2
 62 |       u=3-i ;
 63 |       v=3-j ;
 64 |       dec{t}(i,j,(u-1+(v-1)*2)*3*NO+o) = 0.5 ;
 65 |     end
 66 |   end
 67 | end
 68 | dec = single(cat(4, dec{:})) ;
 69 | 
 70 | % Network
 71 | net.layers = {} ;
 72 | net.layers{end+1} = struct('type','conv', ...
 73 |   'filters', spatialDer, ...
 74 |   'biases', zeros(size(spatialDer,4),1,'single'), ...
 75 |   'stride', 1, 'pad', 1) ;
 76 | if 0
 77 |   net.layers{end+1} = struct('type','noffset', 'param', [1/3*cos((2*pi)/(2*NO)), .5]) ;
 78 |   net.layers{end+1} = struct('type','relu') ;
 79 | else
 80 |   net.layers{end+1} = get_hog_binning_layer(opts) ;
 81 | end
 82 | net.layers{end+1} = struct('type','conv', ...
 83 |   'filters', bilinearFilter, ...
 84 |   'biases', [], ...
 85 |   'stride', binSize, 'pad', binSize/2) ;
 86 | net.layers{end+1} = struct('type','conv', ...
 87 |   'filters', mask, ...
 88 |   'biases', [], ...
 89 |   'stride', 1, 'pad', 0) ;
 90 | net.layers{end+1} = get_hog_norm_layer(opts) ;
 91 | net.layers{end+1} = get_hog_clamp_layer(opts) ;
 92 | net.layers{end+1} = struct('type','conv', ...
 93 |   'filters', dec, ...
 94 |   'biases', [], ...
 95 |   'stride', 1, 'pad', 1) ;
 96 | 
 97 | % -------------------------------------------------------------------------
 98 | %                                                             Sanity checks
 99 | % -------------------------------------------------------------------------
100 | if 0
101 |   rng(0) ;
102 |   % the norm of the gradient derivatives is sqrt(NO) times the norm of the
103 |   % gradient
104 |   for t = 1:100
105 |     x = randn(3) ;
106 |     g = reshape(spatialDer, [], 2*NO)'*x(:) ;
107 |     gx = (x(2,3) - x(2,1))/2 ;
108 |     gy = (x(3,2) - x(1,2))/2 ;
109 |     a(t) = norm([gx gy]);
110 |     b(t) = norm([g]) ; %twice the norm of a
111 |   end
112 |   vl_testsim(mean(b./a), sqrt(NO)) ;
113 | 
114 |   % test HOG clamping layer
115 |   res.x = round(100*randn(20,20,5,3,'single'))/100 ;
116 |   res_ = hog_clamp_forward([],res,struct('x',[])) ;
117 |   res_.dzdx = randn(size(res_.x),'single') ;
118 |   res = hog_clamp_backward([],res,res_) ;
119 | 
120 |   vl_testder(@(x) subsref(...
121 |     hog_clamp_forward([],struct('x',x)),...
122 |     struct('type','.','subs','x')), ...
123 |     res.x, ...
124 |     res_.dzdx, ...
125 |     res.dzdx, ...
126 |     1e-4) ;
127 | 
128 |   % test HOG normalization layer
129 |   ly = get_hog_norm_layer(opts);
130 |   res.x = randn(5,5,3*NO*4,2,'double') ;
131 |   res_ = hog_norm_forward(ly,res,struct('x',[])) ;
132 |   res_.dzdx = randn(size(res_.x),'double') ;
133 |   res = hog_norm_backward(ly,res,res_) ;
134 | 
135 |   vl_testder(@(x) subsref(...
136 |     hog_norm_forward(ly,struct('x',x)),...
137 |     struct('type','.','subs','x')), ...
138 |     res.x, ...
139 |     res_.dzdx, ...
140 |     res.dzdx, ...
141 |     1e-5) ;
142 | 
143 |   % test HOG binning layer
144 |   % mini network: derivatives and normalization
145 |   % we do so as the normalization block works properly only on the
146 |   % output of the derivative layer (due to special cases)
147 |   ly = get_hog_binning_layer(opts) ;
148 |   res.x = randn(20,20,1,3,'single') ;
149 |   res_.x = double(vl_nnconv(res.x, spatialDer, [])) ;
150 |   res__ = hog_binning_forward(ly,res_,struct('x',[])) ;
151 |   res__.dzdx = randn(size(res__.x)) ;
152 |   res_ = hog_binning_backward(ly,res_,res__) ;
153 |   res.dzdx = vl_nnconv(res.x, spatialDer, [], single(res_.dzdx)) ;
154 | 
155 |   vl_testder(@(x) vl_nnconv(x, spatialDer, []), ...
156 |     res.x, ...
157 |     res_.dzdx, ...
158 |     res.dzdx, ...
159 |     1e-2) ;
160 | 
161 |   vl_testder(@(x) subsref(...
162 |     hog_binning_forward(ly,struct('x',double(vl_nnconv(x, spatialDer, []))),res__),...
163 |     struct('type','.','subs','x')), ...
164 |     res.x, ...
165 |     res__.dzdx, ...
166 |     res.dzdx, ...
167 |     1e-4, 0.5) ;
168 | end
169 | 
170 | if 0
171 |   im = imread('peppers.png') ;
172 |   im = im(1:100,1:100,:) ;
173 |   im = rgb2gray(im2single(im)) ;
174 |   res = vl_simplenn(net, im) ;
175 |   res_ = vl_simplenn(net, im*255) ;
176 | 
177 |   L = sqrt(sum(res(2).x.^2,3))/2 ;
178 |   P = bsxfun(@times, 1./max(L,1e-10), res(2).x) ;
179 |   th = real(acos(P)) ;
180 | 
181 |   figure(100) ; clf ;
182 |   subplot(1,2,1) ; vl_imarraysc(res(4).x) ;
183 |   subplot(1,2,2) ; vl_imarraysc(bsxfun(@times, L, max(0,1-8/(2*pi)*th)));
184 | 
185 |   res(6).x(1:5,1:5,1) ./ max(res_(6).x(1:5,1:5,1),1e-10)
186 | end
187 | 
188 | % -------------------------------------------------------------------------
189 | function l = get_hog_norm_layer(opts)
190 | % -------------------------------------------------------------------------
191 | NO = opts.numOrientations ;
192 | sel = [] ;
193 | for i=1:4
194 |   sel = [sel, 2*NO + (1:NO) + 3*NO*(i-1)] ;
195 | end
196 | l.type = 'custom' ;
197 | l.sel = sel ;
198 | l.forward = @hog_norm_forward ;
199 | l.backward = @hog_norm_backward ;
200 | 
201 | function res_ = hog_norm_forward(ly,res,res_)
202 | n2 = sum(res.x(:,:,ly.sel,:).^2,3) ;
203 | n = sqrt(n2) ;
204 | n = max(n, 1e-6) ;
205 | res_.x = bsxfun(@rdivide, res.x, n) ;
206 | 
207 | function res = hog_norm_backward(ly,res,res_)
208 | n2 = sum(res.x(:,:,ly.sel,:).^2,3) ;
209 | n = sqrt(n2) ;
210 | n = max(n, 1e-6) ;
211 | tmp = sum(res_.dzdx.*res.x,3) ./ (n.*n2) ;
212 | res.dzdx = bsxfun(@rdivide, res_.dzdx, n) ;
213 | res.dzdx(:,:,ly.sel,:) = res.dzdx(:,:,ly.sel,:) - bsxfun(@times, tmp, res.x(:,:,ly.sel,:)) ;
214 | 
215 | % -------------------------------------------------------------------------
216 | function l = get_hog_clamp_layer(opts)
217 | % -------------------------------------------------------------------------
218 | l.type = 'custom' ;
219 | l.forward = @hog_clamp_forward ;
220 | l.backward = @hog_clamp_backward ;
221 | 
222 | function res_ = hog_clamp_forward(ly,res,res_)
223 | res_.x = min(res.x,single(0.2)) ;
224 | %res_.x = res.x ;
225 | 
226 | function res = hog_clamp_backward(ly,res,res_)
227 | res.dzdx = res_.dzdx .* (res.x <= 0.2) ;
228 | 
229 | % -------------------------------------------------------------------------
230 | function l = get_hog_binning_layer(opts)
231 | % -------------------------------------------------------------------------
232 | l.type = 'custom' ;
233 | l.NO = opts.numOrientations ;
234 | l.bilinear = opts.bilinearOrientations ;
235 | l.forward = @hog_binning_forward ;
236 | l.backward = @hog_binning_backward ;
237 | 
238 | function res_ = hog_binning_forward(ly,res,res_)
239 | x = res.x ;
240 | n2 = sum(x.^2,3)/ly.NO ;
241 | n = sqrt(n2) ;
242 | if ly.bilinear
243 |   cs = bsxfun(@rdivide, x, max(n, 1e-10)) ;
244 |   cs = max(min(cs,1),-1) ;
245 |   delta = 1 - (2*ly.NO)/(2*pi)*acos(cs) ;
246 |   w = max(0, delta) ;
247 | else
248 |   w = 0*x ;
249 |   [~,best] = max(x, [], 3) ;
250 |   wx = size(w,2);
251 |   wy = size(w,1);
252 |   wz = size(w,3);
253 |   wn = size(w,4);
254 |   w(kron(ones(1,wn), 1:wx*wy) + ...
255 |     +(best(:)'-1)*(wx*wy)+...
256 |     kron((0:wn-1)*(wx*wy*wz), ones(1,wx*wy))) = 1 ;
257 | end
258 | res_.x = bsxfun(@times, n, w) ;
259 | 
260 | function res = hog_binning_backward(ly,res,res_)
261 | % forward computations
262 | x = res.x ;
263 | n2 = sum(x.^2,3)/ly.NO ;
264 | n = sqrt(n2) ;
265 | if ly.bilinear
266 |   cs = bsxfun(@rdivide, x, max(n, 1e-10)) ;
267 |   cs = max(min(cs,1),-1) ;
268 |   delta = 1 - (2*ly.NO)/(2*pi)*acos(cs) ;
269 |   w = max(0, delta) ;
270 | else
271 |   w = 0*x ;
272 |   [~,best] = max(x, [], 3) ;
273 |   wx = size(w,2);
274 |   wy = size(w,1);
275 |   wz = size(w,3);
276 |   wn = size(w,4);
277 |   w(kron(ones(1,wn), 1:wx*wy) + ...
278 |     +(best(:)'-1)*(wx*wy)+...
279 |     kron((0:wn-1)*(wx*wy*wz), ones(1,wx*wy))) = 1 ;
280 | end
281 | 
282 | dn = bsxfun(@rdivide, x/ly.NO, max(n, 1e-10)) ;
283 | if ly.bilinear
284 |   dwdcs = ((2*ly.NO)/(2*pi) ./ sqrt(1.0000001 - cs.^2)) .* (delta > 0) .* res_.dzdx ;
285 |   dw = ...
286 |     + dwdcs .* repmat(1./n, [1 1 2*ly.NO]) ...
287 |     - bsxfun(@times, sum(bsxfun(@rdivide, dwdcs.*x/ly.NO, n.*n2),3), x) ;
288 |   res.dzdx = bsxfun(@times, sum(res_.dzdx .* w, 3), dn) +  bsxfun(@times, n, dw) ;
289 | else
290 |   res.dzdx = bsxfun(@times, sum(res_.dzdx .* w, 3), dn) ;
291 | end
292 | 


--------------------------------------------------------------------------------
/experiments/x0_sigma.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aravindhm/deep-goggle/cc667f02dd079f060542594a3592d6b7feb1137c/experiments/x0_sigma.mat


--------------------------------------------------------------------------------
/helpers/cnn_denormalize.m:
--------------------------------------------------------------------------------
 1 | function im_ = cnn_denormalize(normalization, im)
 2 | % im_ = cnn_denormalize(net.normalization, im)  - denormalize an input
 3 | %    normalized image using the normalization info in the 'normalization'
 4 | %    structure
 5 | 
 6 | im_ = bsxfun(@plus, im, normalization.averageImage) ;
 7 | 
 8 | end
 9 | 
10 | 


--------------------------------------------------------------------------------
/helpers/cnn_normalize.m:
--------------------------------------------------------------------------------
 1 | function im_ = cnn_normalize(normalization, im, colour)
 2 | % im_ = cnn_normalize(normalization, im_, colour) - Normalize an image
 3 | 
 4 | if ~colour && size(im, 3)==3, im=rgb2gray(im); end
 5 | im_ = single(im) ; % note: 255 range
 6 | if colour && size(im_,3)==1, im_=repmat(im_,[1 1 3]) ; end
 7 | im_ = imresize(im_, normalization.imageSize(1:2)) ;
 8 | im_ = im_ - normalization.averageImage ;
 9 | end
10 | 
11 | 


--------------------------------------------------------------------------------
/helpers/compute_features.m:
--------------------------------------------------------------------------------
 1 | function feats = compute_features(net, img, opts)
 2 | % feats = compute_features(net, img) - compute features from a network
 3 | %
 4 | %  Inputs: net - the neural network (same as for vl_simplenn)
 5 | %          img - the input image (H x W x 3 - RGB image)
 6 | %
 7 | %  Output: feats - the reference given as argument to invert_nn.m
 8 | %
 9 | % Author: Aravindh Mahendran
10 | %      New College, University of Oxford
11 | 
12 | % normalize the input image
13 | x0 = opts.normalize(img);
14 | 
15 | % Convert the image into a 4D matrix as required by vl_simplenn
16 | x0 = repmat(x0, [1, 1, 1, 1]);
17 | 
18 | % Run feedforward for network
19 | res = vl_simplenn(net, x0);
20 | feats = res(end).x;
21 | 
22 | end
23 | 


--------------------------------------------------------------------------------
/helpers/get_cnn_denormalize.m:
--------------------------------------------------------------------------------
1 | function fn = get_cnn_denormalize(normalization)
2 | % fn = get_cnn_denormalize(net.normalization) - returns a function handle
3 | %   that can denormalize normalized network inputs
4 | %
5 | % Return value: fn - a function handle called as x_denormalized=fn(x_normalized)
6 | fn = @(x) cnn_denormalize(normalization, x) ;
7 | 
8 | end
9 | 


--------------------------------------------------------------------------------
/helpers/get_cnn_normalize.m:
--------------------------------------------------------------------------------
 1 | function fn = get_cnn_normalize(normalization, colour)
 2 | % fn = get_cnn_normalize(net.normalization, colour) - returns a normalize func
 3 | %     Inputs: normalization - The normalization information of a network's input
 4 | %             color - (option) whether the image is colour or not.
 5 | %     Outputs - fn - the function that can be used as fn(img) to normalize img
 6 | 
 7 | if(nargin < 2)
 8 |   colour = true;
 9 | end
10 | fn = @(x) cnn_normalize(normalization, x, colour) ;
11 | end
12 | 
13 | 


--------------------------------------------------------------------------------