├── .gitmodules ├── startup.m ├── feedForwardSF.m ├── README ├── runExample.m ├── LICENSE ├── sparseFiltering.m └── displayData.m /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "common"] 2 | path = common 3 | url = git://github.com/jngiam/common.git 4 | -------------------------------------------------------------------------------- /startup.m: -------------------------------------------------------------------------------- 1 | %% Add Paths 2 | addpath ([pwd filesep 'common']); 3 | addpath ([pwd filesep 'common/minFunc']); 4 | -------------------------------------------------------------------------------- /feedForwardSF.m: -------------------------------------------------------------------------------- 1 | function R = feedForwardSF(W, X) 2 | % Feed Forward 3 | F = W*X; 4 | Fs = sqrt(F.^2 + 1e-8); 5 | [NFs] = l2row(Fs); 6 | [Fhat] = l2row(NFs'); 7 | R = Fhat'; 8 | end 9 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Sparse Filtering 2 | 3 | * Trains using minFunc 4 | * Dataset available at http://cs.stanford.edu/~jngiam/data/patches.mat 5 | 6 | Cloning (read-only) 7 | =========== 8 | git clone --recursive git://github.com/jngiam/sparseFiltering.git 9 | 10 | (use recursive due to the presence of submodules) 11 | 12 | Quick Start 13 | =========== 14 | 15 | Run "runExample" in Matlab 16 | 17 | 18 | Notes 19 | ===== 20 | 21 | * Trains a 2 layer network! 22 | 23 | -------------------------------------------------------------------------------- /runExample.m: -------------------------------------------------------------------------------- 1 | %% Startup (addpaths) 2 | startup 3 | 4 | %% Clear 5 | clear ; close all ; clc ; 6 | 7 | %% Load Data 8 | % You can obtain patches.mat from 9 | % http://cs.stanford.edu/~jngiam/data/patches.mat 10 | 11 | fprintf('Loading Data\n'); 12 | 13 | % Loads a variable data (size 256x50000) 14 | load patches.mat 15 | 16 | %% Remove DC 17 | data = bsxfun(@minus, data, mean(data)); 18 | 19 | %% Train Layer 1 20 | L1_size = 256; % Increase this for more features 21 | L1 = sparseFiltering(L1_size, data); 22 | 23 | % Show Layer 1 Bases 24 | displayData(L1); 25 | pause; 26 | 27 | %% Feed-forward Layer 1 28 | data1 = feedForwardSF(L1, data); 29 | data1 = bsxfun(@minus, data1, mean(data1)); 30 | 31 | %% Train Layer 2 32 | L2_size = 256; 33 | L2 = sparseFiltering(L2_size, data1); 34 | 35 | %% Visualize Layer 2 36 | figure; 37 | 38 | % Number of L2 units to visualize 39 | num_viz = 10; 40 | 41 | % Visualize different units 42 | offset = 1; 43 | 44 | % Plot the units 45 | for i = 1:num_viz 46 | j = offset+i; 47 | 48 | % Find the sign of the unit with the maximum absolute values 49 | [a, b] = max(abs(L2(j, :))); 50 | sgn = sign(L2(j, b)); 51 | 52 | % Sort and plot units in those direction 53 | [a, b] = sort(sgn*(L2(j, :)), 'descend'); 54 | 55 | % Plot 56 | subplot(1, num_viz, i, 'align'); 57 | displayData(L1(b(1:10), :), [] , 1); 58 | end 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, Jiquan Ngiam 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | 26 | -------------------------------------------------------------------------------- /sparseFiltering.m: -------------------------------------------------------------------------------- 1 | function [optW] = sparseFiltering(N, X) 2 | 3 | % N = # features, X = input data (examples in column) 4 | optW = randn(N, size(X, 1)); 5 | optW = minFunc(@SparseFilteringObj, optW(:), ... 6 | struct('MaxIter', 200, 'Corr', 20), X, N); 7 | optW = reshape(optW, [N, size(X, 1)]); 8 | 9 | end 10 | 11 | function [Obj, DeltaW] = SparseFilteringObj (W, X, N) 12 | % Reshape W into matrix form 13 | W = reshape(W, [N, size(X,1)]); 14 | 15 | % Feed Forward 16 | F = W*X; 17 | Fs = sqrt(F.^2 + 1e-8); 18 | [NFs, L2Fs] = l2row(Fs); 19 | [Fhat, L2Fn] = l2row(NFs'); 20 | 21 | % Compute Objective Function 22 | Obj = sum(sum(Fhat, 2), 1); 23 | 24 | % Backprop through each feedforward step 25 | DeltaW = l2rowg(NFs', Fhat, L2Fn, ones(size(Fhat))); 26 | DeltaW = l2rowg(Fs, NFs, L2Fs, DeltaW'); 27 | DeltaW = (DeltaW .* (F./Fs)) * X'; 28 | DeltaW = DeltaW(:); 29 | end 30 | 31 | % 32 | % The following functions are now in common/ 33 | % 34 | % function [Y,N] = l2row(X) % L2 Normalize X by rows 35 | % % We also use this to normalize by column with l2row(X') 36 | % N = sqrt(sum(X.^2,2) + 1e-8); 37 | % Y = bsxfun(@rdivide,X,N); 38 | % end 39 | % 40 | % function [G] = l2grad(X,Y,N,D) % Backpropagate through Normalization 41 | % G = bsxfun(@rdivide, D, N) - bsxfun(@times, Y, sum(D.*X, 2)./(N.^2)); 42 | % end 43 | % 44 | -------------------------------------------------------------------------------- /displayData.m: -------------------------------------------------------------------------------- 1 | function [h, display_array] = displayData(X, example_width, display_cols) 2 | %DISPLAYDATA Display 2D data in a nice grid 3 | % [h, display_array] = DISPLAYDATA(X, example_width) displays 2D data 4 | % stored in X in a nice grid. It returns the figure handle h and the 5 | % displayed array if requested. 6 | 7 | % Set example_width automatically if not passed in 8 | if ~exist('example_width', 'var') || isempty(example_width) 9 | example_width = round(sqrt(size(X, 2))); 10 | end 11 | 12 | % Gray Image 13 | colormap(gray); 14 | 15 | % Compute rows, cols 16 | [m n] = size(X); 17 | example_height = (n / example_width); 18 | 19 | % Compute number of items to display 20 | if ~exist('display_cols', 'var') 21 | display_cols = floor(sqrt(m)); 22 | end 23 | display_rows = ceil(m / display_cols); 24 | 25 | % Between images padding 26 | pad = 1; 27 | 28 | % Setup blank display 29 | display_array = - ones(pad + display_rows * (example_height + pad), ... 30 | pad + display_cols * (example_width + pad)); 31 | 32 | % Copy each example into a patch on the display array 33 | curr_ex = 1; 34 | for j = 1:display_rows 35 | for i = 1:display_cols 36 | if curr_ex > m, 37 | break; 38 | end 39 | % Copy the patch 40 | 41 | % Get the max value of the patch 42 | max_val = max(abs(X(curr_ex, :))); 43 | display_array(pad + (j - 1) * (example_height + pad) + (1:example_height), ... 44 | pad + (i - 1) * (example_width + pad) + (1:example_width)) = ... 45 | reshape(X(curr_ex, :), example_height, example_width) / max_val; 46 | curr_ex = curr_ex + 1; 47 | end 48 | if curr_ex > m, 49 | break; 50 | end 51 | end 52 | 53 | % Display Image 54 | h = imagesc(display_array, [-1 1]); 55 | 56 | % Do not show axis 57 | axis image off 58 | 59 | drawnow; 60 | 61 | end 62 | --------------------------------------------------------------------------------