├── bhtsne_linux ├── bhtsne_maci ├── images └── tsne_map.png ├── manifold-scm-0.rockspec ├── demos ├── demo_swissroll.lua └── demo_tsne.lua ├── README.md ├── laplacian_eigenmaps.lua ├── lle.lua ├── init.lua └── tsne.lua /bhtsne_linux: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clementfarabet/manifold/HEAD/bhtsne_linux -------------------------------------------------------------------------------- /bhtsne_maci: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clementfarabet/manifold/HEAD/bhtsne_maci -------------------------------------------------------------------------------- /images/tsne_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clementfarabet/manifold/HEAD/images/tsne_map.png -------------------------------------------------------------------------------- /manifold-scm-0.rockspec: -------------------------------------------------------------------------------- 1 | package = "manifold" 2 | version = "scm-0" 3 | 4 | source = { 5 | url = "git://github.com/clementfarabet/manifold" 6 | } 7 | 8 | description = { 9 | summary = "A package to manipulate manifolds", 10 | detailed = [[ 11 | A package to manipulate manifolds. 12 | ]], 13 | homepage = "https://github.com/clementfarabet/manifold", 14 | license = "MIT" 15 | } 16 | 17 | dependencies = { 18 | "torch >= 7.0", 19 | "unsup", 20 | "mnist", 21 | } 22 | 23 | build = { 24 | type = "builtin", 25 | modules = { 26 | ['manifold.init'] = 'init.lua', 27 | ['manifold.tsne'] = 'tsne.lua', 28 | ['manifold.lle'] = 'lle.lua', 29 | ['manifold.laplacian_eigenmaps'] = 'laplacian_eigenmaps.lua', 30 | }, 31 | install = { 32 | bin = { 33 | 'bhtsne_maci', 34 | 'bhtsne_linux', 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /demos/demo_swissroll.lua: -------------------------------------------------------------------------------- 1 | -- This is the Swissroll test... 2 | local mani = require 'manifold' 3 | local g = require 'gnuplot' 4 | 5 | local function newFig(t) g.figure(); g.grid(true); g.title(t) end 6 | local N = 2000 7 | local K = 12 8 | local sigma = 7 9 | local d = 2 10 | 11 | local tt = torch.mul(torch.sqrt(torch.rand(1, N)), 4 * math.pi) 12 | local X = torch.DoubleTensor(3, N) 13 | X[1] = torch.cmul(torch.add(tt, 0.1), torch.cos(tt)) 14 | X[2] = torch.cmul(torch.add(tt, 0.1), torch.sin(tt)) 15 | X[3] = torch.mul(torch.rand(1, N), 8 * math.pi) 16 | 17 | newFig('Original 3D') 18 | g.scatter3(X[1], X[2], X[3]) 19 | 20 | X = X:t() 21 | 22 | print('random embedding...') 23 | Y = mani.embedding.random(X, { 24 | dim = 2, 25 | }) 26 | 27 | newFig('Random') 28 | g.plot(Y, '+') 29 | 30 | print('LLE embedding...') 31 | Y = mani.embedding.lle(X, { 32 | dim = d, 33 | neighbors = K, 34 | tol = 1e-3 35 | }) 36 | 37 | newFig('LLE') 38 | g.plot(Y, '+') 39 | 40 | print('Laplacian Eigenmaps embedding...') 41 | local K = 60 42 | Y = mani.embedding.laplacian_eigenmaps(X, { 43 | dim = 2,neighbors = K,sigma = sigma,normalized = false 44 | }) 45 | 46 | newFig('Laplacian Eigenmaps') 47 | g.plot(Y, '+') 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Manifold 2 | ======== 3 | 4 | A package to manipulate manifolds, for Torch7. 5 | 6 | Install 7 | ------- 8 | 9 | ```sh 10 | luarocks install manifold 11 | ``` 12 | 13 | Dependencies 14 | ------------ 15 | 16 | In order to be able to run the binaries, you need to install the package `libatlas3-base`. 17 | On a Ubuntu machine you can execute the following commands. 18 | 19 | ``` 20 | sudo apt-get update 21 | sudo apt-get install libatlas3-base 22 | ``` 23 | 24 | Use 25 | --- 26 | 27 | ```lua 28 | -- package: 29 | m = require 'manifold' 30 | 31 | -- a dataset: 32 | t = torch.randn(100,10) -- 100 samples, 10-dim each 33 | 34 | -- basic functions: 35 | ns = m.neighbors(t) -- return the matrix of neighbors for all samples (sorted) 36 | ds = m.distances(t) -- return the matrix of distances (L2) 37 | ts = m.removeDuplicates(t) -- remove duplicates from dataset 38 | 39 | -- embeddings: 40 | p = m.embedding.random(t, {dim=2}) -- embed samples into a 2D plane, using random projections 41 | p = m.embedding.lle(t, {dim=2, neighbors=3}) -- embed samples into a 2D plane, using 3 neighbor (LLE) 42 | p = m.embedding.tsne(t, {dim=2, perplexity=30}) -- embed samples into a 2D plane, using tSNE 43 | ``` 44 | 45 | Demos 46 | ----- 47 | 48 | To run the demos, simply type the following commands. 49 | 50 | ```sh 51 | cd demos 52 | qlua demo_swissroll.lua 53 | qlua demo_tsne.lua 54 | ``` 55 | 56 | Below is an example of a t-SNE map produced on 5,000 MNIST digits by the demos/demo_tsne.lua demo. 57 | 58 | ![t-SNE map of 5,000 MNIST digits](images/tsne_map.png) 59 | -------------------------------------------------------------------------------- /laplacian_eigenmaps.lua: -------------------------------------------------------------------------------- 1 | -- Implementation of Laplacian Eigenmaps: 2 | local laplacian_eigenmaps = function(vectors,opts) 3 | 4 | -- args: 5 | opts = opts or {} 6 | local d = opts.dim or 2 7 | local K = opts.neighbors or 5 8 | local sigma = opts.sigma or 1 9 | local normalized = opts.normalized or false 10 | local X = vectors 11 | 12 | -- dependencies: 13 | local pkg = require 'manifold' 14 | 15 | -- dims: 16 | local N,D = X:size(1),X:size(2) 17 | 18 | -- get nearest neighbors: 19 | local neighbors,dists = pkg.neighbors(X) 20 | assert(torch.dist(neighbors[{{},1}]:float(), torch.range(1,N):float()) == 0, 'Laplacian Eigenmaps cannot deal with duplicates') 21 | local neighborhood = neighbors[{ {},{1,K+1} }] 22 | local kernel = dists[{ {},{1,K+1} }] 23 | 24 | -- compute Gaussian kernel: 25 | kernel:cmul(kernel) 26 | kernel:div(-2 * sigma * sigma) 27 | kernel:exp() 28 | local L = torch.zeros(N, N) 29 | for n = 1,N do 30 | for k = 1,K+1 do 31 | L[n][neighborhood[n][k]] = kernel[n][k] 32 | end 33 | end 34 | L:map(L:t(), function(xx, yy) if xx > yy then return xx else return yy end end ) 35 | 36 | -- compute unnormalized graph Laplacian: 37 | if normalized == false then 38 | local kernel_sums = torch.sum(kernel, 2) 39 | L:mul(-1) 40 | for n = 1,N do 41 | L[n][n] = kernel_sums[n] 42 | end 43 | 44 | -- compute normalized graph Laplacian: 45 | else 46 | local kernel_sums = torch.sum(kernel, 2) 47 | kernel_sums:pow(-.5) 48 | L:cmul(kernel_sums:expand(N, N)) 49 | L:cmul(kernel_sums:t():expand(N, N)) 50 | L:map(L:t(), function(xx, yy) if xx > yy then return xx else return yy end end ) 51 | end 52 | 53 | -- compute embedding: 54 | local vals,vectors = torch.eig(L, 'V') 55 | vals = vals[{{},1}] 56 | if normalized == true then 57 | vals,idx = torch.sort(vals,vals:dim(),true) 58 | else 59 | vals,idx = torch.sort(vals) 60 | end 61 | local res = torch.Tensor(N, d) 62 | for i=1,d do 63 | res[{{},i}] = vectors[{ {},{idx[i+1]} }]:clone() 64 | end 65 | 66 | -- return: 67 | return res 68 | end 69 | 70 | -- Return func: 71 | return laplacian_eigenmaps 72 | -------------------------------------------------------------------------------- /lle.lua: -------------------------------------------------------------------------------- 1 | -- LLE: 2 | -- 3 | -- Reference: Sam Roweis & Lawrence Saul, "Nonlinear dimensionality reduction by locally linear embedding", Dec 22, 2000. 4 | -- Original Code (Matlab): http://www.cs.nyu.edu/~roweis/lle/code.html 5 | -- 6 | local lle = function(vectors,opts) 7 | -- args: 8 | opts = opts or {} 9 | local d = opts.dim or 2 10 | local K = opts.neighbors or 2 11 | local dtol = opts.tol or 2 12 | local X = vectors 13 | 14 | -- dependencies: 15 | local pkg = require 'manifold' 16 | 17 | -- dims: 18 | local N,D = X:size(1),X:size(2) 19 | 20 | -- get nearest neighbors: 21 | local neighbors = pkg.neighbors(X) 22 | assert(torch.dist(neighbors[{{},1}]:float(), torch.range(1,N):float()) == 0, 'LLE cannot deal with duplicates') 23 | local neighborhood = neighbors[{ {},{2,2+K-1} }] 24 | 25 | -- solve for reconstruction weights: 26 | local tol = dtol or 0 27 | if K > D then 28 | tol = dtol or 1e-3 -- regularization in this case 29 | end 30 | local W = torch.zeros(N,K) 31 | local neighbors = torch.zeros(K,D) 32 | for ii = 1,N do 33 | -- copy neighbors: 34 | local indexes = neighborhood[ii] 35 | for i = 1,indexes:size(1) do 36 | neighbors[i] = X[indexes[i]] 37 | end 38 | 39 | -- shift point to origin: 40 | local z = neighbors - X[{ {ii,ii},{} }]:clone():expand(K,D) 41 | 42 | -- local covariance matrix: 43 | local C = z * z:t() 44 | 45 | -- regularize 46 | if tol > 0 then 47 | C:add( torch.eye(K)*tol*torch.trace(C) ) 48 | end 49 | 50 | -- solve C*W=1 51 | local right = torch.ones(K,1) 52 | local res = torch.gels(right,C) 53 | W[ii] = res 54 | W[ii]:div(W[ii]:sum()) 55 | end 56 | 57 | -- compute embedding from eigenvectors of cost matrix M = (I-W)' * (I-W) 58 | local M = torch.eye(N) 59 | for ii = 1,N do 60 | local w = W[ii] 61 | local indexes = neighborhood[ii] 62 | for i = 1,indexes:size(1) do 63 | local jj = indexes[i] 64 | M[{ {ii},{jj} }]:add(-w[i]) 65 | M[{ {jj},{ii} }]:add(-w[i]) 66 | for j = 1,indexes:size(1) do 67 | M[{{jj},{indexes[j]}}]:add(w[i]*w[j]) 68 | end 69 | end 70 | end 71 | 72 | -- embedding: 73 | local vals,vectors = torch.eig(M, 'V') 74 | local n = M:size(1) 75 | vals = vals[{{},1}] 76 | vals,idx = torch.sort(vals) 77 | local res = torch.Tensor(vectors:size(1), d) 78 | for i=1,d do 79 | res[{{},i}] = vectors[{ {},{idx[i+1]} }]:clone() 80 | end 81 | res:mul(math.sqrt(N)) 82 | 83 | -- return: 84 | return res 85 | end 86 | 87 | -- Return func: 88 | return lle 89 | -------------------------------------------------------------------------------- /demos/demo_tsne.lua: -------------------------------------------------------------------------------- 1 | local manifold = require 'manifold' 2 | require 'gnuplot' 3 | require 'image' 4 | 5 | -- function to show an MNIST 2D group scatter plot: 6 | local function show_scatter_plot(method, mapped_x, labels) 7 | 8 | -- count label sizes: 9 | local K = 10 10 | local cnts = torch.zeros(K) 11 | for n = 1,labels:nElement() do 12 | cnts[labels[n] + 1] = cnts[labels[n] + 1] + 1 13 | end 14 | 15 | -- separate mapped data per label: 16 | mapped_data = {} 17 | for k = 1,K do 18 | mapped_data[k] = {'Digit ' .. k-1, torch.Tensor(cnts[k], opts.ndims), '+'} 19 | end 20 | local offset = torch.Tensor(K):fill(1) 21 | for n = 1,labels:nElement() do 22 | mapped_data[labels[n] + 1][2][offset[labels[n] + 1]]:copy(mapped_x[n]) 23 | offset[labels[n] + 1] = offset[labels[n] + 1] + 1 24 | end 25 | 26 | -- show results in scatter plot: 27 | gnuplot.figure(); gnuplot.grid(true); gnuplot.title(method) 28 | gnuplot.plot(mapped_data) 29 | end 30 | 31 | 32 | -- show map with original digits: 33 | local function show_map(method, mapped_data, X) 34 | 35 | -- draw map with original digits: 36 | local im_size = 2048 37 | local background = 0 38 | local background_removal = true 39 | map_im = manifold.draw_image_map(mapped_data, X:resize(X:size(1), 1, 28, 28), im_size, background, background_removal) 40 | 41 | -- plot results: 42 | image.display{image=map_im, legend=method, zoom=0.5} 43 | image.savePNG(method .. '.png', map_im) 44 | end 45 | 46 | 47 | -- function that performs demo of t-SNE code on MNIST: 48 | local function demo_tsne() 49 | 50 | -- amount of data to use for test: 51 | local N = 5000 52 | 53 | -- load subset of MNIST test data: 54 | local mnist = require 'mnist' 55 | local testset = mnist.testdataset() 56 | testset.size = N 57 | testset.data = testset.data:narrow(1, 1, N) 58 | testset.label = testset.label:narrow(1, 1, N) 59 | local x = torch.DoubleTensor(testset.data:size()):copy(testset.data) 60 | x:resize(x:size(1), x:size(2) * x:size(3)) 61 | local labels = testset.label 62 | 63 | -- run t-SNE: 64 | local timer = torch.Timer() 65 | opts = {ndims = 2, perplexity = 30, pca = 50, use_bh = false} 66 | mapped_x1 = manifold.embedding.tsne(x, opts) 67 | print('Successfully performed t-SNE in ' .. timer:time().real .. ' seconds.') 68 | show_scatter_plot('t-SNE', mapped_x1, labels) 69 | show_map('t-SNE', mapped_x1, x:clone()) 70 | 71 | -- run Barnes-Hut t-SNE: 72 | opts = {ndims = 2, perplexity = 30, pca = 50, use_bh = true, theta = 0.5} 73 | timer:reset() 74 | mapped_x2 = manifold.embedding.tsne(x, opts) 75 | print('Successfully performed Barnes Hut t-SNE in ' .. timer:time().real .. ' seconds.') 76 | show_scatter_plot('Barnes-Hut t-SNE', mapped_x2, labels) 77 | show_map('Barnes-Hut t-SNE', mapped_x2, x:clone()) 78 | end 79 | 80 | 81 | -- run the demo: 82 | demo_tsne() 83 | -------------------------------------------------------------------------------- /init.lua: -------------------------------------------------------------------------------- 1 | -- Deps: 2 | require 'torch' 3 | 4 | -- Random projections: 5 | local random = function(vectors,opts) 6 | -- args: 7 | opts = opts or {} 8 | local dim = opts.dim or 2 9 | 10 | -- random mapping: 11 | local mapping = torch.randn(vectors:size(2),dim):div(dim) 12 | 13 | -- project: 14 | return vectors * mapping 15 | end 16 | 17 | -- Compute distances: 18 | local distances = function(vectors,norm) 19 | -- args: 20 | local X = vectors 21 | local norm = norm or 2 22 | local N,D = X:size(1),X:size(2) 23 | 24 | -- compute L2 distances: 25 | local distances 26 | if norm == 2 then 27 | local X2 = X:clone():cmul(X):sum(2) 28 | distances = (X*X:t()*-2) + X2:expand(N,N) + X2:reshape(1,N):expand(N,N) 29 | distances:abs():sqrt() 30 | elseif norm == 1 then 31 | distances = X.new(N,N) 32 | local tmp = X.new(N,D) 33 | for i = 1,N do 34 | local x = X[i]:clone():reshape(1,D):expand(N,D) 35 | tmp[{}] = X 36 | local dist = tmp:add(-1,x):abs():sum(2):squeeze() 37 | distances[i] = dist 38 | end 39 | else 40 | error('norm must be 1 or 2') 41 | end 42 | 43 | -- return dists 44 | return distances 45 | end 46 | 47 | -- Compute neighbors: 48 | local neighbors = function(vectors,norm) 49 | -- args: 50 | local X = vectors 51 | local N,D = X:size(1),X:size(2) 52 | 53 | -- compute L2 distances: 54 | local distance = distances(X,norm) 55 | 56 | -- sort: 57 | local dists,index = distance:sort(2) 58 | 59 | -- insure identity for 1st index: 60 | for i = 1,(#distance)[1] do 61 | local id1 = index[{i,1}] 62 | if id1 ~= i then 63 | for j = 2,(#distance)[1] do 64 | local id2 = index[{i,j}] 65 | if id2 == i then 66 | index[{i,j}] = id1 67 | index[{i,1}] = id2 68 | break 69 | end 70 | end 71 | end 72 | end 73 | 74 | -- return index 75 | return index,dists 76 | end 77 | 78 | -- Remove duplicates: 79 | local removeDuplicates = function(vectors) 80 | -- args: 81 | local X = vectors 82 | local N,D = X:size(1),X:size(2) 83 | 84 | -- remove duplicates 85 | local neighbors = neighbors(X) 86 | 87 | -- mark single vectors as ok: 88 | local oks = {} 89 | for i = 1,N do 90 | if neighbors[i][1] == i then 91 | table.insert(oks,i) 92 | end 93 | end 94 | 95 | -- keep singles: 96 | local matrix = torch.Tensor(#oks,D) 97 | for i,ok in ipairs(oks) do 98 | matrix[i] = X[ok] 99 | end 100 | 101 | -- return new filtered matrix: 102 | return matrix,oks 103 | end 104 | 105 | 106 | -- function that draws 2D map of images: 107 | local function draw_image_map(inp_X, images, inp_map_size, inp_background, inp_background_removal) 108 | 109 | -- input options: 110 | local map_size = inp_map_size or 512 111 | local background = inp_background or 0 112 | local background_removal = inp_background_removal or false 113 | 114 | -- check inputs are correct: 115 | local X = inp_X:clone() 116 | local N = X:size(1) 117 | if X:nDimension() ~= 2 or X:size(2) ~= 2 then 118 | error('This function is designed to operate on 2D embeddings only.') 119 | end 120 | if (type(images) ~= 'table' or X:size(1) ~= #images) and (torch.isTensor(images) == false or images:nDimension() ~= 4 or images:size(1) ~= N) then 121 | error('Images should be specified as a list of filenames or as an NxCxHxW tensor.') 122 | end 123 | 124 | -- prepare some variables: 125 | local num_channels = 3 126 | if torch.isTensor(images) then 127 | num_channels = images:size(2) 128 | end 129 | local map_im = torch.DoubleTensor(num_channels, map_size, map_size):fill(background) 130 | X = X:add(-X:min(1):expand(N, 2)) 131 | X = X:cdiv(X:max(1):expand(N, 2)) 132 | 133 | -- fill map with images: 134 | for n = 1,N do 135 | 136 | -- get current image: 137 | local cur_im 138 | if type(images) == 'table' then 139 | cur_im = image.load(images[n]) 140 | else 141 | cur_im = images[n] 142 | end 143 | 144 | -- place current image: 145 | local y_loc = 1 + math.floor(X[n][1] * (map_size - cur_im:size(2))) 146 | local x_loc = 1 + math.floor(X[n][2] * (map_size - cur_im:size(3))) 147 | if background_removal == false then -- no background removal 148 | map_im:sub(1, num_channels, y_loc, y_loc + cur_im:size(2) - 1, 149 | x_loc, x_loc + cur_im:size(3) - 1):copy(cur_im) 150 | else -- background removal 151 | for c = 1,num_channels do 152 | for h = 1,cur_im:size(2) do 153 | for w = 1,cur_im:size(3) do 154 | if cur_im[c][h][w] ~= background then 155 | map_im[c][y_loc + h - 1][x_loc + w - 1] = cur_im[c][h][w] 156 | end 157 | end 158 | end 159 | end 160 | end 161 | end 162 | 163 | -- return map: 164 | return map_im 165 | end 166 | 167 | -- function that draw text map: 168 | local function draw_text_map(X, words, inp_map_size, inp_font_size) 169 | -- NOTE: This function assumes vocabulary is indexed by words, values indicate the index of a word into X! 170 | 171 | -- input options: 172 | local map_size = inp_map_size or 512 173 | local font_size = inp_font_size or 9 174 | 175 | -- check inputs are correct: 176 | local N = X:size(1) 177 | if X:nDimension() ~= 2 or X:size(2) ~= 2 then 178 | error('This function is designed to operate on 2D embeddings only.') 179 | end 180 | if X:size(1) ~= #words then 181 | error('Number of words should match the number of rows in X.') 182 | end 183 | 184 | -- prepare image for rendering: 185 | require 'image' 186 | require 'qtwidget' 187 | require 'qttorch' 188 | local win = qtwidget.newimage(map_size, map_size) 189 | 190 | --render the words: 191 | for key,val in pairs(words) do 192 | win:setfont(qt.QFont{serif = false, size = fontsize}) 193 | win:moveto(math.floor(X[val][1] * map_size), math.floor(X[val][2] * map_size)) 194 | win:show(key) 195 | end 196 | 197 | -- render to tensor: 198 | local map_im = win:image():toTensor() 199 | 200 | -- return text map: 201 | return map_im 202 | end 203 | 204 | 205 | -- Package: 206 | return { 207 | embedding = { 208 | random = random, 209 | lle = require 'manifold.lle', 210 | tsne = require 'manifold.tsne', 211 | laplacian_eigenmaps = require 'manifold.laplacian_eigenmaps', 212 | }, 213 | removeDuplicates = removeDuplicates, 214 | neighbors = neighbors, 215 | distances = distances, 216 | draw_image_map = draw_image_map, 217 | draw_text_map = draw_text_map 218 | } 219 | -------------------------------------------------------------------------------- /tsne.lua: -------------------------------------------------------------------------------- 1 | 2 | -- implementation of P-value computations: 3 | local function x2p(data, perplexity, tol) 4 | 5 | -- allocate all the memory we need: 6 | local eps = 1e-10 7 | local N = data:size(1) 8 | local D = torch.DoubleTensor(N, N) 9 | local y_buf = torch.DoubleTensor(data:size()) 10 | local row_D = torch.DoubleTensor(N, 1) 11 | local row_P = torch.DoubleTensor(N, 1) 12 | local P = torch.DoubleTensor(N, N) 13 | 14 | -- compute pairwise distance matrix: 15 | torch.cmul(y_buf, data, data) 16 | torch.sum(row_D, y_buf, 2) 17 | local row_Dc = torch.expand(row_D, N, N) 18 | torch.mm(D, data, data:t()) 19 | D:mul(-2):add(row_Dc):add(row_Dc:t()) 20 | 21 | -- loop over all instances: 22 | for n = 1,N do 23 | 24 | -- set minimum and maximum values for precision: 25 | local beta = 1 26 | local betamin = -math.huge 27 | local betamax = math.huge 28 | 29 | -- compute the Gaussian kernel and corresponding perplexity: (should put this in a local function!) 30 | row_P:copy(D[n]) 31 | row_D:copy(D[n]) 32 | row_P:mul(-beta):exp() 33 | row_P[n][1] = 0 34 | local sum_P = row_P:sum() 35 | local sum_DP = row_D:cmul(row_P):sum() 36 | local Hi = math.log(sum_P) + beta * sum_DP / sum_P 37 | 38 | -- evaluate whether the perplexity is within tolerance 39 | local H_diff = Hi - math.log(perplexity) 40 | local tries = 0 41 | while math.abs(H_diff) > tol and tries < 50 do 42 | 43 | -- if not, increase of decrease precision: 44 | if H_diff > 0 then 45 | betamin = beta 46 | if betamax == math.huge then 47 | beta = beta * 2 48 | else 49 | beta = (beta + betamax) / 2 50 | end 51 | else 52 | betamax = beta 53 | if betamin == -math.huge then 54 | beta = beta / 2 55 | else 56 | beta = (beta + betamin) / 2 57 | end 58 | end 59 | 60 | -- recompute row of P and correponding perplexity: 61 | row_P:copy(D[n]) 62 | row_D:copy(D[n]) 63 | row_P:mul(-beta):exp() 64 | row_P[n][1] = 0 65 | sum_P = row_P:sum() 66 | sum_DP = row_D:cmul(row_P):sum() 67 | Hi = math.log(sum_P) + beta * sum_DP / sum_P 68 | 69 | -- update error: 70 | H_diff = Hi - math.log(perplexity) 71 | tries = tries + 1 72 | end 73 | 74 | -- set the final row of P: 75 | row_P:div(sum_P) 76 | P[n]:copy(row_P) 77 | end 78 | 79 | -- return output: 80 | return P 81 | end 82 | 83 | 84 | -- implementation of sign function: 85 | local function sign(x) 86 | if x < 0 then 87 | return -1 88 | else 89 | return 1 90 | end 91 | end 92 | 93 | 94 | -- function that runs the Barnes-Hut SNE executable: 95 | local function run_bhtsne(data, opts) 96 | 97 | -- default values: 98 | local ffi = require 'ffi' 99 | opts = opts or {} 100 | local no_dims = opts.dim or 2 101 | local perplexity = opts.perplexity or 30 102 | local theta = opts.theta or 0.5 103 | 104 | -- pack data: 105 | local raw = data:data() 106 | local nchars = data:nElement()*8 + 2*8 + 3*4 107 | local data_char = ffi.new('char[?]', nchars) 108 | local data_int = ffi.cast('int *', data_char) 109 | local data_double = ffi.cast('double *', data_char + 2*4) 110 | local data_int2 = ffi.cast('int *', data_char + 2*4 + 2*8) 111 | local data_double2 = ffi.cast('double *', data_char + 3*4 + 2*8) 112 | data_int[0] = data:size(1) 113 | data_int[1] = data:size(2) 114 | data_double[0] = ffi.cast('double',theta) 115 | data_double[1] = ffi.cast('double',perplexity) 116 | data_int2[0] = no_dims 117 | ffi.copy(data_double2, raw, data:nElement() * 8) 118 | 119 | -- pack argument 120 | local packed = ffi.string(data_char, nchars) 121 | local f = io.open('data.dat','w') 122 | f:write(packed) 123 | f:close() 124 | 125 | -- exec: 126 | local cmd 127 | -- TODO: we should prepend the exact install path to thee binaries 128 | if ffi.os == 'OSX' then 129 | cmd = 'bhtsne_maci' 130 | else 131 | cmd = 'bhtsne_linux' 132 | end 133 | 134 | -- run: 135 | os.execute(cmd) 136 | 137 | -- read result: 138 | local f = io.open('result.dat','r') 139 | local res = f:read('*all') 140 | local data_int = ffi.cast('int *', res) 141 | local n = data_int[0] 142 | local nd = data_int[1] 143 | local data_next = data_int + 2 144 | 145 | -- clear files: 146 | os.remove('data.dat') 147 | os.remove('result.dat') 148 | 149 | -- data? 150 | local odata,lm,costs 151 | if n == 0 then 152 | -- no data (error?) 153 | print('no data found in embedding') 154 | return 155 | else 156 | -- data: 157 | odata = torch.DoubleTensor(n,nd):zero() 158 | local rp = odata:data() 159 | local data_double = ffi.cast('double *', data_next) 160 | ffi.copy(rp, data_double, n*nd*8) 161 | local data_next = data_double + n*nd 162 | 163 | -- next vector: 164 | lm = torch.IntTensor(n) 165 | local rp = lm:data() 166 | local data_int = ffi.cast('int *', data_next) 167 | ffi.copy(rp, data_int, n*4) 168 | local data_next = data_int + n 169 | lm:add(1) 170 | 171 | -- next vector: 172 | costs = torch.DoubleTensor(n):zero() 173 | local rp = costs:data() 174 | local data_double = ffi.cast('double *', data_next) 175 | ffi.copy(rp, data_double, n*8) 176 | end 177 | 178 | -- re-order: 179 | if landmarks == 1 then 180 | local odatar = odata:clone():zero() 181 | for i = 1,lm:size(1) do 182 | odatar[lm[i]] = odata[i] 183 | end 184 | odata = odatar 185 | end 186 | 187 | -- return mapped data: 188 | return odata 189 | end 190 | 191 | 192 | -- implementation of t-SNE in Torch: 193 | local function tsne(data, opts) 194 | 195 | -- options: 196 | opts = opts or {} 197 | local no_dims = opts.dim or 2 198 | local perplexity = opts.perplexity or 30 199 | local use_bh = opts.use_bh or false 200 | local pca_dims = opts.pca or (use_bh and 100) or nil 201 | 202 | -- normalize input data: 203 | data:add(-data:min()) 204 | data:div( data:max()) 205 | 206 | -- first do PCA: 207 | local N = data:size(1) 208 | local D = data:size(2) 209 | if pca_dims and pca_dims < D then 210 | require 'unsup' 211 | print('Performing preprocessing using PCA...') 212 | local lambda,W = unsup.pca(data) 213 | W = W:narrow(2, 1, pca_dims) 214 | data = torch.mm(data, W) 215 | end 216 | 217 | -- run Barnes-Hut binary (when requested): 218 | if use_bh == true then 219 | local mapped_x = run_bhtsne(data, opts) 220 | return mapped_x 221 | end 222 | 223 | -- initialize some variables for the optimization: 224 | local momentum = 0.5 225 | local final_momentum = 0.8 226 | local mom_switch_iter = 250 227 | local stop_lying_iter = 200 228 | local max_iter = 1000 229 | local epsilon = 500 230 | local min_gain = 0.01 231 | local eps = 1e-12 232 | 233 | -- allocate all the memory we need: 234 | local buf = torch.DoubleTensor(N, N) 235 | local num = torch.DoubleTensor(N, N) 236 | local Q = torch.DoubleTensor(N, N) 237 | local y_buf = torch.DoubleTensor(N, no_dims) 238 | local n_buf = torch.DoubleTensor(N, 1) 239 | 240 | -- compute (asymmetric) P-values: 241 | print('Computing P-values...') 242 | local tol = 1e-4 243 | local P = x2p(data, perplexity, tol) 244 | 245 | -- symmetrize P-values: 246 | buf:copy(P) 247 | P:add(buf:t()) 248 | P:div(torch.sum(P)) 249 | 250 | -- compute constant term in KL divergence: 251 | buf:copy(P) 252 | buf:add(eps):log() 253 | local H_P = torch.sum(buf:cmul(P)) 254 | 255 | -- lie about the P-values: 256 | P:mul(4) 257 | 258 | -- initialize the solution, gradient and momentum storage, and gain: 259 | local y_data = torch.randn(N, no_dims):mul(0.0001) 260 | local y_grad = torch.DoubleTensor(N, no_dims) 261 | local y_incs = torch.zeros(N, no_dims) 262 | local y_gain = torch.ones(N, no_dims) 263 | 264 | -- main for-loop: 265 | print('Running t-SNE...') 266 | for iter = 1,max_iter do 267 | 268 | -- compute the joint probability that i and j are neighbors in the map: 269 | torch.cmul(y_buf, y_data, y_data) 270 | torch.sum(n_buf, y_buf, 2) 271 | local cp_n_buf = torch.expand(n_buf, N, N) 272 | torch.mm(num, y_data, y_data:t()) 273 | num:mul(-2) 274 | num:add(cp_n_buf):add(cp_n_buf:t()):add(1) 275 | buf:fill(1) 276 | torch.cdiv(num, buf, num) 277 | for n = 1,N do 278 | num[n][n] = 0 279 | end 280 | torch.div(Q, num, num:sum()) 281 | 282 | -- compute the gradients: 283 | buf:copy(P) 284 | buf:add(-Q) 285 | buf:cmul(num) 286 | torch.sum(n_buf, buf, 2) 287 | num:fill(0) 288 | for n = 1,N do 289 | num[n][n] = n_buf[n][1] 290 | end 291 | num:add(-buf) 292 | torch.mm(y_grad, num, y_data) 293 | y_grad:mul(4) 294 | 295 | -- update the solution: 296 | y_gain:map2(y_grad, y_incs, function(gain, grad, incs) if sign(grad) ~= sign(incs) then return (gain + 0.2) else return math.min(gain * 0.8, .01) end end) 297 | y_incs:mul(momentum) 298 | y_incs:addcmul(-epsilon, y_grad, y_gain) 299 | y_data:add(y_incs) 300 | y_data:add(-torch.mean(y_data, 1):reshape(1, no_dims):expand(y_data:size())) 301 | 302 | -- update learning parameters if necessary: 303 | if iter == mom_switch_iter then 304 | momentum = final_momentum 305 | end 306 | if iter == stop_lying_iter then 307 | P:div(4) 308 | end 309 | 310 | -- print out progress: 311 | if math.fmod(iter, 10) == 0 then 312 | Q:add(eps):log() 313 | local kl = H_P - torch.sum(Q:cmul(P)) 314 | print('Iteration ' .. iter .. ': KL divergence is ' .. kl) 315 | end 316 | end 317 | 318 | -- Compute the ending KL divergence 319 | if math.fmod(max_iter, 10) ~= 0 then 320 | Q:add(eps):log():cmul(P) 321 | end 322 | local kl = H_P - Q:sum() 323 | 324 | -- return output data and kl divergence 325 | return y_data, kl 326 | end 327 | 328 | -- return function: 329 | return tsne 330 | --------------------------------------------------------------------------------