├── .gitignore
├── README.md
├── browser
    ├── js
    │   ├── convnet.js
    │   ├── jquery-1.8.3.min.js
    │   ├── mnist_labels.js
    │   ├── util.js
    │   └── vis.js
    ├── mnist.html
    └── mnist
    │   ├── mnist_batch_0.png
    │   ├── mnist_batch_1.png
    │   ├── mnist_batch_2.png
    │   ├── mnist_batch_20.png
    │   ├── mnist_batch_3.png
    │   ├── mnist_batch_4.png
    │   ├── mnist_batch_5.png
    │   ├── mnist_batch_6.png
    │   ├── mnist_batch_7.png
    │   └── mnist_batch_8.png
└── static
    ├── package.json
    ├── public
        ├── cifar10.html
        ├── cifar10
        │   ├── cifar10_batch_0.png
        │   └── cifar10_batch_50.png
        ├── css
        │   └── jquery-ui.min.css
        ├── image_regression.html
        ├── imgs
        │   ├── balls.jpg
        │   ├── battery.jpg
        │   ├── cat.jpg
        │   ├── chess.png
        │   ├── chip.jpg
        │   ├── debug.jpg
        │   ├── dora.jpg
        │   ├── earth.jpg
        │   ├── esher.png
        │   ├── fox.png
        │   ├── fractal.jpg
        │   ├── gradient.png
        │   ├── jitendra.jpg
        │   ├── lena.png
        │   ├── pencils.png
        │   ├── rainforest.jpg
        │   ├── reddit.jpg
        │   ├── rubiks.jpg
        │   ├── starry.jpg
        │   ├── tesla.jpg
        │   ├── twitter.png
        │   └── usa.png
        ├── js
        │   ├── cifar10_labels.js
        │   ├── convnet.js
        │   ├── jquery-1.8.3.min.js
        │   ├── jquery-ui.min.js
        │   ├── mnist_labels.js
        │   ├── util.js
        │   └── vis.js
        ├── mnist.html
        └── mnist
        │   ├── mnist_batch_0.png
        │   ├── mnist_batch_1.png
        │   ├── mnist_batch_2.png
        │   ├── mnist_batch_20.png
        │   ├── mnist_batch_3.png
        │   ├── mnist_batch_4.png
        │   ├── mnist_batch_5.png
        │   ├── mnist_batch_6.png
        │   ├── mnist_batch_7.png
        │   └── mnist_batch_8.png
    └── server.js


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # JavaScript and Artificial Intelligence
 2 | 
 3 | Samples and demos of JavaScript Artificial Intelligence.
 4 | 
 5 | AI is a broad term, but it is the better I known.
 6 | 
 7 | The samples are for Node.js and for browser. The covered topics (Work in Progress):
 8 | 
 9 | - Tree Evaluation
10 | - Neural Networks
11 | - Genetic Algorithms
12 | - Evolutionary Programming
13 | - Hybrid Approach
14 | - Machine and Deep Learning
15 | - Decision Trees
16 | 
17 | ## Presentation
18 | 
19 | - [Presentation Sources](https://github.com/ajlopez/Talks/JavaScriptAI)
20 | - [Presentation Online](http://ajlopez.github.io/Talks/JavaScriptAI)
21 | 
22 | ## Projects
23 | 
24 | - [ConvNetJs Deep Learning](http://cs.stanford.edu/people/karpathy/convnetjs/)
25 | - [SimpleNeuron Neural Networks](https://github.com/ajlopez/SimpleNeuron)
26 | - [SimpleGA Genetic Algorithms](https://github.com/ajlopez/SimpleGA)
27 | - [SimpleDT Decision Tress](https://github.com/ajlopez/SimpleDT)
28 | - [SimpleGammon Evaluation Backgammon](https://github.com/ajlopez/SimpleGammon)
29 | 
30 | ## Resources
31 | 
32 | - [Artificial Intelligence: A Modern Approach](http://aima.cs.berkeley.edu/)
33 | - [Computer Backgammon](http://www.bkgm.com/articles/Berliner/ComputerBackgammon/)
34 | - [Backgammon Computer Program Beats World Champion](http://www.bkgm.com/articles/Berliner/BackgammonProgramBeatsWorldChamp/)
35 | - [Machine Learning in Action](http://www.manning.com/pharrington/) with source code
36 | - [Binding Machine Learning and node.js](http://blog.bigml.com/2013/06/27/binding-machine-learning-and-node-js/)
37 | - [What is deep learning, and why should you care?](http://radar.oreilly.com/2014/07/what-is-deep-learning-and-why-should-you-care.html)
38 | - [ImageNet Classification with Deep Convolutional Neural Networks](http://www.cs.toronto.edu/~fritz/absps/imagenet.pdf)
39 | - [Dropout: A simple and effective way to improve neural networks](http://videolectures.net/nips2012_hinton_networks/)
40 | - [Rectifier (neural networks)](http://en.wikipedia.org/wiki/Rectifier_(neural_networks))
41 | 
42 | ## Datasets
43 | 
44 | - [Weka Collections of Datasets](http://www.cs.waikato.ac.nz/ml/weka/datasets.html)
45 | - [Machine Learning Databases](http://archive.ics.uci.edu/ml/machine-learning-databases/)
46 | 
47 | 


--------------------------------------------------------------------------------
/browser/js/convnet.js:
--------------------------------------------------------------------------------
   1 | var convnetjs = convnetjs || { REVISION: 'ALPHA' };
   2 | (function(global) {
   3 |   "use strict";
   4 | 
   5 |   // Random number utilities
   6 |   var return_v = false;
   7 |   var v_val = 0.0;
   8 |   var gaussRandom = function() {
   9 |     if(return_v) { 
  10 |       return_v = false;
  11 |       return v_val; 
  12 |     }
  13 |     var u = 2*Math.random()-1;
  14 |     var v = 2*Math.random()-1;
  15 |     var r = u*u + v*v;
  16 |     if(r == 0 || r > 1) return gaussRandom();
  17 |     var c = Math.sqrt(-2*Math.log(r)/r);
  18 |     v_val = v*c; // cache this
  19 |     return_v = true;
  20 |     return u*c;
  21 |   }
  22 |   var randf = function(a, b) { return Math.random()*(b-a)+a; }
  23 |   var randi = function(a, b) { return Math.floor(Math.random()*(b-a)+a); }
  24 |   var randn = function(mu, std){ return mu+gaussRandom()*std; }
  25 | 
  26 |   // Array utilities
  27 |   var zeros = function(n) {
  28 |     if(typeof(n)==='undefined' || isNaN(n)) { return []; }
  29 |     if(typeof ArrayBuffer === 'undefined') {
  30 |       // lacking browser support
  31 |       var arr = new Array(n);
  32 |       for(var i=0;i<n;i++) { arr[i]= 0; }
  33 |       return arr;
  34 |     } else {
  35 |       return new Float64Array(n);
  36 |     }
  37 |   }
  38 | 
  39 |   var arrContains = function(arr, elt) {
  40 |     for(var i=0,n=arr.length;i<n;i++) {
  41 |       if(arr[i]===elt) return true;
  42 |     }
  43 |     return false;
  44 |   }
  45 | 
  46 |   var arrUnique = function(arr) {
  47 |     var b = [];
  48 |     for(var i=0,n=arr.length;i<n;i++) {
  49 |       if(!arrContains(b, arr[i])) {
  50 |         b.push(arr[i]);
  51 |       }
  52 |     }
  53 |     return b;
  54 |   }
  55 | 
  56 |   // return max and min of a given non-empty array.
  57 |   var maxmin = function(w) {
  58 |     if(w.length === 0) { return {}; } // ... ;s
  59 |     var maxv = w[0];
  60 |     var minv = w[0];
  61 |     var maxi = 0;
  62 |     var mini = 0;
  63 |     var n = w.length;
  64 |     for(var i=1;i<n;i++) {
  65 |       if(w[i] > maxv) { maxv = w[i]; maxi = i; } 
  66 |       if(w[i] < minv) { minv = w[i]; mini = i; } 
  67 |     }
  68 |     return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv};
  69 |   }
  70 | 
  71 |   // create random permutation of numbers, in range [0...n-1]
  72 |   var randperm = function(n) {
  73 |     var i = n,
  74 |         j = 0,
  75 |         temp;
  76 |     var array = [];
  77 |     for(var q=0;q<n;q++)array[q]=q;
  78 |     while (i--) {
  79 |         j = Math.floor(Math.random() * (i+1));
  80 |         temp = array[i];
  81 |         array[i] = array[j];
  82 |         array[j] = temp;
  83 |     }
  84 |     return array;
  85 |   }
  86 | 
  87 |   // sample from list lst according to probabilities in list probs
  88 |   // the two lists are of same size, and probs adds up to 1
  89 |   var weightedSample = function(lst, probs) {
  90 |     var p = randf(0, 1.0);
  91 |     var cumprob = 0.0;
  92 |     for(var k=0,n=lst.length;k<n;k++) {
  93 |       cumprob += probs[k];
  94 |       if(p < cumprob) { return lst[k]; }
  95 |     }
  96 |   }
  97 | 
  98 |   // syntactic sugar function for getting default parameter values
  99 |   var getopt = function(opt, field_name, default_value) {
 100 |     if(typeof field_name === 'string') {
 101 |       // case of single string
 102 |       return (typeof opt[field_name] !== 'undefined') ? opt[field_name] : default_value;
 103 |     } else {
 104 |       // assume we are given a list of string instead
 105 |       var ret = default_value;
 106 |       for(var i=0;i<field_name.length;i++) {
 107 |         var f = field_name[i];
 108 |         if (typeof opt[f] !== 'undefined') {
 109 |           ret = opt[f]; // overwrite return value
 110 |         }
 111 |       }
 112 |       return ret;
 113 |     }
 114 |   }
 115 | 
 116 |   function assert(condition, message) {
 117 |     if (!condition) {
 118 |       message = message || "Assertion failed";
 119 |       if (typeof Error !== "undefined") {
 120 |         throw new Error(message);
 121 |       }
 122 |       throw message; // Fallback
 123 |     }
 124 |   }
 125 | 
 126 |   global.randf = randf;
 127 |   global.randi = randi;
 128 |   global.randn = randn;
 129 |   global.zeros = zeros;
 130 |   global.maxmin = maxmin;
 131 |   global.randperm = randperm;
 132 |   global.weightedSample = weightedSample;
 133 |   global.arrUnique = arrUnique;
 134 |   global.arrContains = arrContains;
 135 |   global.getopt = getopt;
 136 |   global.assert = assert;
 137 |   
 138 | })(convnetjs);
 139 | (function(global) {
 140 |   "use strict";
 141 | 
 142 |   // Vol is the basic building block of all data in a net.
 143 |   // it is essentially just a 3D volume of numbers, with a
 144 |   // width (sx), height (sy), and depth (depth).
 145 |   // it is used to hold data for all filters, all volumes,
 146 |   // all weights, and also stores all gradients w.r.t. 
 147 |   // the data. c is optionally a value to initialize the volume
 148 |   // with. If c is missing, fills the Vol with random numbers.
 149 |   var Vol = function(sx, sy, depth, c) {
 150 |     // this is how you check if a variable is an array. Oh, Javascript :)
 151 |     if(Object.prototype.toString.call(sx) === '[object Array]') {
 152 |       // we were given a list in sx, assume 1D volume and fill it up
 153 |       this.sx = 1;
 154 |       this.sy = 1;
 155 |       this.depth = sx.length;
 156 |       // we have to do the following copy because we want to use
 157 |       // fast typed arrays, not an ordinary javascript array
 158 |       this.w = global.zeros(this.depth);
 159 |       this.dw = global.zeros(this.depth);
 160 |       for(var i=0;i<this.depth;i++) {
 161 |         this.w[i] = sx[i];
 162 |       }
 163 |     } else {
 164 |       // we were given dimensions of the vol
 165 |       this.sx = sx;
 166 |       this.sy = sy;
 167 |       this.depth = depth;
 168 |       var n = sx*sy*depth;
 169 |       this.w = global.zeros(n);
 170 |       this.dw = global.zeros(n);
 171 |       if(typeof c === 'undefined') {
 172 |         // weight normalization is done to equalize the output
 173 |         // variance of every neuron, otherwise neurons with a lot
 174 |         // of incoming connections have outputs of larger variance
 175 |         var scale = Math.sqrt(1.0/(sx*sy*depth));
 176 |         for(var i=0;i<n;i++) { 
 177 |           this.w[i] = global.randn(0.0, scale);
 178 |         }
 179 |       } else {
 180 |         for(var i=0;i<n;i++) { 
 181 |           this.w[i] = c;
 182 |         }
 183 |       }
 184 |     }
 185 |   }
 186 | 
 187 |   Vol.prototype = {
 188 |     get: function(x, y, d) { 
 189 |       var ix=((this.sx * y)+x)*this.depth+d;
 190 |       return this.w[ix];
 191 |     },
 192 |     set: function(x, y, d, v) { 
 193 |       var ix=((this.sx * y)+x)*this.depth+d;
 194 |       this.w[ix] = v; 
 195 |     },
 196 |     add: function(x, y, d, v) { 
 197 |       var ix=((this.sx * y)+x)*this.depth+d;
 198 |       this.w[ix] += v; 
 199 |     },
 200 |     get_grad: function(x, y, d) { 
 201 |       var ix = ((this.sx * y)+x)*this.depth+d;
 202 |       return this.dw[ix]; 
 203 |     },
 204 |     set_grad: function(x, y, d, v) { 
 205 |       var ix = ((this.sx * y)+x)*this.depth+d;
 206 |       this.dw[ix] = v; 
 207 |     },
 208 |     add_grad: function(x, y, d, v) { 
 209 |       var ix = ((this.sx * y)+x)*this.depth+d;
 210 |       this.dw[ix] += v; 
 211 |     },
 212 |     cloneAndZero: function() { return new Vol(this.sx, this.sy, this.depth, 0.0)},
 213 |     clone: function() {
 214 |       var V = new Vol(this.sx, this.sy, this.depth, 0.0);
 215 |       var n = this.w.length;
 216 |       for(var i=0;i<n;i++) { V.w[i] = this.w[i]; }
 217 |       return V;
 218 |     },
 219 |     addFrom: function(V) { for(var k=0;k<this.w.length;k++) { this.w[k] += V.w[k]; }},
 220 |     addFromScaled: function(V, a) { for(var k=0;k<this.w.length;k++) { this.w[k] += a*V.w[k]; }},
 221 |     setConst: function(a) { for(var k=0;k<this.w.length;k++) { this.w[k] = a; }},
 222 | 
 223 |     toJSON: function() {
 224 |       // todo: we may want to only save d most significant digits to save space
 225 |       var json = {}
 226 |       json.sx = this.sx; 
 227 |       json.sy = this.sy;
 228 |       json.depth = this.depth;
 229 |       json.w = this.w;
 230 |       return json;
 231 |       // we wont back up gradients to save space
 232 |     },
 233 |     fromJSON: function(json) {
 234 |       this.sx = json.sx;
 235 |       this.sy = json.sy;
 236 |       this.depth = json.depth;
 237 | 
 238 |       var n = this.sx*this.sy*this.depth;
 239 |       this.w = global.zeros(n);
 240 |       this.dw = global.zeros(n);
 241 |       // copy over the elements.
 242 |       for(var i=0;i<n;i++) {
 243 |         this.w[i] = json.w[i];
 244 |       }
 245 |     }
 246 |   }
 247 | 
 248 |   global.Vol = Vol;
 249 | })(convnetjs);
 250 | (function(global) {
 251 |   "use strict";
 252 |   var Vol = global.Vol; // convenience
 253 | 
 254 |   // Volume utilities
 255 |   // intended for use with data augmentation
 256 |   // crop is the size of output
 257 |   // dx,dy are offset wrt incoming volume, of the shift
 258 |   // fliplr is boolean on whether we also want to flip left<->right
 259 |   var augment = function(V, crop, dx, dy, fliplr) {
 260 |     // note assumes square outputs of size crop x crop
 261 |     if(typeof(fliplr)==='undefined') var fliplr = false;
 262 |     if(typeof(dx)==='undefined') var dx = global.randi(0, V.sx - crop);
 263 |     if(typeof(dy)==='undefined') var dy = global.randi(0, V.sy - crop);
 264 |     
 265 |     // randomly sample a crop in the input volume
 266 |     var W;
 267 |     if(crop !== V.sx || dx!==0 || dy!==0) {
 268 |       W = new Vol(crop, crop, V.depth, 0.0);
 269 |       for(var x=0;x<crop;x++) {
 270 |         for(var y=0;y<crop;y++) {
 271 |           if(x+dx<0 || x+dx>=V.sx || y+dy<0 || y+dy>=V.sy) continue; // oob
 272 |           for(var d=0;d<V.depth;d++) {
 273 |            W.set(x,y,d,V.get(x+dx,y+dy,d)); // copy data over
 274 |           }
 275 |         }
 276 |       }
 277 |     } else {
 278 |       W = V;
 279 |     }
 280 | 
 281 |     if(fliplr) {
 282 |       // flip volume horziontally
 283 |       var W2 = W.cloneAndZero();
 284 |       for(var x=0;x<W.sx;x++) {
 285 |         for(var y=0;y<W.sy;y++) {
 286 |           for(var d=0;d<W.depth;d++) {
 287 |            W2.set(x,y,d,W.get(W.sx - x - 1,y,d)); // copy data over
 288 |           }
 289 |         }
 290 |       }
 291 |       W = W2; //swap
 292 |     }
 293 |     return W;
 294 |   }
 295 | 
 296 |   // img is a DOM element that contains a loaded image
 297 |   // returns a Vol of size (W, H, 4). 4 is for RGBA
 298 |   var img_to_vol = function(img, convert_grayscale) {
 299 | 
 300 |     if(typeof(convert_grayscale)==='undefined') var convert_grayscale = false;
 301 | 
 302 |     var canvas = document.createElement('canvas');
 303 |     canvas.width = img.width;
 304 |     canvas.height = img.height;
 305 |     var ctx = canvas.getContext("2d");
 306 | 
 307 |     // due to a Firefox bug
 308 |     try {
 309 |       ctx.drawImage(img, 0, 0);
 310 |     } catch (e) {
 311 |       if (e.name === "NS_ERROR_NOT_AVAILABLE") {
 312 |         // sometimes happens, lets just abort
 313 |         return false;
 314 |       } else {
 315 |         throw e;
 316 |       }
 317 |     }
 318 | 
 319 |     try {
 320 |       var img_data = ctx.getImageData(0, 0, canvas.width, canvas.height);
 321 |     } catch (e) {
 322 |       if(e.name === 'IndexSizeError') {
 323 |         return false; // not sure what causes this sometimes but okay abort
 324 |       } else {
 325 |         throw e;
 326 |       }
 327 |     }
 328 | 
 329 |     // prepare the input: get pixels and normalize them
 330 |     var p = img_data.data;
 331 |     var W = img.width;
 332 |     var H = img.height;
 333 |     var pv = []
 334 |     for(var i=0;i<p.length;i++) {
 335 |       pv.push(p[i]/255.0-0.5); // normalize image pixels to [-0.5, 0.5]
 336 |     }
 337 |     var x = new Vol(W, H, 4, 0.0); //input volume (image)
 338 |     x.w = pv;
 339 | 
 340 |     if(convert_grayscale) {
 341 |       // flatten into depth=1 array
 342 |       var x1 = new Vol(W, H, 1, 0.0);
 343 |       for(var i=0;i<W;i++) {
 344 |         for(var j=0;j<H;j++) {
 345 |           x1.set(i,j,0,x.get(i,j,0));
 346 |         }
 347 |       }
 348 |       x = x1;
 349 |     }
 350 | 
 351 |     return x;
 352 |   }
 353 |   
 354 |   global.augment = augment;
 355 |   global.img_to_vol = img_to_vol;
 356 | 
 357 | })(convnetjs);
 358 | (function(global) {
 359 |   "use strict";
 360 |   var Vol = global.Vol; // convenience
 361 | 
 362 |   // This file contains all layers that do dot products with input,
 363 |   // but usually in a different connectivity pattern and weight sharing
 364 |   // schemes: 
 365 |   // - FullyConn is fully connected dot products 
 366 |   // - ConvLayer does convolutions (so weight sharing spatially)
 367 |   // putting them together in one file because they are very similar
 368 |   var ConvLayer = function(opt) {
 369 |     var opt = opt || {};
 370 | 
 371 |     // required
 372 |     this.out_depth = opt.filters;
 373 |     this.sx = opt.sx; // filter size. Should be odd if possible, it's cleaner.
 374 |     this.in_depth = opt.in_depth;
 375 |     this.in_sx = opt.in_sx;
 376 |     this.in_sy = opt.in_sy;
 377 |     
 378 |     // optional
 379 |     this.sy = typeof opt.sy !== 'undefined' ? opt.sy : this.sx;
 380 |     this.stride = typeof opt.stride !== 'undefined' ? opt.stride : 1; // stride at which we apply filters to input volume
 381 |     this.pad = typeof opt.pad !== 'undefined' ? opt.pad : 0; // amount of 0 padding to add around borders of input volume
 382 |     this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0;
 383 |     this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0;
 384 | 
 385 |     // computed
 386 |     // note we are doing floor, so if the strided convolution of the filter doesnt fit into the input
 387 |     // volume exactly, the output volume will be trimmed and not contain the (incomplete) computed
 388 |     // final application.
 389 |     this.out_sx = Math.floor((this.in_sx + this.pad * 2 - this.sx) / this.stride + 1);
 390 |     this.out_sy = Math.floor((this.in_sy + this.pad * 2 - this.sy) / this.stride + 1);
 391 |     this.layer_type = 'conv';
 392 | 
 393 |     // initializations
 394 |     var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0;
 395 |     this.filters = [];
 396 |     for(var i=0;i<this.out_depth;i++) { this.filters.push(new Vol(this.sx, this.sy, this.in_depth)); }
 397 |     this.biases = new Vol(1, 1, this.out_depth, bias);
 398 |   }
 399 |   ConvLayer.prototype = {
 400 |     forward: function(V, is_training) {
 401 |       // optimized code by @mdda that achieves 2x speedup over previous version
 402 | 
 403 |       this.in_act = V;
 404 |       var A = new Vol(this.out_sx |0, this.out_sy |0, this.out_depth |0, 0.0);
 405 |       
 406 |       var V_sx = V.sx |0;
 407 |       var V_sy = V.sy |0;
 408 |       var xy_stride = this.stride |0;
 409 | 
 410 |       for(var d=0;d<this.out_depth;d++) {
 411 |         var f = this.filters[d];
 412 |         var x = -this.pad |0;
 413 |         var y = -this.pad |0;
 414 |         for(var ay=0; ay<this.out_sy; y+=xy_stride,ay++) {  // xy_stride
 415 |           x = -this.pad |0;
 416 |           for(var ax=0; ax<this.out_sx; x+=xy_stride,ax++) {  // xy_stride
 417 | 
 418 |             // convolve centered at this particular location
 419 |             var a = 0.0;
 420 |             for(var fy=0;fy<f.sy;fy++) {
 421 |               var oy = y+fy; // coordinates in the original input array coordinates
 422 |               for(var fx=0;fx<f.sx;fx++) {
 423 |                 var ox = x+fx;
 424 |                 if(oy>=0 && oy<V_sy && ox>=0 && ox<V_sx) {
 425 |                   for(var fd=0;fd<f.depth;fd++) {
 426 |                     // avoid function call overhead (x2) for efficiency, compromise modularity :(
 427 |                     a += f.w[((f.sx * fy)+fx)*f.depth+fd] * V.w[((V_sx * oy)+ox)*V.depth+fd];
 428 |                   }
 429 |                 }
 430 |               }
 431 |             }
 432 |             a += this.biases.w[d];
 433 |             A.set(ax, ay, d, a);
 434 |           }
 435 |         }
 436 |       }
 437 |       this.out_act = A;
 438 |       return this.out_act;
 439 |     },
 440 |     backward: function() {
 441 | 
 442 |       var V = this.in_act;
 443 |       V.dw = global.zeros(V.w.length); // zero out gradient wrt bottom data, we're about to fill it
 444 | 
 445 |       var V_sx = V.sx |0;
 446 |       var V_sy = V.sy |0;
 447 |       var xy_stride = this.stride |0;
 448 | 
 449 |       for(var d=0;d<this.out_depth;d++) {
 450 |         var f = this.filters[d];
 451 |         var x = -this.pad |0;
 452 |         var y = -this.pad |0;
 453 |         for(var ay=0; ay<this.out_sy; y+=xy_stride,ay++) {  // xy_stride
 454 |           x = -this.pad |0;
 455 |           for(var ax=0; ax<this.out_sx; x+=xy_stride,ax++) {  // xy_stride
 456 | 
 457 |             // convolve centered at this particular location
 458 |             var chain_grad = this.out_act.get_grad(ax,ay,d); // gradient from above, from chain rule
 459 |             for(var fy=0;fy<f.sy;fy++) {
 460 |               var oy = y+fy; // coordinates in the original input array coordinates
 461 |               for(var fx=0;fx<f.sx;fx++) {
 462 |                 var ox = x+fx;
 463 |                 if(oy>=0 && oy<V_sy && ox>=0 && ox<V_sx) {
 464 |                   for(var fd=0;fd<f.depth;fd++) {
 465 |                     // avoid function call overhead (x2) for efficiency, compromise modularity :(
 466 |                     var ix1 = ((V_sx * oy)+ox)*V.depth+fd;
 467 |                     var ix2 = ((f.sx * fy)+fx)*f.depth+fd;
 468 |                     f.dw[ix2] += V.w[ix1]*chain_grad;
 469 |                     V.dw[ix1] += f.w[ix2]*chain_grad;
 470 |                   }
 471 |                 }
 472 |               }
 473 |             }
 474 |             this.biases.dw[d] += chain_grad;
 475 |           }
 476 |         }
 477 |       }
 478 |     },
 479 |     getParamsAndGrads: function() {
 480 |       var response = [];
 481 |       for(var i=0;i<this.out_depth;i++) {
 482 |         response.push({params: this.filters[i].w, grads: this.filters[i].dw, l2_decay_mul: this.l2_decay_mul, l1_decay_mul: this.l1_decay_mul});
 483 |       }
 484 |       response.push({params: this.biases.w, grads: this.biases.dw, l1_decay_mul: 0.0, l2_decay_mul: 0.0});
 485 |       return response;
 486 |     },
 487 |     toJSON: function() {
 488 |       var json = {};
 489 |       json.sx = this.sx; // filter size in x, y dims
 490 |       json.sy = this.sy;
 491 |       json.stride = this.stride;
 492 |       json.in_depth = this.in_depth;
 493 |       json.out_depth = this.out_depth;
 494 |       json.out_sx = this.out_sx;
 495 |       json.out_sy = this.out_sy;
 496 |       json.layer_type = this.layer_type;
 497 |       json.l1_decay_mul = this.l1_decay_mul;
 498 |       json.l2_decay_mul = this.l2_decay_mul;
 499 |       json.pad = this.pad;
 500 |       json.filters = [];
 501 |       for(var i=0;i<this.filters.length;i++) {
 502 |         json.filters.push(this.filters[i].toJSON());
 503 |       }
 504 |       json.biases = this.biases.toJSON();
 505 |       return json;
 506 |     },
 507 |     fromJSON: function(json) {
 508 |       this.out_depth = json.out_depth;
 509 |       this.out_sx = json.out_sx;
 510 |       this.out_sy = json.out_sy;
 511 |       this.layer_type = json.layer_type;
 512 |       this.sx = json.sx; // filter size in x, y dims
 513 |       this.sy = json.sy;
 514 |       this.stride = json.stride;
 515 |       this.in_depth = json.in_depth; // depth of input volume
 516 |       this.filters = [];
 517 |       this.l1_decay_mul = typeof json.l1_decay_mul !== 'undefined' ? json.l1_decay_mul : 1.0;
 518 |       this.l2_decay_mul = typeof json.l2_decay_mul !== 'undefined' ? json.l2_decay_mul : 1.0;
 519 |       this.pad = typeof json.pad !== 'undefined' ? json.pad : 0;
 520 |       for(var i=0;i<json.filters.length;i++) {
 521 |         var v = new Vol(0,0,0,0);
 522 |         v.fromJSON(json.filters[i]);
 523 |         this.filters.push(v);
 524 |       }
 525 |       this.biases = new Vol(0,0,0,0);
 526 |       this.biases.fromJSON(json.biases);
 527 |     }
 528 |   }
 529 | 
 530 |   var FullyConnLayer = function(opt) {
 531 |     var opt = opt || {};
 532 | 
 533 |     // required
 534 |     // ok fine we will allow 'filters' as the word as well
 535 |     this.out_depth = typeof opt.num_neurons !== 'undefined' ? opt.num_neurons : opt.filters;
 536 | 
 537 |     // optional 
 538 |     this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0;
 539 |     this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0;
 540 | 
 541 |     // computed
 542 |     this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
 543 |     this.out_sx = 1;
 544 |     this.out_sy = 1;
 545 |     this.layer_type = 'fc';
 546 | 
 547 |     // initializations
 548 |     var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0;
 549 |     this.filters = [];
 550 |     for(var i=0;i<this.out_depth ;i++) { this.filters.push(new Vol(1, 1, this.num_inputs)); }
 551 |     this.biases = new Vol(1, 1, this.out_depth, bias);
 552 |   }
 553 | 
 554 |   FullyConnLayer.prototype = {
 555 |     forward: function(V, is_training) {
 556 |       this.in_act = V;
 557 |       var A = new Vol(1, 1, this.out_depth, 0.0);
 558 |       var Vw = V.w;
 559 |       for(var i=0;i<this.out_depth;i++) {
 560 |         var a = 0.0;
 561 |         var wi = this.filters[i].w;
 562 |         for(var d=0;d<this.num_inputs;d++) {
 563 |           a += Vw[d] * wi[d]; // for efficiency use Vols directly for now
 564 |         }
 565 |         a += this.biases.w[i];
 566 |         A.w[i] = a;
 567 |       }
 568 |       this.out_act = A;
 569 |       return this.out_act;
 570 |     },
 571 |     backward: function() {
 572 |       var V = this.in_act;
 573 |       V.dw = global.zeros(V.w.length); // zero out the gradient in input Vol
 574 |       
 575 |       // compute gradient wrt weights and data
 576 |       for(var i=0;i<this.out_depth;i++) {
 577 |         var tfi = this.filters[i];
 578 |         var chain_grad = this.out_act.dw[i];
 579 |         for(var d=0;d<this.num_inputs;d++) {
 580 |           V.dw[d] += tfi.w[d]*chain_grad; // grad wrt input data
 581 |           tfi.dw[d] += V.w[d]*chain_grad; // grad wrt params
 582 |         }
 583 |         this.biases.dw[i] += chain_grad;
 584 |       }
 585 |     },
 586 |     getParamsAndGrads: function() {
 587 |       var response = [];
 588 |       for(var i=0;i<this.out_depth;i++) {
 589 |         response.push({params: this.filters[i].w, grads: this.filters[i].dw, l1_decay_mul: this.l1_decay_mul, l2_decay_mul: this.l2_decay_mul});
 590 |       }
 591 |       response.push({params: this.biases.w, grads: this.biases.dw, l1_decay_mul: 0.0, l2_decay_mul: 0.0});
 592 |       return response;
 593 |     },
 594 |     toJSON: function() {
 595 |       var json = {};
 596 |       json.out_depth = this.out_depth;
 597 |       json.out_sx = this.out_sx;
 598 |       json.out_sy = this.out_sy;
 599 |       json.layer_type = this.layer_type;
 600 |       json.num_inputs = this.num_inputs;
 601 |       json.l1_decay_mul = this.l1_decay_mul;
 602 |       json.l2_decay_mul = this.l2_decay_mul;
 603 |       json.filters = [];
 604 |       for(var i=0;i<this.filters.length;i++) {
 605 |         json.filters.push(this.filters[i].toJSON());
 606 |       }
 607 |       json.biases = this.biases.toJSON();
 608 |       return json;
 609 |     },
 610 |     fromJSON: function(json) {
 611 |       this.out_depth = json.out_depth;
 612 |       this.out_sx = json.out_sx;
 613 |       this.out_sy = json.out_sy;
 614 |       this.layer_type = json.layer_type;
 615 |       this.num_inputs = json.num_inputs;
 616 |       this.l1_decay_mul = typeof json.l1_decay_mul !== 'undefined' ? json.l1_decay_mul : 1.0;
 617 |       this.l2_decay_mul = typeof json.l2_decay_mul !== 'undefined' ? json.l2_decay_mul : 1.0;
 618 |       this.filters = [];
 619 |       for(var i=0;i<json.filters.length;i++) {
 620 |         var v = new Vol(0,0,0,0);
 621 |         v.fromJSON(json.filters[i]);
 622 |         this.filters.push(v);
 623 |       }
 624 |       this.biases = new Vol(0,0,0,0);
 625 |       this.biases.fromJSON(json.biases);
 626 |     }
 627 |   }
 628 | 
 629 |   global.ConvLayer = ConvLayer;
 630 |   global.FullyConnLayer = FullyConnLayer;
 631 |   
 632 | })(convnetjs);
 633 | (function(global) {
 634 |   "use strict";
 635 |   var Vol = global.Vol; // convenience
 636 |   
 637 |   var PoolLayer = function(opt) {
 638 | 
 639 |     var opt = opt || {};
 640 | 
 641 |     // required
 642 |     this.sx = opt.sx; // filter size
 643 |     this.in_depth = opt.in_depth;
 644 |     this.in_sx = opt.in_sx;
 645 |     this.in_sy = opt.in_sy;
 646 | 
 647 |     // optional
 648 |     this.sy = typeof opt.sy !== 'undefined' ? opt.sy : this.sx;
 649 |     this.stride = typeof opt.stride !== 'undefined' ? opt.stride : 2;
 650 |     this.pad = typeof opt.pad !== 'undefined' ? opt.pad : 0; // amount of 0 padding to add around borders of input volume
 651 | 
 652 |     // computed
 653 |     this.out_depth = this.in_depth;
 654 |     this.out_sx = Math.floor((this.in_sx + this.pad * 2 - this.sx) / this.stride + 1);
 655 |     this.out_sy = Math.floor((this.in_sy + this.pad * 2 - this.sy) / this.stride + 1);
 656 |     this.layer_type = 'pool';
 657 |     // store switches for x,y coordinates for where the max comes from, for each output neuron
 658 |     this.switchx = global.zeros(this.out_sx*this.out_sy*this.out_depth);
 659 |     this.switchy = global.zeros(this.out_sx*this.out_sy*this.out_depth);
 660 |   }
 661 | 
 662 |   PoolLayer.prototype = {
 663 |     forward: function(V, is_training) {
 664 |       this.in_act = V;
 665 | 
 666 |       var A = new Vol(this.out_sx, this.out_sy, this.out_depth, 0.0);
 667 |       
 668 |       var n=0; // a counter for switches
 669 |       for(var d=0;d<this.out_depth;d++) {
 670 |         var x = -this.pad;
 671 |         var y = -this.pad;
 672 |         for(var ax=0; ax<this.out_sx; x+=this.stride,ax++) {
 673 |           y = -this.pad;
 674 |           for(var ay=0; ay<this.out_sy; y+=this.stride,ay++) {
 675 | 
 676 |             // convolve centered at this particular location
 677 |             var a = -99999; // hopefully small enough ;\
 678 |             var winx=-1,winy=-1;
 679 |             for(var fx=0;fx<this.sx;fx++) {
 680 |               for(var fy=0;fy<this.sy;fy++) {
 681 |                 var oy = y+fy;
 682 |                 var ox = x+fx;
 683 |                 if(oy>=0 && oy<V.sy && ox>=0 && ox<V.sx) {
 684 |                   var v = V.get(ox, oy, d);
 685 |                   // perform max pooling and store pointers to where
 686 |                   // the max came from. This will speed up backprop 
 687 |                   // and can help make nice visualizations in future
 688 |                   if(v > a) { a = v; winx=ox; winy=oy;}
 689 |                 }
 690 |               }
 691 |             }
 692 |             this.switchx[n] = winx;
 693 |             this.switchy[n] = winy;
 694 |             n++;
 695 |             A.set(ax, ay, d, a);
 696 |           }
 697 |         }
 698 |       }
 699 |       this.out_act = A;
 700 |       return this.out_act;
 701 |     },
 702 |     backward: function() { 
 703 |       // pooling layers have no parameters, so simply compute 
 704 |       // gradient wrt data here
 705 |       var V = this.in_act;
 706 |       V.dw = global.zeros(V.w.length); // zero out gradient wrt data
 707 |       var A = this.out_act; // computed in forward pass 
 708 | 
 709 |       var n = 0;
 710 |       for(var d=0;d<this.out_depth;d++) {
 711 |         var x = -this.pad;
 712 |         var y = -this.pad;
 713 |         for(var ax=0; ax<this.out_sx; x+=this.stride,ax++) {
 714 |           y = -this.pad;
 715 |           for(var ay=0; ay<this.out_sy; y+=this.stride,ay++) {
 716 | 
 717 |             var chain_grad = this.out_act.get_grad(ax,ay,d);
 718 |             V.add_grad(this.switchx[n], this.switchy[n], d, chain_grad);
 719 |             n++;
 720 | 
 721 |           }
 722 |         }
 723 |       }
 724 |     },
 725 |     getParamsAndGrads: function() {
 726 |       return [];
 727 |     },
 728 |     toJSON: function() {
 729 |       var json = {};
 730 |       json.sx = this.sx;
 731 |       json.sy = this.sy;
 732 |       json.stride = this.stride;
 733 |       json.in_depth = this.in_depth;
 734 |       json.out_depth = this.out_depth;
 735 |       json.out_sx = this.out_sx;
 736 |       json.out_sy = this.out_sy;
 737 |       json.layer_type = this.layer_type;
 738 |       json.pad = this.pad;
 739 |       return json;
 740 |     },
 741 |     fromJSON: function(json) {
 742 |       this.out_depth = json.out_depth;
 743 |       this.out_sx = json.out_sx;
 744 |       this.out_sy = json.out_sy;
 745 |       this.layer_type = json.layer_type;
 746 |       this.sx = json.sx;
 747 |       this.sy = json.sy;
 748 |       this.stride = json.stride;
 749 |       this.in_depth = json.in_depth;
 750 |       this.pad = typeof json.pad !== 'undefined' ? json.pad : 0; // backwards compatibility
 751 |       this.switchx = global.zeros(this.out_sx*this.out_sy*this.out_depth); // need to re-init these appropriately
 752 |       this.switchy = global.zeros(this.out_sx*this.out_sy*this.out_depth);
 753 |     }
 754 |   }
 755 | 
 756 |   global.PoolLayer = PoolLayer;
 757 | 
 758 | })(convnetjs);
 759 | 
 760 | (function(global) {
 761 |   "use strict";
 762 |   var Vol = global.Vol; // convenience
 763 |   var getopt = global.getopt;
 764 | 
 765 |   var InputLayer = function(opt) {
 766 |     var opt = opt || {};
 767 | 
 768 |     // required: depth
 769 |     this.out_depth = getopt(opt, ['out_depth', 'depth'], 0);
 770 | 
 771 |     // optional: default these dimensions to 1
 772 |     this.out_sx = getopt(opt, ['out_sx', 'sx', 'width'], 1);
 773 |     this.out_sy = getopt(opt, ['out_sy', 'sy', 'height'], 1);
 774 |     
 775 |     // computed
 776 |     this.layer_type = 'input';
 777 |   }
 778 |   InputLayer.prototype = {
 779 |     forward: function(V, is_training) {
 780 |       this.in_act = V;
 781 |       this.out_act = V;
 782 |       return this.out_act; // simply identity function for now
 783 |     },
 784 |     backward: function() { },
 785 |     getParamsAndGrads: function() {
 786 |       return [];
 787 |     },
 788 |     toJSON: function() {
 789 |       var json = {};
 790 |       json.out_depth = this.out_depth;
 791 |       json.out_sx = this.out_sx;
 792 |       json.out_sy = this.out_sy;
 793 |       json.layer_type = this.layer_type;
 794 |       return json;
 795 |     },
 796 |     fromJSON: function(json) {
 797 |       this.out_depth = json.out_depth;
 798 |       this.out_sx = json.out_sx;
 799 |       this.out_sy = json.out_sy;
 800 |       this.layer_type = json.layer_type; 
 801 |     }
 802 |   }
 803 | 
 804 |   global.InputLayer = InputLayer;
 805 | })(convnetjs);
 806 | (function(global) {
 807 |   "use strict";
 808 |   var Vol = global.Vol; // convenience
 809 |   
 810 |   // Layers that implement a loss. Currently these are the layers that 
 811 |   // can initiate a backward() pass. In future we probably want a more 
 812 |   // flexible system that can accomodate multiple losses to do multi-task
 813 |   // learning, and stuff like that. But for now, one of the layers in this
 814 |   // file must be the final layer in a Net.
 815 | 
 816 |   // This is a classifier, with N discrete classes from 0 to N-1
 817 |   // it gets a stream of N incoming numbers and computes the softmax
 818 |   // function (exponentiate and normalize to sum to 1 as probabilities should)
 819 |   var SoftmaxLayer = function(opt) {
 820 |     var opt = opt || {};
 821 | 
 822 |     // computed
 823 |     this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
 824 |     this.out_depth = this.num_inputs;
 825 |     this.out_sx = 1;
 826 |     this.out_sy = 1;
 827 |     this.layer_type = 'softmax';
 828 |   }
 829 | 
 830 |   SoftmaxLayer.prototype = {
 831 |     forward: function(V, is_training) {
 832 |       this.in_act = V;
 833 | 
 834 |       var A = new Vol(1, 1, this.out_depth, 0.0);
 835 | 
 836 |       // compute max activation
 837 |       var as = V.w;
 838 |       var amax = V.w[0];
 839 |       for(var i=1;i<this.out_depth;i++) {
 840 |         if(as[i] > amax) amax = as[i];
 841 |       }
 842 | 
 843 |       // compute exponentials (carefully to not blow up)
 844 |       var es = global.zeros(this.out_depth);
 845 |       var esum = 0.0;
 846 |       for(var i=0;i<this.out_depth;i++) {
 847 |         var e = Math.exp(as[i] - amax);
 848 |         esum += e;
 849 |         es[i] = e;
 850 |       }
 851 | 
 852 |       // normalize and output to sum to one
 853 |       for(var i=0;i<this.out_depth;i++) {
 854 |         es[i] /= esum;
 855 |         A.w[i] = es[i];
 856 |       }
 857 | 
 858 |       this.es = es; // save these for backprop
 859 |       this.out_act = A;
 860 |       return this.out_act;
 861 |     },
 862 |     backward: function(y) {
 863 | 
 864 |       // compute and accumulate gradient wrt weights and bias of this layer
 865 |       var x = this.in_act;
 866 |       x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
 867 | 
 868 |       for(var i=0;i<this.out_depth;i++) {
 869 |         var indicator = i === y ? 1.0 : 0.0;
 870 |         var mul = -(indicator - this.es[i]);
 871 |         x.dw[i] = mul;
 872 |       }
 873 | 
 874 |       // loss is the class negative log likelihood
 875 |       return -Math.log(this.es[y]);
 876 |     },
 877 |     getParamsAndGrads: function() { 
 878 |       return [];
 879 |     },
 880 |     toJSON: function() {
 881 |       var json = {};
 882 |       json.out_depth = this.out_depth;
 883 |       json.out_sx = this.out_sx;
 884 |       json.out_sy = this.out_sy;
 885 |       json.layer_type = this.layer_type;
 886 |       json.num_inputs = this.num_inputs;
 887 |       return json;
 888 |     },
 889 |     fromJSON: function(json) {
 890 |       this.out_depth = json.out_depth;
 891 |       this.out_sx = json.out_sx;
 892 |       this.out_sy = json.out_sy;
 893 |       this.layer_type = json.layer_type;
 894 |       this.num_inputs = json.num_inputs;
 895 |     }
 896 |   }
 897 | 
 898 |   // implements an L2 regression cost layer,
 899 |   // so penalizes \sum_i(||x_i - y_i||^2), where x is its input
 900 |   // and y is the user-provided array of "correct" values.
 901 |   var RegressionLayer = function(opt) {
 902 |     var opt = opt || {};
 903 | 
 904 |     // computed
 905 |     this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
 906 |     this.out_depth = this.num_inputs;
 907 |     this.out_sx = 1;
 908 |     this.out_sy = 1;
 909 |     this.layer_type = 'regression';
 910 |   }
 911 | 
 912 |   RegressionLayer.prototype = {
 913 |     forward: function(V, is_training) {
 914 |       this.in_act = V;
 915 |       this.out_act = V;
 916 |       return V; // identity function
 917 |     },
 918 |     // y is a list here of size num_inputs
 919 |     // or it can be a number if only one value is regressed
 920 |     // or it can be a struct {dim: i, val: x} where we only want to 
 921 |     // regress on dimension i and asking it to have value x
 922 |     backward: function(y) { 
 923 | 
 924 |       // compute and accumulate gradient wrt weights and bias of this layer
 925 |       var x = this.in_act;
 926 |       x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
 927 |       var loss = 0.0;
 928 |       if(y instanceof Array || y instanceof Float64Array) {
 929 |         for(var i=0;i<this.out_depth;i++) {
 930 |           var dy = x.w[i] - y[i];
 931 |           x.dw[i] = dy;
 932 |           loss += 0.5*dy*dy;
 933 |         }
 934 |       } else if(typeof y === 'number') {
 935 |         // lets hope that only one number is being regressed
 936 |         var dy = x.w[0] - y;
 937 |         x.dw[0] = dy;
 938 |         loss += 0.5*dy*dy;
 939 |       } else {
 940 |         // assume it is a struct with entries .dim and .val
 941 |         // and we pass gradient only along dimension dim to be equal to val
 942 |         var i = y.dim;
 943 |         var yi = y.val;
 944 |         var dy = x.w[i] - yi;
 945 |         x.dw[i] = dy;
 946 |         loss += 0.5*dy*dy;
 947 |       }
 948 |       return loss;
 949 |     },
 950 |     getParamsAndGrads: function() { 
 951 |       return [];
 952 |     },
 953 |     toJSON: function() {
 954 |       var json = {};
 955 |       json.out_depth = this.out_depth;
 956 |       json.out_sx = this.out_sx;
 957 |       json.out_sy = this.out_sy;
 958 |       json.layer_type = this.layer_type;
 959 |       json.num_inputs = this.num_inputs;
 960 |       return json;
 961 |     },
 962 |     fromJSON: function(json) {
 963 |       this.out_depth = json.out_depth;
 964 |       this.out_sx = json.out_sx;
 965 |       this.out_sy = json.out_sy;
 966 |       this.layer_type = json.layer_type;
 967 |       this.num_inputs = json.num_inputs;
 968 |     }
 969 |   }
 970 | 
 971 |   var SVMLayer = function(opt) {
 972 |     var opt = opt || {};
 973 | 
 974 |     // computed
 975 |     this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
 976 |     this.out_depth = this.num_inputs;
 977 |     this.out_sx = 1;
 978 |     this.out_sy = 1;
 979 |     this.layer_type = 'svm';
 980 |   }
 981 | 
 982 |   SVMLayer.prototype = {
 983 |     forward: function(V, is_training) {
 984 |       this.in_act = V;
 985 |       this.out_act = V; // nothing to do, output raw scores
 986 |       return V;
 987 |     },
 988 |     backward: function(y) {
 989 | 
 990 |       // compute and accumulate gradient wrt weights and bias of this layer
 991 |       var x = this.in_act;
 992 |       x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
 993 | 
 994 |       // we're using structured loss here, which means that the score
 995 |       // of the ground truth should be higher than the score of any other 
 996 |       // class, by a margin
 997 |       var yscore = x.w[y]; // score of ground truth
 998 |       var margin = 1.0;
 999 |       var loss = 0.0;
1000 |       for(var i=0;i<this.out_depth;i++) {
1001 |         if(y === i) { continue; }
1002 |         var ydiff = -yscore + x.w[i] + margin;
1003 |         if(ydiff > 0) {
1004 |           // violating dimension, apply loss
1005 |           x.dw[i] += 1;
1006 |           x.dw[y] -= 1;
1007 |           loss += ydiff;
1008 |         }
1009 |       }
1010 | 
1011 |       return loss;
1012 |     },
1013 |     getParamsAndGrads: function() { 
1014 |       return [];
1015 |     },
1016 |     toJSON: function() {
1017 |       var json = {};
1018 |       json.out_depth = this.out_depth;
1019 |       json.out_sx = this.out_sx;
1020 |       json.out_sy = this.out_sy;
1021 |       json.layer_type = this.layer_type;
1022 |       json.num_inputs = this.num_inputs;
1023 |       return json;
1024 |     },
1025 |     fromJSON: function(json) {
1026 |       this.out_depth = json.out_depth;
1027 |       this.out_sx = json.out_sx;
1028 |       this.out_sy = json.out_sy;
1029 |       this.layer_type = json.layer_type;
1030 |       this.num_inputs = json.num_inputs;
1031 |     }
1032 |   }
1033 |   
1034 |   global.RegressionLayer = RegressionLayer;
1035 |   global.SoftmaxLayer = SoftmaxLayer;
1036 |   global.SVMLayer = SVMLayer;
1037 | 
1038 | })(convnetjs);
1039 | 
1040 | (function(global) {
1041 |   "use strict";
1042 |   var Vol = global.Vol; // convenience
1043 |   
1044 |   // Implements ReLU nonlinearity elementwise
1045 |   // x -> max(0, x)
1046 |   // the output is in [0, inf)
1047 |   var ReluLayer = function(opt) {
1048 |     var opt = opt || {};
1049 | 
1050 |     // computed
1051 |     this.out_sx = opt.in_sx;
1052 |     this.out_sy = opt.in_sy;
1053 |     this.out_depth = opt.in_depth;
1054 |     this.layer_type = 'relu';
1055 |   }
1056 |   ReluLayer.prototype = {
1057 |     forward: function(V, is_training) {
1058 |       this.in_act = V;
1059 |       var V2 = V.clone();
1060 |       var N = V.w.length;
1061 |       var V2w = V2.w;
1062 |       for(var i=0;i<N;i++) { 
1063 |         if(V2w[i] < 0) V2w[i] = 0; // threshold at 0
1064 |       }
1065 |       this.out_act = V2;
1066 |       return this.out_act;
1067 |     },
1068 |     backward: function() {
1069 |       var V = this.in_act; // we need to set dw of this
1070 |       var V2 = this.out_act;
1071 |       var N = V.w.length;
1072 |       V.dw = global.zeros(N); // zero out gradient wrt data
1073 |       for(var i=0;i<N;i++) {
1074 |         if(V2.w[i] <= 0) V.dw[i] = 0; // threshold
1075 |         else V.dw[i] = V2.dw[i];
1076 |       }
1077 |     },
1078 |     getParamsAndGrads: function() {
1079 |       return [];
1080 |     },
1081 |     toJSON: function() {
1082 |       var json = {};
1083 |       json.out_depth = this.out_depth;
1084 |       json.out_sx = this.out_sx;
1085 |       json.out_sy = this.out_sy;
1086 |       json.layer_type = this.layer_type;
1087 |       return json;
1088 |     },
1089 |     fromJSON: function(json) {
1090 |       this.out_depth = json.out_depth;
1091 |       this.out_sx = json.out_sx;
1092 |       this.out_sy = json.out_sy;
1093 |       this.layer_type = json.layer_type; 
1094 |     }
1095 |   }
1096 | 
1097 |   // Implements Sigmoid nnonlinearity elementwise
1098 |   // x -> 1/(1+e^(-x))
1099 |   // so the output is between 0 and 1.
1100 |   var SigmoidLayer = function(opt) {
1101 |     var opt = opt || {};
1102 | 
1103 |     // computed
1104 |     this.out_sx = opt.in_sx;
1105 |     this.out_sy = opt.in_sy;
1106 |     this.out_depth = opt.in_depth;
1107 |     this.layer_type = 'sigmoid';
1108 |   }
1109 |   SigmoidLayer.prototype = {
1110 |     forward: function(V, is_training) {
1111 |       this.in_act = V;
1112 |       var V2 = V.cloneAndZero();
1113 |       var N = V.w.length;
1114 |       var V2w = V2.w;
1115 |       var Vw = V.w;
1116 |       for(var i=0;i<N;i++) { 
1117 |         V2w[i] = 1.0/(1.0+Math.exp(-Vw[i]));
1118 |       }
1119 |       this.out_act = V2;
1120 |       return this.out_act;
1121 |     },
1122 |     backward: function() {
1123 |       var V = this.in_act; // we need to set dw of this
1124 |       var V2 = this.out_act;
1125 |       var N = V.w.length;
1126 |       V.dw = global.zeros(N); // zero out gradient wrt data
1127 |       for(var i=0;i<N;i++) {
1128 |         var v2wi = V2.w[i];
1129 |         V.dw[i] =  v2wi * (1.0 - v2wi) * V2.dw[i];
1130 |       }
1131 |     },
1132 |     getParamsAndGrads: function() {
1133 |       return [];
1134 |     },
1135 |     toJSON: function() {
1136 |       var json = {};
1137 |       json.out_depth = this.out_depth;
1138 |       json.out_sx = this.out_sx;
1139 |       json.out_sy = this.out_sy;
1140 |       json.layer_type = this.layer_type;
1141 |       return json;
1142 |     },
1143 |     fromJSON: function(json) {
1144 |       this.out_depth = json.out_depth;
1145 |       this.out_sx = json.out_sx;
1146 |       this.out_sy = json.out_sy;
1147 |       this.layer_type = json.layer_type; 
1148 |     }
1149 |   }
1150 | 
1151 |   // Implements Maxout nnonlinearity that computes
1152 |   // x -> max(x)
1153 |   // where x is a vector of size group_size. Ideally of course,
1154 |   // the input size should be exactly divisible by group_size
1155 |   var MaxoutLayer = function(opt) {
1156 |     var opt = opt || {};
1157 | 
1158 |     // required
1159 |     this.group_size = typeof opt.group_size !== 'undefined' ? opt.group_size : 2;
1160 | 
1161 |     // computed
1162 |     this.out_sx = opt.in_sx;
1163 |     this.out_sy = opt.in_sy;
1164 |     this.out_depth = Math.floor(opt.in_depth / this.group_size);
1165 |     this.layer_type = 'maxout';
1166 | 
1167 |     this.switches = global.zeros(this.out_sx*this.out_sy*this.out_depth); // useful for backprop
1168 |   }
1169 |   MaxoutLayer.prototype = {
1170 |     forward: function(V, is_training) {
1171 |       this.in_act = V;
1172 |       var N = this.out_depth; 
1173 |       var V2 = new Vol(this.out_sx, this.out_sy, this.out_depth, 0.0);
1174 | 
1175 |       // optimization branch. If we're operating on 1D arrays we dont have
1176 |       // to worry about keeping track of x,y,d coordinates inside
1177 |       // input volumes. In convnets we do :(
1178 |       if(this.out_sx === 1 && this.out_sy === 1) {
1179 |         for(var i=0;i<N;i++) {
1180 |           var ix = i * this.group_size; // base index offset
1181 |           var a = V.w[ix];
1182 |           var ai = 0;
1183 |           for(var j=1;j<this.group_size;j++) {
1184 |             var a2 = V.w[ix+j];
1185 |             if(a2 > a) {
1186 |               a = a2;
1187 |               ai = j;
1188 |             }
1189 |           }
1190 |           V2.w[i] = a;
1191 |           this.switches[i] = ix + ai;
1192 |         }
1193 |       } else {
1194 |         var n=0; // counter for switches
1195 |         for(var x=0;x<V.sx;x++) {
1196 |           for(var y=0;y<V.sy;y++) {
1197 |             for(var i=0;i<N;i++) {
1198 |               var ix = i * this.group_size;
1199 |               var a = V.get(x, y, ix);
1200 |               var ai = 0;
1201 |               for(var j=1;j<this.group_size;j++) {
1202 |                 var a2 = V.get(x, y, ix+j);
1203 |                 if(a2 > a) {
1204 |                   a = a2;
1205 |                   ai = j;
1206 |                 }
1207 |               }
1208 |               V2.set(x,y,i,a);
1209 |               this.switches[n] = ix + ai;
1210 |               n++;
1211 |             }
1212 |           }
1213 |         }
1214 | 
1215 |       }
1216 |       this.out_act = V2;
1217 |       return this.out_act;
1218 |     },
1219 |     backward: function() {
1220 |       var V = this.in_act; // we need to set dw of this
1221 |       var V2 = this.out_act;
1222 |       var N = this.out_depth;
1223 |       V.dw = global.zeros(V.w.length); // zero out gradient wrt data
1224 | 
1225 |       // pass the gradient through the appropriate switch
1226 |       if(this.out_sx === 1 && this.out_sy === 1) {
1227 |         for(var i=0;i<N;i++) {
1228 |           var chain_grad = V2.dw[i];
1229 |           V.dw[this.switches[i]] = chain_grad;
1230 |         }
1231 |       } else {
1232 |         // bleh okay, lets do this the hard way
1233 |         var n=0; // counter for switches
1234 |         for(var x=0;x<V2.sx;x++) {
1235 |           for(var y=0;y<V2.sy;y++) {
1236 |             for(var i=0;i<N;i++) {
1237 |               var chain_grad = V2.get_grad(x,y,i);
1238 |               V.set_grad(x,y,this.switches[n],chain_grad);
1239 |               n++;
1240 |             }
1241 |           }
1242 |         }
1243 |       }
1244 |     },
1245 |     getParamsAndGrads: function() {
1246 |       return [];
1247 |     },
1248 |     toJSON: function() {
1249 |       var json = {};
1250 |       json.out_depth = this.out_depth;
1251 |       json.out_sx = this.out_sx;
1252 |       json.out_sy = this.out_sy;
1253 |       json.layer_type = this.layer_type;
1254 |       json.group_size = this.group_size;
1255 |       return json;
1256 |     },
1257 |     fromJSON: function(json) {
1258 |       this.out_depth = json.out_depth;
1259 |       this.out_sx = json.out_sx;
1260 |       this.out_sy = json.out_sy;
1261 |       this.layer_type = json.layer_type; 
1262 |       this.group_size = json.group_size;
1263 |       this.switches = global.zeros(this.group_size);
1264 |     }
1265 |   }
1266 | 
1267 |   // a helper function, since tanh is not yet part of ECMAScript. Will be in v6.
1268 |   function tanh(x) {
1269 |     var y = Math.exp(2 * x);
1270 |     return (y - 1) / (y + 1);
1271 |   }
1272 |   // Implements Tanh nnonlinearity elementwise
1273 |   // x -> tanh(x) 
1274 |   // so the output is between -1 and 1.
1275 |   var TanhLayer = function(opt) {
1276 |     var opt = opt || {};
1277 | 
1278 |     // computed
1279 |     this.out_sx = opt.in_sx;
1280 |     this.out_sy = opt.in_sy;
1281 |     this.out_depth = opt.in_depth;
1282 |     this.layer_type = 'tanh';
1283 |   }
1284 |   TanhLayer.prototype = {
1285 |     forward: function(V, is_training) {
1286 |       this.in_act = V;
1287 |       var V2 = V.cloneAndZero();
1288 |       var N = V.w.length;
1289 |       for(var i=0;i<N;i++) { 
1290 |         V2.w[i] = tanh(V.w[i]);
1291 |       }
1292 |       this.out_act = V2;
1293 |       return this.out_act;
1294 |     },
1295 |     backward: function() {
1296 |       var V = this.in_act; // we need to set dw of this
1297 |       var V2 = this.out_act;
1298 |       var N = V.w.length;
1299 |       V.dw = global.zeros(N); // zero out gradient wrt data
1300 |       for(var i=0;i<N;i++) {
1301 |         var v2wi = V2.w[i];
1302 |         V.dw[i] = (1.0 - v2wi * v2wi) * V2.dw[i];
1303 |       }
1304 |     },
1305 |     getParamsAndGrads: function() {
1306 |       return [];
1307 |     },
1308 |     toJSON: function() {
1309 |       var json = {};
1310 |       json.out_depth = this.out_depth;
1311 |       json.out_sx = this.out_sx;
1312 |       json.out_sy = this.out_sy;
1313 |       json.layer_type = this.layer_type;
1314 |       return json;
1315 |     },
1316 |     fromJSON: function(json) {
1317 |       this.out_depth = json.out_depth;
1318 |       this.out_sx = json.out_sx;
1319 |       this.out_sy = json.out_sy;
1320 |       this.layer_type = json.layer_type; 
1321 |     }
1322 |   }
1323 |   
1324 |   global.TanhLayer = TanhLayer;
1325 |   global.MaxoutLayer = MaxoutLayer;
1326 |   global.ReluLayer = ReluLayer;
1327 |   global.SigmoidLayer = SigmoidLayer;
1328 | 
1329 | })(convnetjs);
1330 | 
1331 | (function(global) {
1332 |   "use strict";
1333 |   var Vol = global.Vol; // convenience
1334 | 
1335 |   // An inefficient dropout layer
1336 |   // Note this is not most efficient implementation since the layer before
1337 |   // computed all these activations and now we're just going to drop them :(
1338 |   // same goes for backward pass. Also, if we wanted to be efficient at test time
1339 |   // we could equivalently be clever and upscale during train and copy pointers during test
1340 |   // todo: make more efficient.
1341 |   var DropoutLayer = function(opt) {
1342 |     var opt = opt || {};
1343 | 
1344 |     // computed
1345 |     this.out_sx = opt.in_sx;
1346 |     this.out_sy = opt.in_sy;
1347 |     this.out_depth = opt.in_depth;
1348 |     this.layer_type = 'dropout';
1349 |     this.drop_prob = typeof opt.drop_prob !== 'undefined' ? opt.drop_prob : 0.5;
1350 |     this.dropped = global.zeros(this.out_sx*this.out_sy*this.out_depth);
1351 |   }
1352 |   DropoutLayer.prototype = {
1353 |     forward: function(V, is_training) {
1354 |       this.in_act = V;
1355 |       if(typeof(is_training)==='undefined') { is_training = false; } // default is prediction mode
1356 |       var V2 = V.clone();
1357 |       var N = V.w.length;
1358 |       if(is_training) {
1359 |         // do dropout
1360 |         for(var i=0;i<N;i++) {
1361 |           if(Math.random()<this.drop_prob) { V2.w[i]=0; this.dropped[i] = true; } // drop!
1362 |           else {this.dropped[i] = false;}
1363 |         }
1364 |       } else {
1365 |         // scale the activations during prediction
1366 |         for(var i=0;i<N;i++) { V2.w[i]*=this.drop_prob; }
1367 |       }
1368 |       this.out_act = V2;
1369 |       return this.out_act; // dummy identity function for now
1370 |     },
1371 |     backward: function() {
1372 |       var V = this.in_act; // we need to set dw of this
1373 |       var chain_grad = this.out_act;
1374 |       var N = V.w.length;
1375 |       V.dw = global.zeros(N); // zero out gradient wrt data
1376 |       for(var i=0;i<N;i++) {
1377 |         if(!(this.dropped[i])) { 
1378 |           V.dw[i] = chain_grad.dw[i]; // copy over the gradient
1379 |         }
1380 |       }
1381 |     },
1382 |     getParamsAndGrads: function() {
1383 |       return [];
1384 |     },
1385 |     toJSON: function() {
1386 |       var json = {};
1387 |       json.out_depth = this.out_depth;
1388 |       json.out_sx = this.out_sx;
1389 |       json.out_sy = this.out_sy;
1390 |       json.layer_type = this.layer_type;
1391 |       json.drop_prob = this.drop_prob;
1392 |       return json;
1393 |     },
1394 |     fromJSON: function(json) {
1395 |       this.out_depth = json.out_depth;
1396 |       this.out_sx = json.out_sx;
1397 |       this.out_sy = json.out_sy;
1398 |       this.layer_type = json.layer_type; 
1399 |       this.drop_prob = json.drop_prob;
1400 |     }
1401 |   }
1402 |   
1403 | 
1404 |   global.DropoutLayer = DropoutLayer;
1405 | })(convnetjs);
1406 | (function(global) {
1407 |   "use strict";
1408 |   var Vol = global.Vol; // convenience
1409 |   
1410 |   // a bit experimental layer for now. I think it works but I'm not 100%
1411 |   // the gradient check is a bit funky. I'll look into this a bit later.
1412 |   // Local Response Normalization in window, along depths of volumes
1413 |   var LocalResponseNormalizationLayer = function(opt) {
1414 |     var opt = opt || {};
1415 | 
1416 |     // required
1417 |     this.k = opt.k;
1418 |     this.n = opt.n;
1419 |     this.alpha = opt.alpha;
1420 |     this.beta = opt.beta;
1421 | 
1422 |     // computed
1423 |     this.out_sx = opt.in_sx;
1424 |     this.out_sy = opt.in_sy;
1425 |     this.out_depth = opt.in_depth;
1426 |     this.layer_type = 'lrn';
1427 | 
1428 |     // checks
1429 |     if(this.n%2 === 0) { console.log('WARNING n should be odd for LRN layer'); }
1430 |   }
1431 |   LocalResponseNormalizationLayer.prototype = {
1432 |     forward: function(V, is_training) {
1433 |       this.in_act = V;
1434 | 
1435 |       var A = V.cloneAndZero();
1436 |       this.S_cache_ = V.cloneAndZero();
1437 |       var n2 = Math.floor(this.n/2);
1438 |       for(var x=0;x<V.sx;x++) {
1439 |         for(var y=0;y<V.sy;y++) {
1440 |           for(var i=0;i<V.depth;i++) {
1441 | 
1442 |             var ai = V.get(x,y,i);
1443 | 
1444 |             // normalize in a window of size n
1445 |             var den = 0.0;
1446 |             for(var j=Math.max(0,i-n2);j<=Math.min(i+n2,V.depth-1);j++) {
1447 |               var aa = V.get(x,y,j);
1448 |               den += aa*aa;
1449 |             }
1450 |             den *= this.alpha / this.n;
1451 |             den += this.k;
1452 |             this.S_cache_.set(x,y,i,den); // will be useful for backprop
1453 |             den = Math.pow(den, this.beta);
1454 |             A.set(x,y,i,ai/den);
1455 |           }
1456 |         }
1457 |       }
1458 | 
1459 |       this.out_act = A;
1460 |       return this.out_act; // dummy identity function for now
1461 |     },
1462 |     backward: function() { 
1463 |       // evaluate gradient wrt data
1464 |       var V = this.in_act; // we need to set dw of this
1465 |       V.dw = global.zeros(V.w.length); // zero out gradient wrt data
1466 |       var A = this.out_act; // computed in forward pass 
1467 | 
1468 |       var n2 = Math.floor(this.n/2);
1469 |       for(var x=0;x<V.sx;x++) {
1470 |         for(var y=0;y<V.sy;y++) {
1471 |           for(var i=0;i<V.depth;i++) {
1472 | 
1473 |             var chain_grad = this.out_act.get_grad(x,y,i);
1474 |             var S = this.S_cache_.get(x,y,i);
1475 |             var SB = Math.pow(S, this.beta);
1476 |             var SB2 = SB*SB;
1477 | 
1478 |             // normalize in a window of size n
1479 |             for(var j=Math.max(0,i-n2);j<=Math.min(i+n2,V.depth-1);j++) {              
1480 |               var aj = V.get(x,y,j); 
1481 |               var g = -aj*this.beta*Math.pow(S,this.beta-1)*this.alpha/this.n*2*aj;
1482 |               if(j===i) g+= SB;
1483 |               g /= SB2;
1484 |               g *= chain_grad;
1485 |               V.add_grad(x,y,j,g);
1486 |             }
1487 | 
1488 |           }
1489 |         }
1490 |       }
1491 |     },
1492 |     getParamsAndGrads: function() { return []; },
1493 |     toJSON: function() {
1494 |       var json = {};
1495 |       json.k = this.k;
1496 |       json.n = this.n;
1497 |       json.alpha = this.alpha; // normalize by size
1498 |       json.beta = this.beta;
1499 |       json.out_sx = this.out_sx; 
1500 |       json.out_sy = this.out_sy;
1501 |       json.out_depth = this.out_depth;
1502 |       json.layer_type = this.layer_type;
1503 |       return json;
1504 |     },
1505 |     fromJSON: function(json) {
1506 |       this.k = json.k;
1507 |       this.n = json.n;
1508 |       this.alpha = json.alpha; // normalize by size
1509 |       this.beta = json.beta;
1510 |       this.out_sx = json.out_sx; 
1511 |       this.out_sy = json.out_sy;
1512 |       this.out_depth = json.out_depth;
1513 |       this.layer_type = json.layer_type;
1514 |     }
1515 |   }
1516 |   
1517 | 
1518 |   global.LocalResponseNormalizationLayer = LocalResponseNormalizationLayer;
1519 | })(convnetjs);
1520 | (function(global) {
1521 |   "use strict";
1522 |   var Vol = global.Vol; // convenience
1523 |   var assert = global.assert;
1524 | 
1525 |   // Net manages a set of layers
1526 |   // For now constraints: Simple linear order of layers, first layer input last layer a cost layer
1527 |   var Net = function(options) {
1528 |     this.layers = [];
1529 |   }
1530 | 
1531 |   Net.prototype = {
1532 |     
1533 |     // takes a list of layer definitions and creates the network layer objects
1534 |     makeLayers: function(defs) {
1535 | 
1536 |       // few checks
1537 |       assert(defs.length >= 2, 'Error! At least one input layer and one loss layer are required.');
1538 |       assert(defs[0].type === 'input', 'Error! First layer must be the input layer, to declare size of inputs');
1539 | 
1540 |       // desugar layer_defs for adding activation, dropout layers etc
1541 |       var desugar = function() {
1542 |         var new_defs = [];
1543 |         for(var i=0;i<defs.length;i++) {
1544 |           var def = defs[i];
1545 |           
1546 |           if(def.type==='softmax' || def.type==='svm') {
1547 |             // add an fc layer here, there is no reason the user should
1548 |             // have to worry about this and we almost always want to
1549 |             new_defs.push({type:'fc', num_neurons: def.num_classes});
1550 |           }
1551 | 
1552 |           if(def.type==='regression') {
1553 |             // add an fc layer here, there is no reason the user should
1554 |             // have to worry about this and we almost always want to
1555 |             new_defs.push({type:'fc', num_neurons: def.num_neurons});
1556 |           }
1557 | 
1558 |           if((def.type==='fc' || def.type==='conv') 
1559 |               && typeof(def.bias_pref) === 'undefined'){
1560 |             def.bias_pref = 0.0;
1561 |             if(typeof def.activation !== 'undefined' && def.activation === 'relu') {
1562 |               def.bias_pref = 0.1; // relus like a bit of positive bias to get gradients early
1563 |               // otherwise it's technically possible that a relu unit will never turn on (by chance)
1564 |               // and will never get any gradient and never contribute any computation. Dead relu.
1565 |             }
1566 |           }
1567 | 
1568 |           new_defs.push(def);
1569 | 
1570 |           if(typeof def.activation !== 'undefined') {
1571 |             if(def.activation==='relu') { new_defs.push({type:'relu'}); }
1572 |             else if (def.activation==='sigmoid') { new_defs.push({type:'sigmoid'}); }
1573 |             else if (def.activation==='tanh') { new_defs.push({type:'tanh'}); }
1574 |             else if (def.activation==='maxout') {
1575 |               // create maxout activation, and pass along group size, if provided
1576 |               var gs = def.group_size !== 'undefined' ? def.group_size : 2;
1577 |               new_defs.push({type:'maxout', group_size:gs});
1578 |             }
1579 |             else { console.log('ERROR unsupported activation ' + def.activation); }
1580 |           }
1581 |           if(typeof def.drop_prob !== 'undefined' && def.type !== 'dropout') {
1582 |             new_defs.push({type:'dropout', drop_prob: def.drop_prob});
1583 |           }
1584 | 
1585 |         }
1586 |         return new_defs;
1587 |       }
1588 |       defs = desugar(defs);
1589 | 
1590 |       // create the layers
1591 |       this.layers = [];
1592 |       for(var i=0;i<defs.length;i++) {
1593 |         var def = defs[i];
1594 |         if(i>0) {
1595 |           var prev = this.layers[i-1];
1596 |           def.in_sx = prev.out_sx;
1597 |           def.in_sy = prev.out_sy;
1598 |           def.in_depth = prev.out_depth;
1599 |         }
1600 | 
1601 |         switch(def.type) {
1602 |           case 'fc': this.layers.push(new global.FullyConnLayer(def)); break;
1603 |           case 'lrn': this.layers.push(new global.LocalResponseNormalizationLayer(def)); break;
1604 |           case 'dropout': this.layers.push(new global.DropoutLayer(def)); break;
1605 |           case 'input': this.layers.push(new global.InputLayer(def)); break;
1606 |           case 'softmax': this.layers.push(new global.SoftmaxLayer(def)); break;
1607 |           case 'regression': this.layers.push(new global.RegressionLayer(def)); break;
1608 |           case 'conv': this.layers.push(new global.ConvLayer(def)); break;
1609 |           case 'pool': this.layers.push(new global.PoolLayer(def)); break;
1610 |           case 'relu': this.layers.push(new global.ReluLayer(def)); break;
1611 |           case 'sigmoid': this.layers.push(new global.SigmoidLayer(def)); break;
1612 |           case 'tanh': this.layers.push(new global.TanhLayer(def)); break;
1613 |           case 'maxout': this.layers.push(new global.MaxoutLayer(def)); break;
1614 |           case 'svm': this.layers.push(new global.SVMLayer(def)); break;
1615 |           default: console.log('ERROR: UNRECOGNIZED LAYER TYPE: ' + def.type);
1616 |         }
1617 |       }
1618 |     },
1619 | 
1620 |     // forward prop the network. 
1621 |     // The trainer class passes is_training = true, but when this function is
1622 |     // called from outside (not from the trainer), it defaults to prediction mode
1623 |     forward: function(V, is_training) {
1624 |       if(typeof(is_training) === 'undefined') is_training = false;
1625 |       var act = this.layers[0].forward(V, is_training);
1626 |       for(var i=1;i<this.layers.length;i++) {
1627 |         act = this.layers[i].forward(act, is_training);
1628 |       }
1629 |       return act;
1630 |     },
1631 | 
1632 |     getCostLoss: function(V, y) {
1633 |       this.forward(V, false);
1634 |       var N = this.layers.length;
1635 |       var loss = this.layers[N-1].backward(y);
1636 |       return loss;
1637 |     },
1638 |     
1639 |     // backprop: compute gradients wrt all parameters
1640 |     backward: function(y) {
1641 |       var N = this.layers.length;
1642 |       var loss = this.layers[N-1].backward(y); // last layer assumed to be loss layer
1643 |       for(var i=N-2;i>=0;i--) { // first layer assumed input
1644 |         this.layers[i].backward();
1645 |       }
1646 |       return loss;
1647 |     },
1648 |     getParamsAndGrads: function() {
1649 |       // accumulate parameters and gradients for the entire network
1650 |       var response = [];
1651 |       for(var i=0;i<this.layers.length;i++) {
1652 |         var layer_reponse = this.layers[i].getParamsAndGrads();
1653 |         for(var j=0;j<layer_reponse.length;j++) {
1654 |           response.push(layer_reponse[j]);
1655 |         }
1656 |       }
1657 |       return response;
1658 |     },
1659 |     getPrediction: function() {
1660 |       // this is a convenience function for returning the argmax
1661 |       // prediction, assuming the last layer of the net is a softmax
1662 |       var S = this.layers[this.layers.length-1];
1663 |       assert(S.layer_type === 'softmax', 'getPrediction function assumes softmax as last layer of the net!');
1664 | 
1665 |       var p = S.out_act.w;
1666 |       var maxv = p[0];
1667 |       var maxi = 0;
1668 |       for(var i=1;i<p.length;i++) {
1669 |         if(p[i] > maxv) { maxv = p[i]; maxi = i;}
1670 |       }
1671 |       return maxi; // return index of the class with highest class probability
1672 |     },
1673 |     toJSON: function() {
1674 |       var json = {};
1675 |       json.layers = [];
1676 |       for(var i=0;i<this.layers.length;i++) {
1677 |         json.layers.push(this.layers[i].toJSON());
1678 |       }
1679 |       return json;
1680 |     },
1681 |     fromJSON: function(json) {
1682 |       this.layers = [];
1683 |       for(var i=0;i<json.layers.length;i++) {
1684 |         var Lj = json.layers[i]
1685 |         var t = Lj.layer_type;
1686 |         var L;
1687 |         if(t==='input') { L = new global.InputLayer(); }
1688 |         if(t==='relu') { L = new global.ReluLayer(); }
1689 |         if(t==='sigmoid') { L = new global.SigmoidLayer(); }
1690 |         if(t==='tanh') { L = new global.TanhLayer(); }
1691 |         if(t==='dropout') { L = new global.DropoutLayer(); }
1692 |         if(t==='conv') { L = new global.ConvLayer(); }
1693 |         if(t==='pool') { L = new global.PoolLayer(); }
1694 |         if(t==='lrn') { L = new global.LocalResponseNormalizationLayer(); }
1695 |         if(t==='softmax') { L = new global.SoftmaxLayer(); }
1696 |         if(t==='regression') { L = new global.RegressionLayer(); }
1697 |         if(t==='fc') { L = new global.FullyConnLayer(); }
1698 |         if(t==='maxout') { L = new global.MaxoutLayer(); }
1699 |         if(t==='svm') { L = new global.SVMLayer(); }
1700 |         L.fromJSON(Lj);
1701 |         this.layers.push(L);
1702 |       }
1703 |     }
1704 |   }
1705 |   
1706 |   global.Net = Net;
1707 | })(convnetjs);
1708 | (function(global) {
1709 |   "use strict";
1710 |   var Vol = global.Vol; // convenience
1711 | 
1712 |   var Trainer = function(net, options) {
1713 | 
1714 |     this.net = net;
1715 | 
1716 |     var options = options || {};
1717 |     this.learning_rate = typeof options.learning_rate !== 'undefined' ? options.learning_rate : 0.01;
1718 |     this.l1_decay = typeof options.l1_decay !== 'undefined' ? options.l1_decay : 0.0;
1719 |     this.l2_decay = typeof options.l2_decay !== 'undefined' ? options.l2_decay : 0.0;
1720 |     this.batch_size = typeof options.batch_size !== 'undefined' ? options.batch_size : 1;
1721 |     this.method = typeof options.method !== 'undefined' ? options.method : 'sgd'; // sgd/adagrad/adadelta/windowgrad/netsterov
1722 | 
1723 |     this.momentum = typeof options.momentum !== 'undefined' ? options.momentum : 0.9;
1724 |     this.ro = typeof options.ro !== 'undefined' ? options.ro : 0.95; // used in adadelta
1725 |     this.eps = typeof options.eps !== 'undefined' ? options.eps : 1e-6; // used in adadelta
1726 | 
1727 |     this.k = 0; // iteration counter
1728 |     this.gsum = []; // last iteration gradients (used for momentum calculations)
1729 |     this.xsum = []; // used in adadelta
1730 |   }
1731 | 
1732 |   Trainer.prototype = {
1733 |     train: function(x, y) {
1734 | 
1735 |       var start = new Date().getTime();
1736 |       this.net.forward(x, true); // also set the flag that lets the net know we're just training
1737 |       var end = new Date().getTime();
1738 |       var fwd_time = end - start;
1739 | 
1740 |       var start = new Date().getTime();
1741 |       var cost_loss = this.net.backward(y);
1742 |       var l2_decay_loss = 0.0;
1743 |       var l1_decay_loss = 0.0;
1744 |       var end = new Date().getTime();
1745 |       var bwd_time = end - start;
1746 |       
1747 |       this.k++;
1748 |       if(this.k % this.batch_size === 0) {
1749 | 
1750 |         var pglist = this.net.getParamsAndGrads();
1751 | 
1752 |         // initialize lists for accumulators. Will only be done once on first iteration
1753 |         if(this.gsum.length === 0 && (this.method !== 'sgd' || this.momentum > 0.0)) {
1754 |           // only vanilla sgd doesnt need either lists
1755 |           // momentum needs gsum
1756 |           // adagrad needs gsum
1757 |           // adadelta needs gsum and xsum
1758 |           for(var i=0;i<pglist.length;i++) {
1759 |             this.gsum.push(global.zeros(pglist[i].params.length));
1760 |             if(this.method === 'adadelta') {
1761 |               this.xsum.push(global.zeros(pglist[i].params.length));
1762 |             } else {
1763 |               this.xsum.push([]); // conserve memory
1764 |             }
1765 |           }
1766 |         }
1767 | 
1768 |         // perform an update for all sets of weights
1769 |         for(var i=0;i<pglist.length;i++) {
1770 |           var pg = pglist[i]; // param, gradient, other options in future (custom learning rate etc)
1771 |           var p = pg.params;
1772 |           var g = pg.grads;
1773 | 
1774 |           // learning rate for some parameters.
1775 |           var l2_decay_mul = typeof pg.l2_decay_mul !== 'undefined' ? pg.l2_decay_mul : 1.0;
1776 |           var l1_decay_mul = typeof pg.l1_decay_mul !== 'undefined' ? pg.l1_decay_mul : 1.0;
1777 |           var l2_decay = this.l2_decay * l2_decay_mul;
1778 |           var l1_decay = this.l1_decay * l1_decay_mul;
1779 | 
1780 |           var plen = p.length;
1781 |           for(var j=0;j<plen;j++) {
1782 |             l2_decay_loss += l2_decay*p[j]*p[j]/2; // accumulate weight decay loss
1783 |             l1_decay_loss += l1_decay*Math.abs(p[j]);
1784 |             var l1grad = l1_decay * (p[j] > 0 ? 1 : -1);
1785 |             var l2grad = l2_decay * (p[j]);
1786 | 
1787 |             var gij = (l2grad + l1grad + g[j]) / this.batch_size; // raw batch gradient
1788 | 
1789 |             var gsumi = this.gsum[i];
1790 |             var xsumi = this.xsum[i];
1791 |             if(this.method === 'adagrad') {
1792 |               // adagrad update
1793 |               gsumi[j] = gsumi[j] + gij * gij;
1794 |               var dx = - this.learning_rate / Math.sqrt(gsumi[j] + this.eps) * gij;
1795 |               p[j] += dx;
1796 |             } else if(this.method === 'windowgrad') {
1797 |               // this is adagrad but with a moving window weighted average
1798 |               // so the gradient is not accumulated over the entire history of the run. 
1799 |               // it's also referred to as Idea #1 in Zeiler paper on Adadelta. Seems reasonable to me!
1800 |               gsumi[j] = this.ro * gsumi[j] + (1-this.ro) * gij * gij;
1801 |               var dx = - this.learning_rate / Math.sqrt(gsumi[j] + this.eps) * gij; // eps added for better conditioning
1802 |               p[j] += dx;
1803 |             } else if(this.method === 'adadelta') {
1804 |               // assume adadelta if not sgd or adagrad
1805 |               gsumi[j] = this.ro * gsumi[j] + (1-this.ro) * gij * gij;
1806 |               var dx = - Math.sqrt((xsumi[j] + this.eps)/(gsumi[j] + this.eps)) * gij;
1807 |               xsumi[j] = this.ro * xsumi[j] + (1-this.ro) * dx * dx; // yes, xsum lags behind gsum by 1.
1808 |               p[j] += dx;
1809 |             } else if(this.method === 'nesterov') {
1810 |             	var dx = gsumi[j];
1811 |             	gsumi[j] = gsumi[j] * this.momentum + this.learning_rate * gij;
1812 |                 dx = this.momentum * dx - (1.0 + this.momentum) * gsumi[j];
1813 |                 p[j] += dx;
1814 |             } else {
1815 |               // assume SGD
1816 |               if(this.momentum > 0.0) {
1817 |                 // momentum update
1818 |                 var dx = this.momentum * gsumi[j] - this.learning_rate * gij; // step
1819 |                 gsumi[j] = dx; // back this up for next iteration of momentum
1820 |                 p[j] += dx; // apply corrected gradient
1821 |               } else {
1822 |                 // vanilla sgd
1823 |                 p[j] +=  - this.learning_rate * gij;
1824 |               }
1825 |             }
1826 |             g[j] = 0.0; // zero out gradient so that we can begin accumulating anew
1827 |           }
1828 |         }
1829 |       }
1830 | 
1831 |       // appending softmax_loss for backwards compatibility, but from now on we will always use cost_loss
1832 |       // in future, TODO: have to completely redo the way loss is done around the network as currently 
1833 |       // loss is a bit of a hack. Ideally, user should specify arbitrary number of loss functions on any layer
1834 |       // and it should all be computed correctly and automatically. 
1835 |       return {fwd_time: fwd_time, bwd_time: bwd_time, 
1836 |               l2_decay_loss: l2_decay_loss, l1_decay_loss: l1_decay_loss,
1837 |               cost_loss: cost_loss, softmax_loss: cost_loss, 
1838 |               loss: cost_loss + l1_decay_loss + l2_decay_loss}
1839 |     }
1840 |   }
1841 |   
1842 |   global.Trainer = Trainer;
1843 |   global.SGDTrainer = Trainer; // backwards compatibility
1844 | })(convnetjs);
1845 | 
1846 | (function(global) {
1847 |   "use strict";
1848 | 
1849 |   // used utilities, make explicit local references
1850 |   var randf = global.randf;
1851 |   var randi = global.randi;
1852 |   var Net = global.Net;
1853 |   var Trainer = global.Trainer;
1854 |   var maxmin = global.maxmin;
1855 |   var randperm = global.randperm;
1856 |   var weightedSample = global.weightedSample;
1857 |   var getopt = global.getopt;
1858 |   var arrUnique = global.arrUnique;
1859 | 
1860 |   /*
1861 |   A MagicNet takes data: a list of convnetjs.Vol(), and labels
1862 |   which for now are assumed to be class indeces 0..K. MagicNet then:
1863 |   - creates data folds for cross-validation
1864 |   - samples candidate networks
1865 |   - evaluates candidate networks on all data folds
1866 |   - produces predictions by model-averaging the best networks
1867 |   */
1868 |   var MagicNet = function(data, labels, opt) {
1869 |     var opt = opt || {};
1870 |     if(typeof data === 'undefined') { data = []; }
1871 |     if(typeof labels === 'undefined') { labels = []; }
1872 | 
1873 |     // required inputs
1874 |     this.data = data; // store these pointers to data
1875 |     this.labels = labels;
1876 | 
1877 |     // optional inputs
1878 |     this.train_ratio = getopt(opt, 'train_ratio', 0.7);
1879 |     this.num_folds = getopt(opt, 'num_folds', 10);
1880 |     this.num_candidates = getopt(opt, 'num_candidates', 50); // we evaluate several in parallel
1881 |     // how many epochs of data to train every network? for every fold?
1882 |     // higher values mean higher accuracy in final results, but more expensive
1883 |     this.num_epochs = getopt(opt, 'num_epochs', 50); 
1884 |     // number of best models to average during prediction. Usually higher = better
1885 |     this.ensemble_size = getopt(opt, 'ensemble_size', 10);
1886 | 
1887 |     // candidate parameters
1888 |     this.batch_size_min = getopt(opt, 'batch_size_min', 10);
1889 |     this.batch_size_max = getopt(opt, 'batch_size_max', 300);
1890 |     this.l2_decay_min = getopt(opt, 'l2_decay_min', -4);
1891 |     this.l2_decay_max = getopt(opt, 'l2_decay_max', 2);
1892 |     this.learning_rate_min = getopt(opt, 'learning_rate_min', -4);
1893 |     this.learning_rate_max = getopt(opt, 'learning_rate_max', 0);
1894 |     this.momentum_min = getopt(opt, 'momentum_min', 0.9);
1895 |     this.momentum_max = getopt(opt, 'momentum_max', 0.9);
1896 |     this.neurons_min = getopt(opt, 'neurons_min', 5);
1897 |     this.neurons_max = getopt(opt, 'neurons_max', 30);
1898 | 
1899 |     // computed
1900 |     this.folds = []; // data fold indices, gets filled by sampleFolds()
1901 |     this.candidates = []; // candidate networks that are being currently evaluated
1902 |     this.evaluated_candidates = []; // history of all candidates that were fully evaluated on all folds
1903 |     this.unique_labels = arrUnique(labels);
1904 |     this.iter = 0; // iteration counter, goes from 0 -> num_epochs * num_training_data
1905 |     this.foldix = 0; // index of active fold
1906 | 
1907 |     // callbacks
1908 |     this.finish_fold_callback = null;
1909 |     this.finish_batch_callback = null;
1910 | 
1911 |     // initializations
1912 |     if(this.data.length > 0) {
1913 |       this.sampleFolds();
1914 |       this.sampleCandidates();
1915 |     }
1916 |   };
1917 | 
1918 |   MagicNet.prototype = {
1919 | 
1920 |     // sets this.folds to a sampling of this.num_folds folds
1921 |     sampleFolds: function() {
1922 |       var N = this.data.length;
1923 |       var num_train = Math.floor(this.train_ratio * N);
1924 |       this.folds = []; // flush folds, if any
1925 |       for(var i=0;i<this.num_folds;i++) {
1926 |         var p = randperm(N);
1927 |         this.folds.push({train_ix: p.slice(0, num_train), test_ix: p.slice(num_train, N)});
1928 |       }
1929 |     },
1930 | 
1931 |     // returns a random candidate network
1932 |     sampleCandidate: function() {
1933 |       var input_depth = this.data[0].w.length;
1934 |       var num_classes = this.unique_labels.length;
1935 | 
1936 |       // sample network topology and hyperparameters
1937 |       var layer_defs = [];
1938 |       layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth: input_depth});
1939 |       var nl = weightedSample([0,1,2,3], [0.2, 0.3, 0.3, 0.2]); // prefer nets with 1,2 hidden layers
1940 |       for(var q=0;q<nl;q++) {
1941 |         var ni = randi(this.neurons_min, this.neurons_max);
1942 |         var act = ['tanh','maxout','relu'][randi(0,3)];
1943 |         if(randf(0,1)<0.5) {
1944 |           var dp = Math.random();
1945 |           layer_defs.push({type:'fc', num_neurons: ni, activation: act, drop_prob: dp});
1946 |         } else {
1947 |           layer_defs.push({type:'fc', num_neurons: ni, activation: act});
1948 |         }
1949 |       }
1950 |       layer_defs.push({type:'softmax', num_classes: num_classes});
1951 |       var net = new Net();
1952 |       net.makeLayers(layer_defs);
1953 | 
1954 |       // sample training hyperparameters
1955 |       var bs = randi(this.batch_size_min, this.batch_size_max); // batch size
1956 |       var l2 = Math.pow(10, randf(this.l2_decay_min, this.l2_decay_max)); // l2 weight decay
1957 |       var lr = Math.pow(10, randf(this.learning_rate_min, this.learning_rate_max)); // learning rate
1958 |       var mom = randf(this.momentum_min, this.momentum_max); // momentum. Lets just use 0.9, works okay usually ;p
1959 |       var tp = randf(0,1); // trainer type
1960 |       var trainer_def;
1961 |       if(tp<0.33) {
1962 |         trainer_def = {method:'adadelta', batch_size:bs, l2_decay:l2};
1963 |       } else if(tp<0.66) {
1964 |         trainer_def = {method:'adagrad', learning_rate: lr, batch_size:bs, l2_decay:l2};
1965 |       } else {
1966 |         trainer_def = {method:'sgd', learning_rate: lr, momentum: mom, batch_size:bs, l2_decay:l2};
1967 |       }
1968 |       
1969 |       var trainer = new Trainer(net, trainer_def);
1970 | 
1971 |       var cand = {};
1972 |       cand.acc = [];
1973 |       cand.accv = 0; // this will maintained as sum(acc) for convenience
1974 |       cand.layer_defs = layer_defs;
1975 |       cand.trainer_def = trainer_def;
1976 |       cand.net = net;
1977 |       cand.trainer = trainer;
1978 |       return cand;
1979 |     },
1980 | 
1981 |     // sets this.candidates with this.num_candidates candidate nets
1982 |     sampleCandidates: function() {
1983 |       this.candidates = []; // flush, if any
1984 |       for(var i=0;i<this.num_candidates;i++) {
1985 |         var cand = this.sampleCandidate();
1986 |         this.candidates.push(cand);
1987 |       }
1988 |     },
1989 | 
1990 |     step: function() {
1991 |       
1992 |       // run an example through current candidate
1993 |       this.iter++;
1994 | 
1995 |       // step all candidates on a random data point
1996 |       var fold = this.folds[this.foldix]; // active fold
1997 |       var dataix = fold.train_ix[randi(0, fold.train_ix.length)];
1998 |       for(var k=0;k<this.candidates.length;k++) {
1999 |         var x = this.data[dataix];
2000 |         var l = this.labels[dataix];
2001 |         this.candidates[k].trainer.train(x, l);
2002 |       }
2003 | 
2004 |       // process consequences: sample new folds, or candidates
2005 |       var lastiter = this.num_epochs * fold.train_ix.length;
2006 |       if(this.iter >= lastiter) {
2007 |         // finished evaluation of this fold. Get final validation
2008 |         // accuracies, record them, and go on to next fold.
2009 |         var val_acc = this.evalValErrors();
2010 |         for(var k=0;k<this.candidates.length;k++) {
2011 |           var c = this.candidates[k];
2012 |           c.acc.push(val_acc[k]);
2013 |           c.accv += val_acc[k];
2014 |         }
2015 |         this.iter = 0; // reset step number
2016 |         this.foldix++; // increment fold
2017 | 
2018 |         if(this.finish_fold_callback !== null) {
2019 |           this.finish_fold_callback();
2020 |         }
2021 | 
2022 |         if(this.foldix >= this.folds.length) {
2023 |           // we finished all folds as well! Record these candidates
2024 |           // and sample new ones to evaluate.
2025 |           for(var k=0;k<this.candidates.length;k++) {
2026 |             this.evaluated_candidates.push(this.candidates[k]);
2027 |           }
2028 |           // sort evaluated candidates according to accuracy achieved
2029 |           this.evaluated_candidates.sort(function(a, b) { 
2030 |             return (a.accv / a.acc.length) 
2031 |                  > (b.accv / b.acc.length) 
2032 |                  ? -1 : 1;
2033 |           });
2034 |           // and clip only to the top few ones (lets place limit at 3*ensemble_size)
2035 |           // otherwise there are concerns with keeping these all in memory 
2036 |           // if MagicNet is being evaluated for a very long time
2037 |           if(this.evaluated_candidates.length > 3 * this.ensemble_size) {
2038 |             this.evaluated_candidates = this.evaluated_candidates.slice(0, 3 * this.ensemble_size);
2039 |           }
2040 |           if(this.finish_batch_callback !== null) {
2041 |             this.finish_batch_callback();
2042 |           }
2043 |           this.sampleCandidates(); // begin with new candidates
2044 |           this.foldix = 0; // reset this
2045 |         } else {
2046 |           // we will go on to another fold. reset all candidates nets
2047 |           for(var k=0;k<this.candidates.length;k++) {
2048 |             var c = this.candidates[k];
2049 |             var net = new Net();
2050 |             net.makeLayers(c.layer_defs);
2051 |             var trainer = new Trainer(net, c.trainer_def);
2052 |             c.net = net;
2053 |             c.trainer = trainer;
2054 |           }
2055 |         }
2056 |       }
2057 |     },
2058 | 
2059 |     evalValErrors: function() {
2060 |       // evaluate candidates on validation data and return performance of current networks
2061 |       // as simple list
2062 |       var vals = [];
2063 |       var fold = this.folds[this.foldix]; // active fold
2064 |       for(var k=0;k<this.candidates.length;k++) {
2065 |         var net = this.candidates[k].net;
2066 |         var v = 0.0;
2067 |         for(var q=0;q<fold.test_ix.length;q++) {
2068 |           var x = this.data[fold.test_ix[q]];
2069 |           var l = this.labels[fold.test_ix[q]];
2070 |           net.forward(x);
2071 |           var yhat = net.getPrediction();
2072 |           v += (yhat === l ? 1.0 : 0.0); // 0 1 loss
2073 |         }
2074 |         v /= fold.test_ix.length; // normalize
2075 |         vals.push(v);
2076 |       }
2077 |       return vals;
2078 |     },
2079 | 
2080 |     // returns prediction scores for given test data point, as Vol
2081 |     // uses an averaged prediction from the best ensemble_size models
2082 |     // x is a Vol.
2083 |     predict_soft: function(data) {
2084 |       // forward prop the best networks
2085 |       // and accumulate probabilities at last layer into a an output Vol
2086 | 
2087 |       var eval_candidates = [];
2088 |       var nv = 0;
2089 |       if(this.evaluated_candidates.length === 0) {
2090 |         // not sure what to do here, first batch of nets hasnt evaluated yet
2091 |         // lets just predict with current candidates.
2092 |         nv = this.candidates.length;
2093 |         eval_candidates = this.candidates;
2094 |       } else {
2095 |         // forward prop the best networks from evaluated_candidates
2096 |         nv = Math.min(this.ensemble_size, this.evaluated_candidates.length);
2097 |         eval_candidates = this.evaluated_candidates
2098 |       }
2099 | 
2100 |       // forward nets of all candidates and average the predictions
2101 |       var xout, n;
2102 |       for(var j=0;j<nv;j++) {
2103 |         var net = eval_candidates[j].net;
2104 |         var x = net.forward(data);
2105 |         if(j===0) { 
2106 |           xout = x; 
2107 |           n = x.w.length; 
2108 |         } else {
2109 |           // add it on
2110 |           for(var d=0;d<n;d++) {
2111 |             xout.w[d] += x.w[d];
2112 |           }
2113 |         }
2114 |       }
2115 |       // produce average
2116 |       for(var d=0;d<n;d++) {
2117 |         xout.w[d] /= nv;
2118 |       }
2119 |       return xout;
2120 |     },
2121 | 
2122 |     predict: function(data) {
2123 |       var xout = this.predict_soft(data);
2124 |       if(xout.w.length !== 0) {
2125 |         var stats = maxmin(xout.w);
2126 |         var predicted_label = stats.maxi; 
2127 |       } else {
2128 |         var predicted_label = -1; // error out
2129 |       }
2130 |       return predicted_label;
2131 | 
2132 |     },
2133 | 
2134 |     toJSON: function() {
2135 |       // dump the top ensemble_size networks as a list
2136 |       var nv = Math.min(this.ensemble_size, this.evaluated_candidates.length);
2137 |       var json = {};
2138 |       json.nets = [];
2139 |       for(var i=0;i<nv;i++) {
2140 |         json.nets.push(this.evaluated_candidates[i].net.toJSON());
2141 |       }
2142 |       return json;
2143 |     },
2144 | 
2145 |     fromJSON: function(json) {
2146 |       this.ensemble_size = json.nets.length;
2147 |       this.evaluated_candidates = [];
2148 |       for(var i=0;i<this.ensemble_size;i++) {
2149 |         var net = new Net();
2150 |         net.fromJSON(json.nets[i]);
2151 |         var dummy_candidate = {};
2152 |         dummy_candidate.net = net;
2153 |         this.evaluated_candidates.push(dummy_candidate);
2154 |       }
2155 |     },
2156 | 
2157 |     // callback functions
2158 |     // called when a fold is finished, while evaluating a batch
2159 |     onFinishFold: function(f) { this.finish_fold_callback = f; },
2160 |     // called when a batch of candidates has finished evaluating
2161 |     onFinishBatch: function(f) { this.finish_batch_callback = f; }
2162 |     
2163 |   };
2164 | 
2165 |   global.MagicNet = MagicNet;
2166 | })(convnetjs);
2167 | (function(lib) {
2168 |   "use strict";
2169 |   if (typeof module === "undefined" || typeof module.exports === "undefined") {
2170 |     window.jsfeat = lib; // in ordinary browser attach library to window
2171 |   } else {
2172 |     module.exports = lib; // in nodejs
2173 |   }
2174 | })(convnetjs);
2175 | 


--------------------------------------------------------------------------------
/browser/js/util.js:
--------------------------------------------------------------------------------
 1 | 
 2 | // contains various utility functions 
 3 | var cnnutil = (function(exports){
 4 | 
 5 |   // a window stores _size_ number of values
 6 |   // and returns averages. Useful for keeping running
 7 |   // track of validation or training accuracy during SGD
 8 |   var Window = function(size, minsize) {
 9 |     this.v = [];
10 |     this.size = typeof(size)==='undefined' ? 100 : size;
11 |     this.minsize = typeof(minsize)==='undefined' ? 20 : minsize;
12 |     this.sum = 0;
13 |   }
14 |   Window.prototype = {
15 |     add: function(x) {
16 |       this.v.push(x);
17 |       this.sum += x;
18 |       if(this.v.length>this.size) {
19 |         var xold = this.v.shift();
20 |         this.sum -= xold;
21 |       }
22 |     },
23 |     get_average: function() {
24 |       if(this.v.length < this.minsize) return -1;
25 |       else return this.sum/this.v.length;
26 |     },
27 |     reset: function(x) {
28 |       this.v = [];
29 |       this.sum = 0;
30 |     }
31 |   }
32 | 
33 |   // returns min, max and indeces of an array
34 |   var maxmin = function(w) {
35 |     if(w.length === 0) { return {}; } // ... ;s
36 | 
37 |     var maxv = w[0];
38 |     var minv = w[0];
39 |     var maxi = 0;
40 |     var mini = 0;
41 |     for(var i=1;i<w.length;i++) {
42 |       if(w[i] > maxv) { maxv = w[i]; maxi = i; } 
43 |       if(w[i] < minv) { minv = w[i]; mini = i; } 
44 |     }
45 |     return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv};
46 |   }
47 | 
48 |   // returns string representation of float
49 |   // but truncated to length of d digits
50 |   var f2t = function(x, d) {
51 |     if(typeof(d)==='undefined') { var d = 5; }
52 |     var dd = 1.0 * Math.pow(10, d);
53 |     return '' + Math.floor(x*dd)/dd;
54 |   }
55 | 
56 |   exports = exports || {};
57 |   exports.Window = Window;
58 |   exports.maxmin = maxmin;
59 |   exports.f2t = f2t;
60 |   return exports;
61 | 
62 | })(typeof module != 'undefined' && module.exports);  // add exports to module.exports if in node.js
63 | 
64 | 
65 | 


--------------------------------------------------------------------------------
/browser/js/vis.js:
--------------------------------------------------------------------------------
  1 | 
  2 | // contains various utility functions 
  3 | var cnnvis = (function(exports){
  4 | 
  5 |   // can be used to graph loss, or accuract over time
  6 |   var Graph = function(options) {
  7 |     var options = options || {};
  8 |     this.step_horizon = options.step_horizon || 1000;
  9 |     
 10 |     this.pts = [];
 11 |     
 12 |     this.maxy = -9999;
 13 |     this.miny = 9999;
 14 |   }
 15 | 
 16 |   Graph.prototype = {
 17 |     // canv is the canvas we wish to update with this new datapoint
 18 |     add: function(step, y) {
 19 |       var time = new Date().getTime(); // in ms
 20 |       if(y>this.maxy*0.99) this.maxy = y*1.05;
 21 |       if(y<this.miny*1.01) this.miny = y*0.95;
 22 | 
 23 |       this.pts.push({step: step, time: time, y: y});
 24 |       if(step > this.step_horizon) this.step_horizon *= 2;
 25 |     },
 26 |     // elt is a canvas we wish to draw into
 27 |     drawSelf: function(canv) {
 28 |       
 29 |       var pad = 25;
 30 |       var H = canv.height;
 31 |       var W = canv.width;
 32 |       var ctx = canv.getContext('2d');
 33 | 
 34 |       ctx.clearRect(0, 0, W, H);
 35 |       ctx.font="10px Georgia";
 36 | 
 37 |       var f2t = function(x) {
 38 |         var dd = 1.0 * Math.pow(10, 2);
 39 |         return '' + Math.floor(x*dd)/dd;
 40 |       }
 41 | 
 42 |       // draw guidelines and values
 43 |       ctx.strokeStyle = "#999";
 44 |       ctx.beginPath();
 45 |       var ng = 10;
 46 |       for(var i=0;i<=ng;i++) {
 47 |         var xpos = i/ng*(W-2*pad)+pad;
 48 |         ctx.moveTo(xpos, pad);
 49 |         ctx.lineTo(xpos, H-pad);
 50 |         ctx.fillText(f2t(i/ng*this.step_horizon/1000)+'k',xpos,H-pad+14);
 51 |       }
 52 |       for(var i=0;i<=ng;i++) {
 53 |         var ypos = i/ng*(H-2*pad)+pad;
 54 |         ctx.moveTo(pad, ypos);
 55 |         ctx.lineTo(W-pad, ypos);
 56 |         ctx.fillText(f2t((ng-i)/ng*(this.maxy-this.miny) + this.miny), 0, ypos);
 57 |       }
 58 |       ctx.stroke();
 59 | 
 60 |       var N = this.pts.length;
 61 |       if(N<2) return;
 62 | 
 63 |       // draw the actual curve
 64 |       var t = function(x, y, s) {
 65 |         var tx = x / s.step_horizon * (W-pad*2) + pad;
 66 |         var ty = H - ((y-s.miny) / (s.maxy-s.miny) * (H-pad*2) + pad);
 67 |         return {tx:tx, ty:ty}
 68 |       }
 69 | 
 70 |       ctx.strokeStyle = "red";
 71 |       ctx.beginPath()
 72 |       for(var i=0;i<N;i++) {
 73 |         // draw line from i-1 to i
 74 |         var p = this.pts[i];
 75 |         var pt = t(p.step, p.y, this);
 76 |         if(i===0) ctx.moveTo(pt.tx, pt.ty);
 77 |         else ctx.lineTo(pt.tx, pt.ty);
 78 |       }
 79 |       ctx.stroke();
 80 |     }
 81 |   }
 82 | 
 83 |   // same as graph but draws multiple lines. For now I'm lazy and duplicating
 84 |   // the code, but in future I will merge these two more nicely.
 85 |   var MultiGraph = function(legend, options) {
 86 |     var options = options || {};
 87 |     this.step_horizon = options.step_horizon || 1000;
 88 | 
 89 |     if(typeof options.maxy !== 'undefined') this.maxy_forced = options.maxy;
 90 |     if(typeof options.miny !== 'undefined') this.miny_forced = options.miny;
 91 | 
 92 |     this.pts = [];
 93 |     
 94 |     this.maxy = -9999;
 95 |     this.miny = 9999;
 96 |     this.numlines = 0;
 97 | 
 98 |     this.numlines = legend.length;
 99 |     this.legend = legend;
100 |     this.styles = ["red", "blue", "green", "black", "magenta", "cyan", "purple", "aqua", "olive", "lime", "navy"];
101 |     // 17 basic colors: aqua, black, blue, fuchsia, gray, green, lime, maroon, navy, olive, orange, purple, red, silver, teal, white, and yellow
102 |   }
103 | 
104 |   MultiGraph.prototype = {
105 |     // canv is the canvas we wish to update with this new datapoint
106 |     add: function(step, yl) {
107 |       var time = new Date().getTime(); // in ms
108 |       var n = yl.length;
109 |       for(var k=0;k<n;k++) {
110 |         var y = yl[k];
111 |         if(y>this.maxy*0.99) this.maxy = y*1.05;
112 |         if(y<this.miny*1.01) this.miny = y*0.95;
113 |       }
114 | 
115 |       if(typeof this.maxy_forced !== 'undefined') this.maxy = this.maxy_forced;
116 |       if(typeof this.miny_forced !== 'undefined') this.miny = this.miny_forced;
117 | 
118 |       this.pts.push({step: step, time: time, yl: yl});
119 |       if(step > this.step_horizon) this.step_horizon *= 2;
120 |     },
121 |     // elt is a canvas we wish to draw into
122 |     drawSelf: function(canv) {
123 |       
124 |       var pad = 25;
125 |       var H = canv.height;
126 |       var W = canv.width;
127 |       var ctx = canv.getContext('2d');
128 | 
129 |       ctx.clearRect(0, 0, W, H);
130 |       ctx.font="10px Georgia";
131 | 
132 |       var f2t = function(x) {
133 |         var dd = 1.0 * Math.pow(10, 2);
134 |         return '' + Math.floor(x*dd)/dd;
135 |       }
136 | 
137 |       // draw guidelines and values
138 |       ctx.strokeStyle = "#999";
139 |       ctx.beginPath();
140 |       var ng = 10;
141 |       for(var i=0;i<=ng;i++) {
142 |         var xpos = i/ng*(W-2*pad)+pad;
143 |         ctx.moveTo(xpos, pad);
144 |         ctx.lineTo(xpos, H-pad);
145 |         ctx.fillText(f2t(i/ng*this.step_horizon/1000)+'k',xpos,H-pad+14);
146 |       }
147 |       for(var i=0;i<=ng;i++) {
148 |         var ypos = i/ng*(H-2*pad)+pad;
149 |         ctx.moveTo(pad, ypos);
150 |         ctx.lineTo(W-pad, ypos);
151 |         ctx.fillText(f2t((ng-i)/ng*(this.maxy-this.miny) + this.miny), 0, ypos);
152 |       }
153 |       ctx.stroke();
154 | 
155 |       var N = this.pts.length;
156 |       if(N<2) return;
157 | 
158 |       // draw legend
159 |       for(var k=0;k<this.numlines;k++) {
160 |         ctx.fillStyle = this.styles[k % this.styles.length];
161 |         ctx.fillText(this.legend[k], W-pad-100, pad+20+k*16);
162 |       }
163 |       ctx.fillStyle = "black";
164 | 
165 |       // draw the actual curve
166 |       var t = function(x, y, s) {
167 |         var tx = x / s.step_horizon * (W-pad*2) + pad;
168 |         var ty = H - ((y-s.miny) / (s.maxy-s.miny) * (H-pad*2) + pad);
169 |         return {tx:tx, ty:ty}
170 |       }
171 |       for(var k=0;k<this.numlines;k++) {
172 | 
173 |         ctx.strokeStyle = this.styles[k % this.styles.length];
174 |         ctx.beginPath()
175 |         for(var i=0;i<N;i++) {
176 |           // draw line from i-1 to i
177 |           var p = this.pts[i];
178 |           var pt = t(p.step, p.yl[k], this);
179 |           if(i===0) ctx.moveTo(pt.tx, pt.ty);
180 |           else ctx.lineTo(pt.tx, pt.ty);
181 |         }
182 |         ctx.stroke();
183 |       }
184 | 
185 |     }
186 |   }
187 | 
188 |   exports = exports || {};
189 |   exports.Graph = Graph;
190 |   exports.MultiGraph = MultiGraph;
191 |   return exports;
192 | 
193 | })(typeof module != 'undefined' && module.exports);  // add exports to module.exports if in node.js
194 | 
195 | 
196 | 


--------------------------------------------------------------------------------
/browser/mnist.html:
--------------------------------------------------------------------------------
  1 | <!doctype html>
  2 | <html lang="en">
  3 | <head>
  4 |   <meta charset="utf-8">
  5 |   <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  6 |   <title>ConvNetJS MNIST demo</title>
  7 |   <meta name="description" content="">
  8 |   <meta name="author" content="">
  9 | 
 10 | <style>
 11 | .layer {
 12 |   border: 1px solid #999;
 13 |   margin-bottom: 5px;
 14 |   text-align: left;
 15 |   padding: 10px;
 16 | }
 17 | .layer_act {
 18 |   width: 450px;
 19 |   float: right;
 20 | }
 21 | .ltconv {
 22 |   background-color: #FDD;
 23 | }
 24 | .ltrelu {
 25 |   background-color: #FDF;
 26 | }
 27 | .ltpool {
 28 |   background-color: #DDF;
 29 | }
 30 | .ltsoftmax {
 31 |   background-color: #FFD;
 32 | }
 33 | .ltfc {
 34 |   background-color: #DFF;
 35 | }
 36 | .ltlrn {
 37 |   background-color: #DFD; 
 38 | }
 39 | .ltdropout {
 40 |   background-color: #AAA;  
 41 | }
 42 | .ltitle {
 43 |   color: #333;
 44 |   font-size: 18px;
 45 | }
 46 | .actmap {
 47 |   margin: 1px;
 48 | }
 49 | #trainstats {
 50 |   text-align: left;
 51 | }
 52 | .clear {
 53 |   clear: both;
 54 | }
 55 | #wrap {
 56 |   width: 800px;
 57 |   margin-left: auto;
 58 |   margin-right: auto;
 59 | }
 60 | h1 {
 61 |   font-size: 16px;
 62 |   color: #333;
 63 |   background-color: #DDD;
 64 |   border-bottom: 1px #999 solid;
 65 |   text-align: center;
 66 | }
 67 | .secpart {
 68 |   width: 400px;
 69 |   float: left;
 70 | }
 71 | #lossgraph {
 72 |   /*border: 1px solid #F0F;*/
 73 |   width: 100%;
 74 | }
 75 | .probsdiv canvas {
 76 |   float: left;
 77 | }
 78 | .probsdiv {
 79 |   height: 60px;
 80 |   width: 180px;
 81 |   display: inline-block;
 82 |   font-size: 12px;
 83 |   box-shadow: 0px 0px 2px 2px #EEE;
 84 |   margin: 5px;
 85 |   padding: 5px;
 86 |   color: black;
 87 | }
 88 | .pp {
 89 |   margin: 1px;
 90 |   padding: 1px;
 91 | }
 92 | #testset_acc {
 93 |   margin-bottom: 200px;
 94 | }
 95 | body {
 96 |   font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
 97 | }
 98 | </style>
 99 | 
100 | <script src="js/jquery-1.8.3.min.js"></script>
101 | <script src="js/vis.js"></script>
102 | <script src="js/util.js"></script>
103 | <script src="js/convnet.js"></script>
104 | 
105 | <script src="js/mnist_labels.js"></script>
106 | 
107 | <script>
108 | 
109 | var layer_defs, net, trainer;
110 | var t = "layer_defs = [];\n\
111 | layer_defs.push({type:'input', out_sx:24, out_sy:24, out_depth:1});\n\
112 | layer_defs.push({type:'conv', sx:5, filters:8, stride:1, pad:2, activation:'relu'});\n\
113 | layer_defs.push({type:'pool', sx:2, stride:2});\n\
114 | layer_defs.push({type:'conv', sx:5, filters:16, stride:1, pad:2, activation:'relu'});\n\
115 | layer_defs.push({type:'pool', sx:3, stride:3});\n\
116 | layer_defs.push({type:'softmax', num_classes:10});\n\
117 | \n\
118 | net = new convnetjs.Net();\n\
119 | net.makeLayers(layer_defs);\n\
120 | \n\
121 | trainer = new convnetjs.SGDTrainer(net, {method:'adadelta', batch_size:20, l2_decay:0.001});\n\
122 | ";
123 | 
124 | // ------------------------
125 | // BEGIN MNIST SPECIFIC STUFF
126 | // ------------------------
127 | classes_txt = ['0','1','2','3','4','5','6','7','8','9'];
128 | 
129 | var use_validation_data = true;
130 | var sample_training_instance = function() {
131 | 
132 |   // find an unloaded batch
133 |   var bi = Math.floor(Math.random()*loaded_train_batches.length);
134 |   var b = loaded_train_batches[bi];
135 |   var k = Math.floor(Math.random()*3000); // sample within the batch
136 |   var n = b*3000+k;
137 | 
138 |   // load more batches over time
139 |   if(step_num%5000===0 && step_num>0) {
140 |     for(var i=0;i<num_batches;i++) {
141 |       if(!loaded[i]) {
142 |         // load it
143 |         load_data_batch(i);
144 |         break; // okay for now
145 |       }
146 |     }
147 |   }
148 | 
149 |   // fetch the appropriate row of the training image and reshape into a Vol
150 |   var p = img_data[b].data;
151 |   var x = new convnetjs.Vol(28,28,1,0.0);
152 |   var W = 28*28;
153 |   for(var i=0;i<W;i++) {
154 |     var ix = ((W * k) + i) * 4;
155 |     x.w[i] = p[ix]/255.0;
156 |   }
157 |   x = convnetjs.augment(x, 24);
158 | 
159 |   var isval = use_validation_data && n%10===0 ? true : false;
160 |   return {x:x, label:labels[n], isval:isval};
161 | }
162 | 
163 | // sample a random testing instance
164 | var sample_test_instance = function() {
165 |   var b = 20;
166 |   var k = Math.floor(Math.random()*3000);
167 |   var n = b*3000+k;
168 | 
169 |   var p = img_data[b].data;
170 |   var x = new convnetjs.Vol(28,28,1,0.0);
171 |   var W = 28*28;
172 |   for(var i=0;i<W;i++) {
173 |     var ix = ((W * k) + i) * 4;
174 |     x.w[i] = p[ix]/255.0;
175 |   }
176 |   var xs = [];
177 |   for(var i=0;i<4;i++) {
178 |     xs.push(convnetjs.augment(x, 24));
179 |   }
180 |   // return multiple augmentations, and we will average the network over them
181 |   // to increase performance
182 |   return {x:xs, label:labels[n]};
183 | }
184 | 
185 | var num_batches = 21; // 20 training batches, 1 test
186 | var data_img_elts = new Array(num_batches);
187 | var img_data = new Array(num_batches);
188 | var loaded = new Array(num_batches);
189 | var loaded_train_batches = [];
190 | 
191 | // int main
192 | $(window).load(function() {
193 | 
194 |   $("#newnet").val(t);
195 |   eval($("#newnet").val());
196 | 
197 |   update_net_param_display();
198 | 
199 |   for(var k=0;k<loaded.length;k++) { loaded[k] = false; }
200 | 
201 |   load_data_batch(0); // async load train set batch 0 (6 total train batches)
202 |   load_data_batch(20); // async load test set (batch 6)
203 |   start_fun();
204 | });
205 | 
206 | var start_fun = function() {
207 |   if(loaded[0] && loaded[20]) { 
208 |     console.log('starting!'); 
209 |     setInterval(load_and_step, 0); // lets go!
210 |   }
211 |   else { setTimeout(start_fun, 200); } // keep checking
212 | }
213 | 
214 | var load_data_batch = function(batch_num) {
215 |   // Load the dataset with JS in background
216 |   data_img_elts[batch_num] = new Image();
217 |   var data_img_elt = data_img_elts[batch_num];
218 |   data_img_elt.onload = function() { 
219 |     var data_canvas = document.createElement('canvas');
220 |     data_canvas.width = data_img_elt.width;
221 |     data_canvas.height = data_img_elt.height;
222 |     var data_ctx = data_canvas.getContext("2d");
223 |     data_ctx.drawImage(data_img_elt, 0, 0); // copy it over... bit wasteful :(
224 |     img_data[batch_num] = data_ctx.getImageData(0, 0, data_canvas.width, data_canvas.height);
225 |     loaded[batch_num] = true;
226 |     if(batch_num < 20) { loaded_train_batches.push(batch_num); }
227 |     console.log('finished loading data batch ' + batch_num);
228 |   };
229 |   data_img_elt.src = "mnist/mnist_batch_" + batch_num + ".png";
230 | }
231 | 
232 | // ------------------------
233 | // END MNIST SPECIFIC STUFF
234 | // ------------------------
235 | 
236 | var maxmin = cnnutil.maxmin;
237 | var f2t = cnnutil.f2t;
238 | 
239 | // elt is the element to add all the canvas activation drawings into
240 | // A is the Vol() to use
241 | // scale is a multiplier to make the visualizations larger. Make higher for larger pictures
242 | // if grads is true then gradients are used instead
243 | var draw_activations = function(elt, A, scale, grads) {
244 | 
245 |   var s = scale || 2; // scale
246 |   var draw_grads = false;
247 |   if(typeof(grads) !== 'undefined') draw_grads = grads;
248 | 
249 |   // get max and min activation to scale the maps automatically
250 |   var w = draw_grads ? A.dw : A.w;
251 |   var mm = maxmin(w);
252 | 
253 |   // create the canvas elements, draw and add to DOM
254 |   for(var d=0;d<A.depth;d++) {
255 | 
256 |     var canv = document.createElement('canvas');
257 |     canv.className = 'actmap';
258 |     var W = A.sx * s;
259 |     var H = A.sy * s;
260 |     canv.width = W;
261 |     canv.height = H;
262 |     var ctx = canv.getContext('2d');
263 |     var g = ctx.createImageData(W, H);
264 | 
265 |     for(var x=0;x<A.sx;x++) {
266 |       for(var y=0;y<A.sy;y++) {
267 |         if(draw_grads) {
268 |           var dval = Math.floor((A.get_grad(x,y,d)-mm.minv)/mm.dv*255);
269 |         } else {
270 |           var dval = Math.floor((A.get(x,y,d)-mm.minv)/mm.dv*255);  
271 |         }
272 |         for(var dx=0;dx<s;dx++) {
273 |           for(var dy=0;dy<s;dy++) {
274 |             var pp = ((W * (y*s+dy)) + (dx + x*s)) * 4;
275 |             for(var i=0;i<3;i++) { g.data[pp + i] = dval; } // rgb
276 |             g.data[pp+3] = 255; // alpha channel
277 |           }
278 |         }
279 |       }
280 |     }
281 |     ctx.putImageData(g, 0, 0);
282 |     elt.appendChild(canv);
283 |   }  
284 | }
285 | 
286 | var visualize_activations = function(net, elt) {
287 | 
288 |   // clear the element
289 |   elt.innerHTML = "";
290 | 
291 |   // show activations in each layer
292 |   var N = net.layers.length;
293 |   for(var i=0;i<N;i++) {
294 |     var L = net.layers[i];
295 | 
296 |     var layer_div = document.createElement('div');
297 | 
298 |     // visualize activations
299 |     var activations_div = document.createElement('div');
300 |     activations_div.appendChild(document.createTextNode('Activations:'));
301 |     activations_div.appendChild(document.createElement('br'));
302 |     activations_div.className = 'layer_act';
303 |     var scale = 2;
304 |     if(L.layer_type==='softmax' || L.layer_type==='fc') scale = 10; // for softmax
305 |     draw_activations(activations_div, L.out_act, scale);
306 | 
307 |     // visualize data gradients
308 |     if(L.layer_type !== 'softmax') {
309 |       var grad_div = document.createElement('div');
310 |       grad_div.appendChild(document.createTextNode('Data Gradients:'));
311 |       grad_div.appendChild(document.createElement('br'));
312 |       grad_div.className = 'layer_grad';
313 |       var scale = 2;
314 |       if(L.layer_type==='softmax' || L.layer_type==='fc') scale = 10; // for softmax
315 |       draw_activations(grad_div, L.out_act, scale, true);
316 |       activations_div.appendChild(grad_div);
317 |     }
318 | 
319 |     // visualize filters if they are of reasonable size
320 |     if(L.layer_type === 'conv') {
321 |       var filters_div = document.createElement('div');
322 |       if(L.filters[0].sx>3) {
323 |         // actual weights
324 |         filters_div.appendChild(document.createTextNode('Weights:'));
325 |         filters_div.appendChild(document.createElement('br'));
326 |         for(var j=0;j<L.filters.length;j++) {
327 |           filters_div.appendChild(document.createTextNode('('));
328 |           draw_activations(filters_div, L.filters[j], 2);
329 |           filters_div.appendChild(document.createTextNode(')'));
330 |         }
331 |         // gradients
332 |         filters_div.appendChild(document.createElement('br'));
333 |         filters_div.appendChild(document.createTextNode('Weight Gradients:'));
334 |         filters_div.appendChild(document.createElement('br'));
335 |         for(var j=0;j<L.filters.length;j++) {
336 |           filters_div.appendChild(document.createTextNode('('));
337 |           draw_activations(filters_div, L.filters[j], 2, true);
338 |           filters_div.appendChild(document.createTextNode(')'));
339 |         }
340 |       } else {
341 |         filters_div.appendChild(document.createTextNode('Weights hidden, too small'));
342 |       }
343 |       activations_div.appendChild(filters_div);
344 |     }
345 |     layer_div.appendChild(activations_div);
346 | 
347 |     // print some stats on left of the layer
348 |     layer_div.className = 'layer ' + 'lt' + L.layer_type;
349 |     var title_div = document.createElement('div');
350 |     title_div.className = 'ltitle'
351 |     var t = L.layer_type + ' (' + L.out_sx + 'x' + L.out_sy + 'x' + L.out_depth + ')';
352 |     title_div.appendChild(document.createTextNode(t));
353 |     layer_div.appendChild(title_div);
354 | 
355 |     if(L.layer_type==='conv') {
356 |       var t = 'filter size ' + L.filters[0].sx + 'x' + L.filters[0].sy + 'x' + L.filters[0].depth + ', stride ' + L.stride;
357 |       layer_div.appendChild(document.createTextNode(t));
358 |       layer_div.appendChild(document.createElement('br'));
359 |     }
360 |     if(L.layer_type==='pool') {
361 |       var t = 'pooling size ' + L.sx + 'x' + L.sy + ', stride ' + L.stride;
362 |       layer_div.appendChild(document.createTextNode(t));
363 |       layer_div.appendChild(document.createElement('br'));
364 |     }
365 | 
366 |     // find min, max activations and display them
367 |     var mma = maxmin(L.out_act.w);
368 |     var t = 'max activation: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
369 |     layer_div.appendChild(document.createTextNode(t));
370 |     layer_div.appendChild(document.createElement('br'));
371 | 
372 |     var mma = maxmin(L.out_act.dw);
373 |     var t = 'max gradient: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
374 |     layer_div.appendChild(document.createTextNode(t));
375 |     layer_div.appendChild(document.createElement('br'));
376 | 
377 |     // number of parameters
378 |     if(L.layer_type==='conv') {
379 |       var tot_params = L.sx*L.sy*L.in_depth*L.filters.length + L.filters.length;
380 |       var t = 'parameters: ' + L.filters.length + 'x' + L.sx + 'x' + L.sy + 'x' + L.in_depth + '+' + L.filters.length + ' = ' + tot_params;
381 |       layer_div.appendChild(document.createTextNode(t));
382 |       layer_div.appendChild(document.createElement('br'));
383 |     }
384 |     if(L.layer_type==='fc') {
385 |       var tot_params = L.num_inputs*L.filters.length + L.filters.length;
386 |       var t = 'parameters: ' + L.filters.length + 'x' + L.num_inputs + '+' + L.filters.length + ' = ' + tot_params;
387 |       layer_div.appendChild(document.createTextNode(t));
388 |       layer_div.appendChild(document.createElement('br'));
389 |     }
390 | 
391 |     // css madness needed here...
392 |     var clear = document.createElement('div');
393 |     clear.className = 'clear';
394 |     layer_div.appendChild(clear);
395 | 
396 |     elt.appendChild(layer_div);
397 |   }
398 | }
399 | 
400 | // loads a training image and trains on it with the network
401 | var paused = false;
402 | var load_and_step = function() {
403 |   if(paused) return; 
404 | 
405 |   var sample = sample_training_instance();
406 |   step(sample); // process this image
407 | }
408 | 
409 | // evaluate current network on test set
410 | var test_predict = function() {
411 |   var num_classes = net.layers[net.layers.length-1].out_depth;
412 | 
413 |   document.getElementById('testset_acc').innerHTML = '';
414 |   // grab a random test image
415 |   for(num=0;num<50;num++) {
416 |     var sample = sample_test_instance();
417 |     var y = sample.label;  // ground truth label
418 | 
419 |     // forward prop it through the network
420 |     var aavg = new convnetjs.Vol(1,1,num_classes,0.0);
421 |     // ensures we always have a list, regardless if above returns single item or list
422 |     var xs = [].concat(sample.x);
423 |     var n = xs.length;
424 |     for(var i=0;i<n;i++) {
425 |       var a = net.forward(xs[i]);
426 |       aavg.addFrom(a);
427 |     }
428 |     var preds = [];
429 |     for(var k=0;k<aavg.w.length;k++) { preds.push({k:k,p:aavg.w[k]}); }
430 |     preds.sort(function(a,b){return a.p<b.p ? 1:-1;});
431 |     
432 |     var div = document.createElement('div');
433 |     div.className = 'testdiv';
434 | 
435 |     // draw the image into a canvas
436 |     draw_activations(div, xs[0], 2); // draw Vol into canv
437 | 
438 |     // add predictions
439 |     var probsdiv = document.createElement('div');
440 |     div.className = 'probsdiv';
441 |     var t = '';
442 |     for(var k=0;k<3;k++) {
443 |       var col = preds[k].k===y ? 'rgb(85,187,85)' : 'rgb(187,85,85)';
444 |       t += '<div class=\"pp\" style=\"width:' + Math.floor(preds[k].p/n*100) + 'px; margin-left: 60px; background-color:' + col + ';\">' + classes_txt[preds[k].k] + '</div>'
445 |     }
446 |     probsdiv.innerHTML = t;
447 |     div.appendChild(probsdiv);
448 | 
449 |     // add it into DOM
450 |     $("#testset_acc").append(div).fadeIn(1000);
451 |   }
452 | }
453 | 
454 | 
455 | var lossGraph = new cnnvis.Graph();
456 | var xLossWindow = new cnnutil.Window(100);
457 | var wLossWindow = new cnnutil.Window(100);
458 | var trainAccWindow = new cnnutil.Window(100);
459 | var valAccWindow = new cnnutil.Window(100);
460 | var step_num = 0;
461 | var step = function(sample) {
462 | 
463 |   var x = sample.x;
464 |   var y = sample.label;
465 | 
466 |   if(sample.isval) {
467 |     // use x to build our estimate of validation error
468 |     net.forward(x);
469 |     var yhat = net.getPrediction();
470 |     var val_acc = yhat === y ? 1.0 : 0.0;
471 |     valAccWindow.add(val_acc);
472 |     return; // get out
473 |   }
474 | 
475 |   // train on it with network
476 |   var stats = trainer.train(x, y);
477 |   var lossx = stats.cost_loss;
478 |   var lossw = stats.l2_decay_loss;
479 | 
480 |   // keep track of stats such as the average training error and loss
481 |   var yhat = net.getPrediction();
482 |   var train_acc = yhat === y ? 1.0 : 0.0;
483 |   xLossWindow.add(lossx);
484 |   wLossWindow.add(lossw);
485 |   trainAccWindow.add(train_acc);
486 | 
487 |   // visualize training status
488 |   var train_elt = document.getElementById("trainstats");
489 |   train_elt.innerHTML = '';
490 |   var t = 'Forward time per example: ' + stats.fwd_time + 'ms';
491 |   train_elt.appendChild(document.createTextNode(t));
492 |   train_elt.appendChild(document.createElement('br'));
493 |   var t = 'Backprop time per example: ' + stats.bwd_time + 'ms';
494 |   train_elt.appendChild(document.createTextNode(t));
495 |   train_elt.appendChild(document.createElement('br'));
496 |   var t = 'Classification loss: ' + f2t(xLossWindow.get_average());
497 |   train_elt.appendChild(document.createTextNode(t));
498 |   train_elt.appendChild(document.createElement('br'));
499 |   var t = 'L2 Weight decay loss: ' + f2t(wLossWindow.get_average());
500 |   train_elt.appendChild(document.createTextNode(t));
501 |   train_elt.appendChild(document.createElement('br'));
502 |   var t = 'Training accuracy: ' + f2t(trainAccWindow.get_average());
503 |   train_elt.appendChild(document.createTextNode(t));
504 |   train_elt.appendChild(document.createElement('br'));
505 |   var t = 'Validation accuracy: ' + f2t(valAccWindow.get_average());
506 |   train_elt.appendChild(document.createTextNode(t));
507 |   train_elt.appendChild(document.createElement('br'));
508 |   var t = 'Examples seen: ' + step_num;
509 |   train_elt.appendChild(document.createTextNode(t));
510 |   train_elt.appendChild(document.createElement('br'));
511 | 
512 |   // visualize activations
513 |   if(step_num % 100 === 0) {
514 |     var vis_elt = document.getElementById("visnet");
515 |     visualize_activations(net, vis_elt);
516 |   }
517 | 
518 |   // log progress to graph, (full loss)
519 |   if(step_num % 200 === 0) {
520 |     var xa = xLossWindow.get_average();
521 |     var xw = wLossWindow.get_average();
522 |     if(xa >= 0 && xw >= 0) { // if they are -1 it means not enough data was accumulated yet for estimates
523 |       lossGraph.add(step_num, xa + xw);
524 |       lossGraph.drawSelf(document.getElementById("lossgraph"));
525 |     }
526 |   }
527 | 
528 |   // run prediction on test set
529 |   if(step_num % 1000 === 0) {
530 |     test_predict();
531 |   }
532 |   step_num++;
533 | }
534 | 
535 | // user settings 
536 | var change_lr = function() {
537 |   trainer.learning_rate = parseFloat(document.getElementById("lr_input").value);
538 |   update_net_param_display();
539 | }
540 | var change_momentum = function() {
541 |   trainer.momentum = parseFloat(document.getElementById("momentum_input").value);
542 |   update_net_param_display();
543 | }
544 | var change_batch_size = function() {
545 |   trainer.batch_size = parseFloat(document.getElementById("batch_size_input").value);
546 |   update_net_param_display();
547 | }
548 | var change_decay = function() {
549 |   trainer.l2_decay = parseFloat(document.getElementById("decay_input").value);
550 |   update_net_param_display();
551 | }
552 | var update_net_param_display = function() {
553 |   document.getElementById('lr_input').value = trainer.learning_rate;
554 |   document.getElementById('momentum_input').value = trainer.momentum;
555 |   document.getElementById('batch_size_input').value = trainer.batch_size;
556 |   document.getElementById('decay_input').value = trainer.l2_decay;
557 | }
558 | var toggle_pause = function() {
559 |   paused = !paused;
560 |   var btn = document.getElementById('buttontp');
561 |   if(paused) { btn.value = 'resume' }
562 |   else { btn.value = 'pause'; }
563 | }
564 | var dump_json = function() {
565 |   document.getElementById("dumpjson").value = JSON.stringify(net.toJSON());
566 | }
567 | var clear_graph = function() {
568 |   lossGraph = new cnnvis.Graph(); // reinit graph too 
569 | }
570 | var reset_all = function() {
571 |   update_net_param_display();
572 | 
573 |   // reinit windows that keep track of val/train accuracies
574 |   xLossWindow.reset();
575 |   wLossWindow.reset();
576 |   trainAccWindow.reset();
577 |   valAccWindow.reset();
578 |   lossGraph = new cnnvis.Graph(); // reinit graph too
579 |   step_num = 0;
580 | }
581 | var load_from_json = function() {
582 |   var jsonString = document.getElementById("dumpjson").value;
583 |   var json = JSON.parse(jsonString);
584 |   net = new convnetjs.Net();
585 |   net.fromJSON(json);
586 |   reset_all();
587 | }
588 | var change_net = function() {
589 |   eval($("#newnet").val());
590 |   reset_all();
591 | }
592 | 
593 | </script>
594 | </head>
595 | <body>
596 |   <div id="wrap">
597 |   <h2 style="text-align: center;"><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvNetJS</a> MNIST demo</h2>
598 |   <h1>Description</h1>
599 |   <p>
600 |     This demo trains a Convolutional Neural Network on the <a href="http://yann.lecun.com/exdb/mnist/">MNIST digits dataset</a> in your browser, with nothing but Javascript. The dataset is fairly easy and one should expect to get somewhere around 99% accuracy within few minutes. I used <a href="mnist_parse.zip">this python script</a> to parse the <a href="http://deeplearning.net/tutorial/gettingstarted.html">original files</a> into batches of images that can be easily loaded into page DOM with img tags.
601 |   </p>
602 |   <p>
603 |     This network takes a 28x28 MNIST image and crops a random 24x24 window before training on it (this technique is called data augmentation and improves generalization). Similarly to do prediction, 4 random crops are sampled and the probabilities across all crops are averaged to produce final predictions. The network runs at about 5ms for both forward and backward pass on my reasonably decent Ubuntu+Chrome machine.
604 |   </p>
605 |   <p>
606 |     By default, in this demo we're using Adadelta which is one of per-parameter adaptive step size methods, so we don't have to worry about changing learning rates or momentum over time. However, I still included the text fields for changing these if you'd like to play around with SGD+Momentum trainer.
607 |   </p>
608 |   <p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
609 |   <h1>Training Stats</h1>
610 |   <div class="divsec" style="270px;">
611 |     <div class="secpart">
612 |       <input id="buttontp" type="submit" value="pause" onclick="toggle_pause();" style="width: 100px; height:30px; background-color: #FCC;"/>
613 |       <div id="trainstats"></div>
614 | 
615 |       <div id="controls">
616 |         Learning rate: <input name="lri" type="text" maxlength="20" id="lr_input"/>
617 |         <input id="buttonlr" type="submit" value="change" onclick="change_lr();"/>
618 |         <br />
619 | 
620 |         Momentum: <input name="momi" type="text" maxlength="20" id="momentum_input"/>
621 |         <input id="buttonmom" type="submit" value="change" onclick="change_momentum();"/>
622 |         <br />
623 | 
624 |         Batch size: <input name="bsi" type="text" maxlength="20" id="batch_size_input"/>
625 |         <input id="buttonbs" type="submit" value="change" onclick="change_batch_size();"/>
626 |         <br />
627 | 
628 |         Weight decay: <input name="wdi" type="text" maxlength="20" id="decay_input"/>
629 |         <input id="buttonwd" type="submit" value="change" onclick="change_decay();"/>
630 |       </div>
631 | 
632 |       <input id="buttondj" type="submit" value="save network snapshot as JSON" onclick="dump_json();"/><br />
633 |       <input id="buttonlfj" type="submit" value="init network from JSON snapshot" onclick="load_from_json();"/><br />
634 |       <textarea id="dumpjson"></textarea>
635 |     </div>
636 |     <div class="secpart">
637 |       
638 |       <div>
639 |         Loss:<br />
640 |         <canvas id="lossgraph">
641 |         </canvas>
642 |         <br />
643 |         <input id="buttoncg" type="submit" value="clear graph" onclick="clear_graph();"/>
644 |       </div>
645 |     </div>
646 |     <div style="clear:both;"></div>
647 |   </div>
648 | 
649 |   <h1>Instantiate a Network and Trainer</h1>
650 |   <div>
651 |     <textarea id="newnet" style="width:100%; height:200px;"></textarea><br />
652 |     <input id="buttonnn" type="submit" value="change network" onclick="change_net();" style="width:200px;height:30px;"/>
653 |   </div>
654 | 
655 |   <div class="divsec">
656 |   <h1>Network Visualization</h1>
657 |     <div id="visnet"></div>
658 |   </div>
659 |   
660 |   <div class="divsec">
661 |   <h1>Example predictions on Test set</h1>
662 |     <div id="testset_acc"></div>
663 |   </div>
664 | 
665 |   </div>  
666 | </body>
667 | </html>
668 | 
669 | 
670 | 
671 | 


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_0.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_1.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_2.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_20.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_20.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_3.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_4.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_5.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_6.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_6.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_7.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_7.png


--------------------------------------------------------------------------------
/browser/mnist/mnist_batch_8.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/browser/mnist/mnist_batch_8.png


--------------------------------------------------------------------------------
/static/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "static",
 3 |   "private": true,
 4 |   "version": "0.0.0",
 5 |   "description": "Static server",
 6 |   "main": "server.js",
 7 |   "scripts": {
 8 |     "test": "echo \"Error: no test specified\" && exit 1",
 9 |     "start": "node server.js"
10 |   },
11 |   "author": "",
12 |   "license": "ISC",
13 |   "dependencies": {
14 |     "node-static": "~0.7.6"
15 |   }
16 | }
17 | 


--------------------------------------------------------------------------------
/static/public/cifar10.html:
--------------------------------------------------------------------------------
  1 | <!doctype html>
  2 | <html lang="en">
  3 | <head>
  4 |   <meta charset="utf-8">
  5 |   <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  6 |   <title>ConvNetJS CIFAR-10 demo</title>
  7 |   <meta name="description" content="">
  8 |   <meta name="author" content="">
  9 | 
 10 | <style>
 11 | .layer {
 12 |   border: 1px solid #999;
 13 |   margin-bottom: 5px;
 14 |   text-align: left;
 15 |   padding: 10px;
 16 | }
 17 | .layer_act {
 18 |   width: 500px;
 19 |   float: right;
 20 | }
 21 | .ltconv {
 22 |   background-color: #FDD;
 23 | }
 24 | .ltrelu {
 25 |   background-color: #FDF;
 26 | }
 27 | .ltpool {
 28 |   background-color: #DDF;
 29 | }
 30 | .ltsoftmax {
 31 |   background-color: #FFD;
 32 | }
 33 | .ltfc {
 34 |   background-color: #DFF;
 35 | }
 36 | .ltlrn {
 37 |   background-color: #DFD; 
 38 | }
 39 | .ltitle {
 40 |   color: #333;
 41 |   font-size: 18px;
 42 | }
 43 | .actmap {
 44 |   margin: 1px;
 45 | }
 46 | #trainstats {
 47 |   text-align: left;
 48 | }
 49 | .clear {
 50 |   clear: both;
 51 | }
 52 | #wrap {
 53 |   width: 800px;
 54 |   margin-left: auto;
 55 |   margin-right: auto;
 56 | }
 57 | h1 {
 58 |   font-size: 16px;
 59 |   color: #333;
 60 |   background-color: #DDD;
 61 |   border-bottom: 1px #999 solid;
 62 |   text-align: center;
 63 | }
 64 | .secpart {
 65 |   width: 400px;
 66 |   float: left;
 67 | }
 68 | #lossgraph {
 69 |   /*border: 1px solid #F0F;*/
 70 |   width: 100%;
 71 | }
 72 | .probsdiv canvas {
 73 |   float: left;
 74 | }
 75 | .probsdiv {
 76 |   height: 60px;
 77 |   width: 180px;
 78 |   display: inline-block;
 79 |   font-size: 12px;
 80 |   box-shadow: 0px 0px 2px 2px #EEE;
 81 |   margin: 5px;
 82 |   padding: 5px;
 83 |   color: black;
 84 | }
 85 | .pp {
 86 |   margin: 1px;
 87 |   padding: 1px;
 88 | }
 89 | #testset_vis {
 90 |   margin-bottom: 200px;
 91 | }
 92 | body {
 93 |   font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
 94 | }
 95 | </style>
 96 | 
 97 | <script src="js/jquery-1.8.3.min.js"></script>
 98 | <script src="js/vis.js"></script>
 99 | <script src="js/util.js"></script>
100 | <script src="js/convnet.js"></script>
101 | 
102 | <script src="js/cifar10_labels.js"></script>
103 | 
104 | <script>
105 | 
106 | var layer_defs, net, trainer;
107 | var t = "layer_defs = [];\n\
108 | layer_defs.push({type:'input', out_sx:32, out_sy:32, out_depth:3});\n\
109 | layer_defs.push({type:'conv', sx:5, filters:16, stride:1, pad:2, activation:'relu'});\n\
110 | layer_defs.push({type:'pool', sx:2, stride:2});\n\
111 | layer_defs.push({type:'conv', sx:5, filters:20, stride:1, pad:2, activation:'relu'});\n\
112 | layer_defs.push({type:'pool', sx:2, stride:2});\n\
113 | layer_defs.push({type:'conv', sx:5, filters:20, stride:1, pad:2, activation:'relu'});\n\
114 | layer_defs.push({type:'pool', sx:2, stride:2});\n\
115 | layer_defs.push({type:'softmax', num_classes:10});\n\
116 | \n\
117 | net = new convnetjs.Net();\n\
118 | net.makeLayers(layer_defs);\n\
119 | \n\
120 | trainer = new convnetjs.SGDTrainer(net, {method:'adadelta', batch_size:4, l2_decay:0.0001});\n\
121 | ";
122 | 
123 | // ------------------------
124 | // BEGIN MNIST SPECIFIC STUFF
125 | // ------------------------
126 | var classes_txt = ['airplane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'];
127 | 
128 | var use_validation_data = true;
129 | var sample_training_instance = function() {
130 | 
131 |   // find an unloaded batch
132 |   var bi = Math.floor(Math.random()*loaded_train_batches.length);
133 |   var b = loaded_train_batches[bi];
134 |   var k = Math.floor(Math.random()*1000); // sample within the batch
135 |   var n = b*1000+k;
136 | 
137 |   // load more batches over time
138 |   if(step_num%2000===0 && step_num>0) {
139 |     for(var i=0;i<num_batches;i++) {
140 |       if(!loaded[i]) {
141 |         // load it
142 |         load_data_batch(i);
143 |         break; // okay for now
144 |       }
145 |     }
146 |   }
147 | 
148 |   // fetch the appropriate row of the training image and reshape into a Vol
149 |   var p = img_data[b].data;
150 |   var x = new convnetjs.Vol(32,32,3,0.0);
151 |   var W = 32*32;
152 |   var j=0;
153 |   for(var dc=0;dc<3;dc++) {
154 |     var i=0;
155 |     for(var xc=0;xc<32;xc++) {
156 |       for(var yc=0;yc<32;yc++) {
157 |         var ix = ((W * k) + i) * 4 + dc;
158 |         x.set(yc,xc,dc,p[ix]/255.0-0.5);
159 |         i++;
160 |       }
161 |     }
162 |   }
163 |   var dx = Math.floor(Math.random()*5-2);
164 |   var dy = Math.floor(Math.random()*5-2);
165 |   x = convnetjs.augment(x, 32, dx, dy, Math.random()<0.5); //maybe flip horizontally
166 | 
167 |   var isval = use_validation_data && n%10===0 ? true : false;
168 |   return {x:x, label:labels[n], isval:isval};
169 | }
170 | 
171 | // sample a random testing instance
172 | var sample_test_instance = function() {
173 | 
174 |   var b = test_batch;
175 |   var k = Math.floor(Math.random()*1000);
176 |   var n = b*1000+k;
177 | 
178 |   var p = img_data[b].data;
179 |   var x = new convnetjs.Vol(32,32,3,0.0);
180 |   var W = 32*32;
181 |   var j=0;
182 |   for(var dc=0;dc<3;dc++) {
183 |     var i=0;
184 |     for(var xc=0;xc<32;xc++) {
185 |       for(var yc=0;yc<32;yc++) {
186 |         var ix = ((W * k) + i) * 4 + dc;
187 |         x.set(yc,xc,dc,p[ix]/255.0-0.5);
188 |         i++;
189 |       }
190 |     }
191 |   }
192 | 
193 |   // distort position and maybe flip
194 |   var xs = [];
195 |   //xs.push(x, 32, 0, 0, false); // push an un-augmented copy
196 |   for(var k=0;k<6;k++) {
197 |     var dx = Math.floor(Math.random()*5-2);
198 |     var dy = Math.floor(Math.random()*5-2);
199 |     xs.push(convnetjs.augment(x, 32, dx, dy, k>2));
200 |   }
201 |   
202 |   // return multiple augmentations, and we will average the network over them
203 |   // to increase performance
204 |   return {x:xs, label:labels[n]};
205 | }
206 | 
207 | var num_batches = 51; // 20 training batches, 1 test
208 | var test_batch = 50;
209 | var data_img_elts = new Array(num_batches);
210 | var img_data = new Array(num_batches);
211 | var loaded = new Array(num_batches);
212 | var loaded_train_batches = [];
213 | 
214 | // int main
215 | $(window).load(function() {
216 | 
217 |   $("#newnet").val(t);
218 |   eval($("#newnet").val());
219 |   
220 |   update_net_param_display();
221 | 
222 |   for(var k=0;k<loaded.length;k++) { loaded[k] = false; }
223 | 
224 |   load_data_batch(0); // async load train set batch 0 (6 total train batches)
225 |   load_data_batch(test_batch); // async load test set (batch 6)
226 |   start_fun();
227 | });
228 | 
229 | var start_fun = function() {
230 |   if(loaded[0] && loaded[test_batch]) { 
231 |     console.log('starting!'); 
232 |     setInterval(load_and_step, 0); // lets go!
233 |   }
234 |   else { setTimeout(start_fun, 200); } // keep checking
235 | }
236 | 
237 | var load_data_batch = function(batch_num) {
238 |   // Load the dataset with JS in background
239 |   data_img_elts[batch_num] = new Image();
240 |   var data_img_elt = data_img_elts[batch_num];
241 |   data_img_elt.onload = function() { 
242 |     var data_canvas = document.createElement('canvas');
243 |     data_canvas.width = data_img_elt.width;
244 |     data_canvas.height = data_img_elt.height;
245 |     var data_ctx = data_canvas.getContext("2d");
246 |     data_ctx.drawImage(data_img_elt, 0, 0); // copy it over... bit wasteful :(
247 |     img_data[batch_num] = data_ctx.getImageData(0, 0, data_canvas.width, data_canvas.height);
248 |     loaded[batch_num] = true;
249 |     if(batch_num < test_batch) { loaded_train_batches.push(batch_num); }
250 |     console.log('finished loading data batch ' + batch_num);
251 |   };
252 |   data_img_elt.src = "cifar10/cifar10_batch_" + batch_num + ".png";
253 | }
254 | 
255 | // ------------------------
256 | // END MNIST SPECIFIC STUFF
257 | // ------------------------
258 | 
259 | var maxmin = cnnutil.maxmin;
260 | var f2t = cnnutil.f2t;
261 | 
262 | // elt is the element to add all the canvas activation drawings into
263 | // A is the Vol() to use
264 | // scale is a multiplier to make the visualizations larger. Make higher for larger pictures
265 | // if grads is true then gradients are used instead
266 | var draw_activations = function(elt, A, scale, grads) {
267 | 
268 |   var s = scale || 2; // scale
269 |   var draw_grads = false;
270 |   if(typeof(grads) !== 'undefined') draw_grads = grads;
271 | 
272 |   // get max and min activation to scale the maps automatically
273 |   var w = draw_grads ? A.dw : A.w;
274 |   var mm = maxmin(w);
275 | 
276 |   // create the canvas elements, draw and add to DOM
277 |   for(var d=0;d<A.depth;d++) {
278 | 
279 |     var canv = document.createElement('canvas');
280 |     canv.className = 'actmap';
281 |     var W = A.sx * s;
282 |     var H = A.sy * s;
283 |     canv.width = W;
284 |     canv.height = H;
285 |     var ctx = canv.getContext('2d');
286 |     var g = ctx.createImageData(W, H);
287 | 
288 |     for(var x=0;x<A.sx;x++) {
289 |       for(var y=0;y<A.sy;y++) {
290 |         if(draw_grads) {
291 |           var dval = Math.floor((A.get_grad(x,y,d)-mm.minv)/mm.dv*255);
292 |         } else {
293 |           var dval = Math.floor((A.get(x,y,d)-mm.minv)/mm.dv*255);  
294 |         }
295 |         for(var dx=0;dx<s;dx++) {
296 |           for(var dy=0;dy<s;dy++) {
297 |             var pp = ((W * (y*s+dy)) + (dx + x*s)) * 4;
298 |             for(var i=0;i<3;i++) { g.data[pp + i] = dval; } // rgb
299 |             g.data[pp+3] = 255; // alpha channel
300 |           }
301 |         }
302 |       }
303 |     }
304 |     ctx.putImageData(g, 0, 0);
305 |     elt.appendChild(canv);
306 |   }  
307 | }
308 | 
309 | var draw_activations_COLOR = function(elt, A, scale, grads) {
310 | 
311 |   var s = scale || 2; // scale
312 |   var draw_grads = false;
313 |   if(typeof(grads) !== 'undefined') draw_grads = grads;
314 | 
315 |   // get max and min activation to scale the maps automatically
316 |   var w = draw_grads ? A.dw : A.w;
317 |   var mm = maxmin(w);
318 | 
319 |   var canv = document.createElement('canvas');
320 |   canv.className = 'actmap';
321 |   var W = A.sx * s;
322 |   var H = A.sy * s;
323 |   canv.width = W;
324 |   canv.height = H;
325 |   var ctx = canv.getContext('2d');
326 |   var g = ctx.createImageData(W, H);
327 |   for(var d=0;d<3;d++) {
328 |     for(var x=0;x<A.sx;x++) {
329 |       for(var y=0;y<A.sy;y++) {
330 |         if(draw_grads) {
331 |           var dval = Math.floor((A.get_grad(x,y,d)-mm.minv)/mm.dv*255);
332 |         } else {
333 |           var dval = Math.floor((A.get(x,y,d)-mm.minv)/mm.dv*255);  
334 |         }
335 |         for(var dx=0;dx<s;dx++) {
336 |           for(var dy=0;dy<s;dy++) {
337 |             var pp = ((W * (y*s+dy)) + (dx + x*s)) * 4;
338 |             g.data[pp + d] = dval;
339 |             if(d===0) g.data[pp+3] = 255; // alpha channel
340 |           }
341 |         }
342 |       }
343 |     }
344 |   }
345 |   ctx.putImageData(g, 0, 0);
346 |   elt.appendChild(canv);
347 | }
348 | 
349 | var visualize_activations = function(net, elt) {
350 | 
351 |   // clear the element
352 |   elt.innerHTML = "";
353 | 
354 |   // show activations in each layer
355 |   var N = net.layers.length;
356 |   for(var i=0;i<N;i++) {
357 |     var L = net.layers[i];
358 | 
359 |     var layer_div = document.createElement('div');
360 | 
361 |     // visualize activations
362 |     var activations_div = document.createElement('div');
363 |     activations_div.appendChild(document.createTextNode('Activations:'));
364 |     activations_div.appendChild(document.createElement('br'));
365 |     activations_div.className = 'layer_act';
366 |     var scale = 2;
367 |     if(L.layer_type==='softmax' || L.layer_type==='fc') scale = 10; // for softmax
368 |     
369 |     // HACK to draw in color in input layer
370 |     if(i===0) {
371 |       draw_activations_COLOR(activations_div, L.out_act, scale);
372 |       draw_activations_COLOR(activations_div, L.out_act, scale, true);
373 | 
374 |       /*
375 |       // visualize positive and negative components of the gradient separately
376 |       var dd = L.out_act.clone();
377 |       var ni = L.out_act.w.length;
378 |       for(var q=0;q<ni;q++) { var dwq = L.out_act.dw[q]; dd.w[q] = dwq > 0 ? dwq : 0.0; }
379 |       draw_activations_COLOR(activations_div, dd, scale);
380 |       for(var q=0;q<ni;q++) { var dwq = L.out_act.dw[q]; dd.w[q] = dwq < 0 ? -dwq : 0.0; }
381 |       draw_activations_COLOR(activations_div, dd, scale);
382 |       */
383 | 
384 |       /*
385 |       // visualize what the network would like the image to look like more
386 |       var dd = L.out_act.clone();
387 |       var ni = L.out_act.w.length;
388 |       for(var q=0;q<ni;q++) { var dwq = L.out_act.dw[q]; dd.w[q] -= 20*dwq; }
389 |       draw_activations_COLOR(activations_div, dd, scale);
390 |       */
391 | 
392 |       /*
393 |       // visualize gradient magnitude
394 |       var dd = L.out_act.clone();
395 |       var ni = L.out_act.w.length;
396 |       for(var q=0;q<ni;q++) { var dwq = L.out_act.dw[q]; dd.w[q] = dwq*dwq; }
397 |       draw_activations_COLOR(activations_div, dd, scale);
398 |       */
399 | 
400 |     } else {
401 |       draw_activations(activations_div, L.out_act, scale);
402 |     } 
403 | 
404 |     // visualize data gradients
405 |     if(L.layer_type !== 'softmax' && L.layer_type !== 'input' ) {
406 |       var grad_div = document.createElement('div');
407 |       grad_div.appendChild(document.createTextNode('Activation Gradients:'));
408 |       grad_div.appendChild(document.createElement('br'));
409 |       grad_div.className = 'layer_grad';
410 |       var scale = 2;
411 |       if(L.layer_type==='softmax' || L.layer_type==='fc') scale = 10; // for softmax
412 |       draw_activations(grad_div, L.out_act, scale, true);
413 |       activations_div.appendChild(grad_div);
414 |     }
415 | 
416 |     // visualize filters if they are of reasonable size
417 |     if(L.layer_type === 'conv') {
418 |       var filters_div = document.createElement('div');
419 |       if(L.filters[0].sx>3) {
420 |         // actual weights
421 |         filters_div.appendChild(document.createTextNode('Weights:'));
422 |         filters_div.appendChild(document.createElement('br'));
423 |         for(var j=0;j<L.filters.length;j++) {
424 |           // HACK to draw in color for first layer conv filters
425 |           if(i===1) {
426 |             draw_activations_COLOR(filters_div, L.filters[j], 2);
427 |           } else {
428 |             filters_div.appendChild(document.createTextNode('('));
429 |             draw_activations(filters_div, L.filters[j], 2);
430 |             filters_div.appendChild(document.createTextNode(')'));
431 |           }
432 |         }
433 |         // gradients
434 |         filters_div.appendChild(document.createElement('br'));
435 |         filters_div.appendChild(document.createTextNode('Weight Gradients:'));
436 |         filters_div.appendChild(document.createElement('br'));
437 |         for(var j=0;j<L.filters.length;j++) {
438 |           if(i===1) { draw_activations_COLOR(filters_div, L.filters[j], 2, true); }
439 |           else {
440 |             filters_div.appendChild(document.createTextNode('('));
441 |             draw_activations(filters_div, L.filters[j], 2, true);
442 |             filters_div.appendChild(document.createTextNode(')'));
443 |           }
444 |         }
445 |       } else {
446 |         filters_div.appendChild(document.createTextNode('Weights hidden, too small'));
447 |       }
448 |       activations_div.appendChild(filters_div);
449 |     }
450 |     layer_div.appendChild(activations_div);
451 | 
452 |     // print some stats on left of the layer
453 |     layer_div.className = 'layer ' + 'lt' + L.layer_type;
454 |     var title_div = document.createElement('div');
455 |     title_div.className = 'ltitle'
456 |     var t = L.layer_type + ' (' + L.out_sx + 'x' + L.out_sy + 'x' + L.out_depth + ')';
457 |     title_div.appendChild(document.createTextNode(t));
458 |     layer_div.appendChild(title_div);
459 | 
460 |     if(L.layer_type==='conv') {
461 |       var t = 'filter size ' + L.filters[0].sx + 'x' + L.filters[0].sy + 'x' + L.filters[0].depth + ', stride ' + L.stride;
462 |       layer_div.appendChild(document.createTextNode(t));
463 |       layer_div.appendChild(document.createElement('br'));
464 |     }
465 |     if(L.layer_type==='pool') {
466 |       var t = 'pooling size ' + L.sx + 'x' + L.sy + ', stride ' + L.stride;
467 |       layer_div.appendChild(document.createTextNode(t));
468 |       layer_div.appendChild(document.createElement('br'));
469 |     }
470 | 
471 |     // find min, max activations and display them
472 |     var mma = maxmin(L.out_act.w);
473 |     var t = 'max activation: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
474 |     layer_div.appendChild(document.createTextNode(t));
475 |     layer_div.appendChild(document.createElement('br'));
476 | 
477 |     var mma = maxmin(L.out_act.dw);
478 |     var t = 'max gradient: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
479 |     layer_div.appendChild(document.createTextNode(t));
480 |     layer_div.appendChild(document.createElement('br'));
481 | 
482 |     // number of parameters
483 |     if(L.layer_type==='conv' || L.layer_type==='local') {
484 |       var tot_params = L.sx*L.sy*L.in_depth*L.filters.length + L.filters.length;
485 |       var t = 'parameters: ' + L.filters.length + 'x' + L.sx + 'x' + L.sy + 'x' + L.in_depth + '+' + L.filters.length + ' = ' + tot_params;
486 |       layer_div.appendChild(document.createTextNode(t));
487 |       layer_div.appendChild(document.createElement('br'));
488 |     }
489 |     if(L.layer_type==='fc') {
490 |       var tot_params = L.num_inputs*L.filters.length + L.filters.length;
491 |       var t = 'parameters: ' + L.filters.length + 'x' + L.num_inputs + '+' + L.filters.length + ' = ' + tot_params;
492 |       layer_div.appendChild(document.createTextNode(t));
493 |       layer_div.appendChild(document.createElement('br'));
494 |     }
495 | 
496 |     // css madness needed here...
497 |     var clear = document.createElement('div');
498 |     clear.className = 'clear';
499 |     layer_div.appendChild(clear);
500 | 
501 |     elt.appendChild(layer_div);
502 |   }
503 | }
504 | 
505 | // loads a training image and trains on it with the network
506 | var paused = false;
507 | var load_and_step = function() {
508 |   if(paused) return; 
509 | 
510 |   var sample = sample_training_instance();
511 |   step(sample); // process this image
512 |   
513 |   //setTimeout(load_and_step, 0); // schedule the next iteration
514 | }
515 | 
516 | // evaluate current network on test set
517 | var test_predict = function() {
518 |   var num_classes = net.layers[net.layers.length-1].out_depth;
519 | 
520 |   document.getElementById('testset_acc').innerHTML = '';
521 |   var num_total = 0;
522 |   var num_correct = 0;
523 | 
524 |   // grab a random test image
525 |   for(num=0;num<4;num++) {
526 |     var sample = sample_test_instance();
527 |     var y = sample.label;  // ground truth label
528 | 
529 |     // forward prop it through the network
530 |     var aavg = new convnetjs.Vol(1,1,num_classes,0.0);
531 |     // ensures we always have a list, regardless if above returns single item or list
532 |     var xs = [].concat(sample.x);
533 |     var n = xs.length;
534 |     for(var i=0;i<n;i++) {
535 |       var a = net.forward(xs[i]);
536 |       aavg.addFrom(a);
537 |     }
538 |     var preds = [];
539 |     for(var k=0;k<aavg.w.length;k++) { preds.push({k:k,p:aavg.w[k]}); }
540 |     preds.sort(function(a,b){return a.p<b.p ? 1:-1;});
541 |     
542 |     var correct = preds[0].k===y;
543 |     if(correct) num_correct++;
544 |     num_total++;
545 | 
546 |     var div = document.createElement('div');
547 |     div.className = 'testdiv';
548 | 
549 |     // draw the image into a canvas
550 |     draw_activations_COLOR(div, xs[0], 2); // draw Vol into canv
551 | 
552 |     // add predictions
553 |     var probsdiv = document.createElement('div');
554 |     div.className = 'probsdiv';
555 |     var t = '';
556 |     for(var k=0;k<3;k++) {
557 |       var col = preds[k].k===y ? 'rgb(85,187,85)' : 'rgb(187,85,85)';
558 |       t += '<div class=\"pp\" style=\"width:' + Math.floor(preds[k].p/n*100) + 'px; margin-left: 70px; background-color:' + col + ';\">' + classes_txt[preds[k].k] + '</div>'
559 |     }
560 |     probsdiv.innerHTML = t;
561 |     div.appendChild(probsdiv);
562 | 
563 |     // add it into DOM
564 |     $(div).prependTo($("#testset_vis")).hide().fadeIn('slow').slideDown('slow');
565 |     if($(".probsdiv").length>200) {
566 |       $("#testset_vis > .probsdiv").last().remove(); // pop to keep upper bound of shown items
567 |     }
568 |   }
569 |   testAccWindow.add(num_correct/num_total);
570 |   $("#testset_acc").text('test accuracy based on last 200 test images: ' + testAccWindow.get_average());  
571 | }
572 | 
573 | var lossGraph = new cnnvis.Graph();
574 | var xLossWindow = new cnnutil.Window(100);
575 | var wLossWindow = new cnnutil.Window(100);
576 | var trainAccWindow = new cnnutil.Window(100);
577 | var valAccWindow = new cnnutil.Window(100);
578 | var testAccWindow = new cnnutil.Window(50, 1);
579 | var step_num = 0;
580 | var step = function(sample) {
581 | 
582 |   var x = sample.x;
583 |   var y = sample.label;
584 | 
585 |   if(sample.isval) {
586 |     // use x to build our estimate of validation error
587 |     net.forward(x);
588 |     var yhat = net.getPrediction();
589 |     var val_acc = yhat === y ? 1.0 : 0.0;
590 |     valAccWindow.add(val_acc);
591 |     return; // get out
592 |   }
593 | 
594 |   // train on it with network
595 |   var stats = trainer.train(x, y);
596 |   var lossx = stats.cost_loss;
597 |   var lossw = stats.l2_decay_loss;
598 | 
599 |   // keep track of stats such as the average training error and loss
600 |   var yhat = net.getPrediction();
601 |   var train_acc = yhat === y ? 1.0 : 0.0;
602 |   xLossWindow.add(lossx);
603 |   wLossWindow.add(lossw);
604 |   trainAccWindow.add(train_acc);
605 | 
606 |   // visualize training status
607 |   var train_elt = document.getElementById("trainstats");
608 |   train_elt.innerHTML = '';
609 |   var t = 'Forward time per example: ' + stats.fwd_time + 'ms';
610 |   train_elt.appendChild(document.createTextNode(t));
611 |   train_elt.appendChild(document.createElement('br'));
612 |   var t = 'Backprop time per example: ' + stats.bwd_time + 'ms';
613 |   train_elt.appendChild(document.createTextNode(t));
614 |   train_elt.appendChild(document.createElement('br'));
615 |   var t = 'Classification loss: ' + f2t(xLossWindow.get_average());
616 |   train_elt.appendChild(document.createTextNode(t));
617 |   train_elt.appendChild(document.createElement('br'));
618 |   var t = 'L2 Weight decay loss: ' + f2t(wLossWindow.get_average());
619 |   train_elt.appendChild(document.createTextNode(t));
620 |   train_elt.appendChild(document.createElement('br'));
621 |   var t = 'Training accuracy: ' + f2t(trainAccWindow.get_average());
622 |   train_elt.appendChild(document.createTextNode(t));
623 |   train_elt.appendChild(document.createElement('br'));
624 |   var t = 'Validation accuracy: ' + f2t(valAccWindow.get_average());
625 |   train_elt.appendChild(document.createTextNode(t));
626 |   train_elt.appendChild(document.createElement('br'));
627 |   var t = 'Examples seen: ' + step_num;
628 |   train_elt.appendChild(document.createTextNode(t));
629 |   train_elt.appendChild(document.createElement('br'));
630 | 
631 |   // visualize activations
632 |   if(step_num % 100 === 0) {
633 |     var vis_elt = document.getElementById("visnet");
634 |     visualize_activations(net, vis_elt);
635 |   }
636 | 
637 |   // log progress to graph, (full loss)
638 |   if(step_num % 200 === 0) {
639 |     var xa = xLossWindow.get_average();
640 |     var xw = wLossWindow.get_average();
641 |     if(xa >= 0 && xw >= 0) { // if they are -1 it means not enough data was accumulated yet for estimates
642 |       lossGraph.add(step_num, xa + xw);
643 |       lossGraph.drawSelf(document.getElementById("lossgraph"));
644 |     }
645 |   }
646 | 
647 |   // run prediction on test set
648 |   if((step_num % 100 === 0 && step_num > 0) || step_num===100) {
649 |     test_predict();
650 |   }
651 |   step_num++;
652 | }
653 | 
654 | // user settings 
655 | var change_lr = function() {
656 |   trainer.learning_rate = parseFloat(document.getElementById("lr_input").value);
657 |   update_net_param_display();
658 | }
659 | var change_momentum = function() {
660 |   trainer.momentum = parseFloat(document.getElementById("momentum_input").value);
661 |   update_net_param_display();
662 | }
663 | var change_batch_size = function() {
664 |   trainer.batch_size = parseFloat(document.getElementById("batch_size_input").value);
665 |   update_net_param_display();
666 | }
667 | var change_decay = function() {
668 |   trainer.l2_decay = parseFloat(document.getElementById("decay_input").value);
669 |   update_net_param_display();
670 | }
671 | var update_net_param_display = function() {
672 |   document.getElementById('lr_input').value = trainer.learning_rate;
673 |   document.getElementById('momentum_input').value = trainer.momentum;
674 |   document.getElementById('batch_size_input').value = trainer.batch_size;
675 |   document.getElementById('decay_input').value = trainer.l2_decay;
676 | }
677 | var toggle_pause = function() {
678 |   paused = !paused;
679 |   var btn = document.getElementById('buttontp');
680 |   if(paused) { btn.value = 'resume' }
681 |   else { btn.value = 'pause'; }
682 | }
683 | var dump_json = function() {
684 |   document.getElementById("dumpjson").value = JSON.stringify(this.net.toJSON());
685 | }
686 | var clear_graph = function() {
687 |   lossGraph = new cnnvis.Graph(); // reinit graph too 
688 | }
689 | var reset_all = function() {
690 |   // reinit trainer
691 |   trainer = new convnetjs.SGDTrainer(net, {learning_rate:trainer.learning_rate, momentum:trainer.momentum, batch_size:trainer.batch_size, l2_decay:trainer.l2_decay});
692 |   update_net_param_display();
693 | 
694 |   // reinit windows that keep track of val/train accuracies
695 |   xLossWindow.reset();
696 |   wLossWindow.reset();
697 |   trainAccWindow.reset();
698 |   valAccWindow.reset();
699 |   testAccWindow.reset();
700 |   lossGraph = new cnnvis.Graph(); // reinit graph too
701 |   step_num = 0;
702 | }
703 | var load_from_json = function() {
704 |   var jsonString = document.getElementById("dumpjson").value;
705 |   var json = JSON.parse(jsonString);
706 |   net = new convnetjs.Net();
707 |   net.fromJSON(json);
708 |   reset_all();
709 | }
710 | 
711 | var load_pretrained = function() {
712 |   $.getJSON("cifar10_snapshot.json", function(json){
713 |     net = new convnetjs.Net();
714 |     net.fromJSON(json);
715 |     trainer.learning_rate = 0.0001;
716 |     trainer.momentum = 0.9;
717 |     trainer.batch_size = 2;
718 |     trainer.l2_decay = 0.00001;
719 |     reset_all();
720 |   });
721 | }
722 | 
723 | var change_net = function() {
724 |   eval($("#newnet").val());
725 |   reset_all();
726 | }
727 | 
728 | </script>
729 | 
730 | </head>
731 | <body>
732 |   <div id="wrap">
733 |   <h2 style="text-align: center;"><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvNetJS</a> CIFAR-10 demo</h2>
734 |   <h1>Description</h1>
735 |   <p>
736 |     This demo trains a Convolutional Neural Network on the <a href="http://www.cs.toronto.edu/~kriz/cifar.html">CIFAR-10 dataset</a> in your browser, with nothing but Javascript. The state of the art on this dataset is about 90% accuracy and human performance is at about 94% (not perfect as the dataset can be a bit ambiguous). I used <a href="cifar10_parse.zip">this python script</a> to parse the <a href="http://www.cs.toronto.edu/~kriz/cifar.html">original files</a> (python version) into batches of images that can be easily loaded into page DOM with img tags.
737 |   </p>
738 |   <p>This dataset is more difficult and it takes longer to train a network. Data augmentation includes random flipping and random image shifts by up to 2px horizontally and verically.</p>
739 |   <p>
740 |     By default, in this demo we're using Adadelta which is one of per-parameter adaptive step size methods, so we don't have to worry about changing learning rates or momentum over time. However, I still included the text fields for changing these if you'd like to play around with SGD+Momentum trainer.
741 |   </p>
742 |   <p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
743 |   <h1>Training Stats</h1>
744 |   <div class="divsec" style="270px;">
745 |     <div class="secpart">
746 |       <input id="buttontp" type="submit" value="pause" onclick="toggle_pause();"/>
747 |       <div id="trainstats"></div>
748 | 
749 |       <div id="controls">
750 |         Learning rate: <input name="lri" type="text" maxlength="20" id="lr_input"/>
751 |         <input id="buttonlr" type="submit" value="change" onclick="change_lr();"/>
752 |         <br />
753 | 
754 |         Momentum: <input name="momi" type="text" maxlength="20" id="momentum_input"/>
755 |         <input id="buttonmom" type="submit" value="change" onclick="change_momentum();"/>
756 |         <br />
757 | 
758 |         Batch size: <input name="bsi" type="text" maxlength="20" id="batch_size_input"/>
759 |         <input id="buttonbs" type="submit" value="change" onclick="change_batch_size();"/>
760 |         <br />
761 | 
762 |         Weight decay: <input name="wdi" type="text" maxlength="20" id="decay_input"/>
763 |         <input id="buttonwd" type="submit" value="change" onclick="change_decay();"/>
764 |       </div>
765 | 
766 |       <input id="buttondj" type="submit" value="save network snapshot as JSON" onclick="dump_json();"/><br />
767 |       <input id="buttonlfj" type="submit" value="init network from JSON snapshot" onclick="load_from_json();"/><br />
768 |       <textarea id="dumpjson"></textarea>
769 |       <br>
770 | 
771 |       <input id="buttonpre" type="submit" value="load a pretrained network (achieves ~80% accuracy)" onclick="load_pretrained();" style="height: 30px; width: 400px;"/><br />
772 | 
773 |     </div>
774 |     <div class="secpart">
775 |       
776 |       <div>
777 |         Loss:<br />
778 |         <canvas id="lossgraph">
779 |         </canvas>
780 |         <br />
781 |         <input id="buttoncg" type="submit" value="clear graph" onclick="clear_graph();"/>
782 |       </div>
783 |     </div>
784 |     <div style="clear:both;"></div>
785 |   </div>
786 | 
787 |   <h1>Instantiate a Network and Trainer</h1>
788 |   <div>
789 |     <textarea id="newnet" style="width:100%; height:200px;"></textarea><br />
790 |     <input id="buttonnn" type="submit" value="change network" onclick="change_net();" style="width:200px;height:30px;"/>
791 |   </div>
792 | 
793 |   <div class="divsec">
794 |   <h1>Network Visualization</h1>
795 |     <div id="visnet"></div>
796 |   </div>
797 |   
798 |   <div class="divsec">
799 |   <h1>Example predictions on Test set</h1>
800 |     <div id="testset_acc"></div>
801 |     <div id="testset_vis"></div>
802 |   </div>
803 | 
804 |   </div>  
805 | </body>
806 | </html>
807 | 
808 | 
809 | 
810 | 


--------------------------------------------------------------------------------
/static/public/cifar10/cifar10_batch_0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/cifar10/cifar10_batch_0.png


--------------------------------------------------------------------------------
/static/public/cifar10/cifar10_batch_50.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/cifar10/cifar10_batch_50.png


--------------------------------------------------------------------------------
/static/public/css/jquery-ui.min.css:
--------------------------------------------------------------------------------
1 | /*! jQuery UI - v1.10.4 - 2014-01-17
2 | * http://jqueryui.com
3 | * Includes: jquery.ui.core.css, jquery.ui.accordion.css, jquery.ui.autocomplete.css, jquery.ui.button.css, jquery.ui.datepicker.css, jquery.ui.dialog.css, jquery.ui.menu.css, jquery.ui.progressbar.css, jquery.ui.resizable.css, jquery.ui.selectable.css, jquery.ui.slider.css, jquery.ui.spinner.css, jquery.ui.tabs.css, jquery.ui.tooltip.css, jquery.ui.theme.css
4 | * Copyright 2014 jQuery Foundation and other contributors; Licensed MIT */ .ui-helper-hidden{display:none}.ui-helper-hidden-accessible{border:0;clip:rect(0 0 0 0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.ui-helper-reset{margin:0;padding:0;border:0;outline:0;line-height:1.3;text-decoration:none;font-size:100%;list-style:none}.ui-helper-clearfix:before,.ui-helper-clearfix:after{content:"";display:table;border-collapse:collapse}.ui-helper-clearfix:after{clear:both}.ui-helper-clearfix{min-height:0}.ui-helper-zfix{width:100%;height:100%;top:0;left:0;position:absolute;opacity:0;filter:Alpha(Opacity=0)}.ui-front{z-index:100}.ui-state-disabled{cursor:default!important}.ui-icon{display:block;text-indent:-99999px;overflow:hidden;background-repeat:no-repeat}.ui-widget-overlay{position:fixed;top:0;left:0;width:100%;height:100%}.ui-accordion .ui-accordion-header{display:block;cursor:pointer;position:relative;margin-top:2px;padding:.5em .5em .5em .7em;min-height:0}.ui-accordion .ui-accordion-icons{padding-left:2.2em}.ui-accordion .ui-accordion-noicons{padding-left:.7em}.ui-accordion .ui-accordion-icons .ui-accordion-icons{padding-left:2.2em}.ui-accordion .ui-accordion-header .ui-accordion-header-icon{position:absolute;left:.5em;top:50%;margin-top:-8px}.ui-accordion .ui-accordion-content{padding:1em 2.2em;border-top:0;overflow:auto}.ui-autocomplete{position:absolute;top:0;left:0;cursor:default}.ui-button{display:inline-block;position:relative;padding:0;line-height:normal;margin-right:.1em;cursor:pointer;vertical-align:middle;text-align:center;overflow:visible}.ui-button,.ui-button:link,.ui-button:visited,.ui-button:hover,.ui-button:active{text-decoration:none}.ui-button-icon-only{width:2.2em}button.ui-button-icon-only{width:2.4em}.ui-button-icons-only{width:3.4em}button.ui-button-icons-only{width:3.7em}.ui-button .ui-button-text{display:block;line-height:normal}.ui-button-text-only .ui-button-text{padding:.4em 1em}.ui-button-icon-only .ui-button-text,.ui-button-icons-only .ui-button-text{padding:.4em;text-indent:-9999999px}.ui-button-text-icon-primary .ui-button-text,.ui-button-text-icons .ui-button-text{padding:.4em 1em .4em 2.1em}.ui-button-text-icon-secondary .ui-button-text,.ui-button-text-icons .ui-button-text{padding:.4em 2.1em .4em 1em}.ui-button-text-icons .ui-button-text{padding-left:2.1em;padding-right:2.1em}input.ui-button{padding:.4em 1em}.ui-button-icon-only .ui-icon,.ui-button-text-icon-primary .ui-icon,.ui-button-text-icon-secondary .ui-icon,.ui-button-text-icons .ui-icon,.ui-button-icons-only .ui-icon{position:absolute;top:50%;margin-top:-8px}.ui-button-icon-only .ui-icon{left:50%;margin-left:-8px}.ui-button-text-icon-primary .ui-button-icon-primary,.ui-button-text-icons .ui-button-icon-primary,.ui-button-icons-only .ui-button-icon-primary{left:.5em}.ui-button-text-icon-secondary .ui-button-icon-secondary,.ui-button-text-icons .ui-button-icon-secondary,.ui-button-icons-only .ui-button-icon-secondary{right:.5em}.ui-buttonset{margin-right:7px}.ui-buttonset .ui-button{margin-left:0;margin-right:-.3em}input.ui-button::-moz-focus-inner,button.ui-button::-moz-focus-inner{border:0;padding:0}.ui-datepicker{width:17em;padding:.2em .2em 0;display:none}.ui-datepicker .ui-datepicker-header{position:relative;padding:.2em 0}.ui-datepicker .ui-datepicker-prev,.ui-datepicker .ui-datepicker-next{position:absolute;top:2px;width:1.8em;height:1.8em}.ui-datepicker .ui-datepicker-prev-hover,.ui-datepicker .ui-datepicker-next-hover{top:1px}.ui-datepicker .ui-datepicker-prev{left:2px}.ui-datepicker .ui-datepicker-next{right:2px}.ui-datepicker .ui-datepicker-prev-hover{left:1px}.ui-datepicker .ui-datepicker-next-hover{right:1px}.ui-datepicker .ui-datepicker-prev span,.ui-datepicker .ui-datepicker-next span{display:block;position:absolute;left:50%;margin-left:-8px;top:50%;margin-top:-8px}.ui-datepicker .ui-datepicker-title{margin:0 2.3em;line-height:1.8em;text-align:center}.ui-datepicker .ui-datepicker-title select{font-size:1em;margin:1px 0}.ui-datepicker select.ui-datepicker-month,.ui-datepicker select.ui-datepicker-year{width:49%}.ui-datepicker table{width:100%;font-size:.9em;border-collapse:collapse;margin:0 0 .4em}.ui-datepicker th{padding:.7em .3em;text-align:center;font-weight:700;border:0}.ui-datepicker td{border:0;padding:1px}.ui-datepicker td span,.ui-datepicker td a{display:block;padding:.2em;text-align:right;text-decoration:none}.ui-datepicker .ui-datepicker-buttonpane{background-image:none;margin:.7em 0 0;padding:0 .2em;border-left:0;border-right:0;border-bottom:0}.ui-datepicker .ui-datepicker-buttonpane button{float:right;margin:.5em .2em .4em;cursor:pointer;padding:.2em .6em .3em;width:auto;overflow:visible}.ui-datepicker .ui-datepicker-buttonpane button.ui-datepicker-current{float:left}.ui-datepicker.ui-datepicker-multi{width:auto}.ui-datepicker-multi .ui-datepicker-group{float:left}.ui-datepicker-multi .ui-datepicker-group table{width:95%;margin:0 auto .4em}.ui-datepicker-multi-2 .ui-datepicker-group{width:50%}.ui-datepicker-multi-3 .ui-datepicker-group{width:33.3%}.ui-datepicker-multi-4 .ui-datepicker-group{width:25%}.ui-datepicker-multi .ui-datepicker-group-last .ui-datepicker-header,.ui-datepicker-multi .ui-datepicker-group-middle .ui-datepicker-header{border-left-width:0}.ui-datepicker-multi .ui-datepicker-buttonpane{clear:left}.ui-datepicker-row-break{clear:both;width:100%;font-size:0}.ui-datepicker-rtl{direction:rtl}.ui-datepicker-rtl .ui-datepicker-prev{right:2px;left:auto}.ui-datepicker-rtl .ui-datepicker-next{left:2px;right:auto}.ui-datepicker-rtl .ui-datepicker-prev:hover{right:1px;left:auto}.ui-datepicker-rtl .ui-datepicker-next:hover{left:1px;right:auto}.ui-datepicker-rtl .ui-datepicker-buttonpane{clear:right}.ui-datepicker-rtl .ui-datepicker-buttonpane button{float:left}.ui-datepicker-rtl .ui-datepicker-buttonpane button.ui-datepicker-current,.ui-datepicker-rtl .ui-datepicker-group{float:right}.ui-datepicker-rtl .ui-datepicker-group-last .ui-datepicker-header,.ui-datepicker-rtl .ui-datepicker-group-middle .ui-datepicker-header{border-right-width:0;border-left-width:1px}.ui-dialog{overflow:hidden;position:absolute;top:0;left:0;padding:.2em;outline:0}.ui-dialog .ui-dialog-titlebar{padding:.4em 1em;position:relative}.ui-dialog .ui-dialog-title{float:left;margin:.1em 0;white-space:nowrap;width:90%;overflow:hidden;text-overflow:ellipsis}.ui-dialog .ui-dialog-titlebar-close{position:absolute;right:.3em;top:50%;width:20px;margin:-10px 0 0 0;padding:1px;height:20px}.ui-dialog .ui-dialog-content{position:relative;border:0;padding:.5em 1em;background:0;overflow:auto}.ui-dialog .ui-dialog-buttonpane{text-align:left;border-width:1px 0 0;background-image:none;margin-top:.5em;padding:.3em 1em .5em .4em}.ui-dialog .ui-dialog-buttonpane .ui-dialog-buttonset{float:right}.ui-dialog .ui-dialog-buttonpane button{margin:.5em .4em .5em 0;cursor:pointer}.ui-dialog .ui-resizable-se{width:12px;height:12px;right:-5px;bottom:-5px;background-position:16px 16px}.ui-draggable .ui-dialog-titlebar{cursor:move}.ui-menu{list-style:none;padding:2px;margin:0;display:block;outline:0}.ui-menu .ui-menu{margin-top:-3px;position:absolute}.ui-menu .ui-menu-item{margin:0;padding:0;width:100%;list-style-image:url(data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7)}.ui-menu .ui-menu-divider{margin:5px -2px 5px -2px;height:0;font-size:0;line-height:0;border-width:1px 0 0}.ui-menu .ui-menu-item a{text-decoration:none;display:block;padding:2px .4em;line-height:1.5;min-height:0;font-weight:400}.ui-menu .ui-menu-item a.ui-state-focus,.ui-menu .ui-menu-item a.ui-state-active{font-weight:400;margin:-1px}.ui-menu .ui-state-disabled{font-weight:400;margin:.4em 0 .2em;line-height:1.5}.ui-menu .ui-state-disabled a{cursor:default}.ui-menu-icons{position:relative}.ui-menu-icons .ui-menu-item a{position:relative;padding-left:2em}.ui-menu .ui-icon{position:absolute;top:.2em;left:.2em}.ui-menu .ui-menu-icon{position:static;float:right}.ui-progressbar{height:2em;text-align:left;overflow:hidden}.ui-progressbar .ui-progressbar-value{margin:-1px;height:100%}.ui-progressbar .ui-progressbar-overlay{background:url(images/animated-overlay.gif);height:100%;filter:alpha(opacity=25);opacity:.25}.ui-progressbar-indeterminate .ui-progressbar-value{background-image:none}.ui-resizable{position:relative}.ui-resizable-handle{position:absolute;font-size:.1px;display:block}.ui-resizable-disabled .ui-resizable-handle,.ui-resizable-autohide .ui-resizable-handle{display:none}.ui-resizable-n{cursor:n-resize;height:7px;width:100%;top:-5px;left:0}.ui-resizable-s{cursor:s-resize;height:7px;width:100%;bottom:-5px;left:0}.ui-resizable-e{cursor:e-resize;width:7px;right:-5px;top:0;height:100%}.ui-resizable-w{cursor:w-resize;width:7px;left:-5px;top:0;height:100%}.ui-resizable-se{cursor:se-resize;width:12px;height:12px;right:1px;bottom:1px}.ui-resizable-sw{cursor:sw-resize;width:9px;height:9px;left:-5px;bottom:-5px}.ui-resizable-nw{cursor:nw-resize;width:9px;height:9px;left:-5px;top:-5px}.ui-resizable-ne{cursor:ne-resize;width:9px;height:9px;right:-5px;top:-5px}.ui-selectable-helper{position:absolute;z-index:100;border:1px dotted #000}.ui-slider{position:relative;text-align:left}.ui-slider .ui-slider-handle{position:absolute;z-index:2;width:1.2em;height:1.2em;cursor:default}.ui-slider .ui-slider-range{position:absolute;z-index:1;font-size:.7em;display:block;border:0;background-position:0 0}.ui-slider.ui-state-disabled .ui-slider-handle,.ui-slider.ui-state-disabled .ui-slider-range{filter:inherit}.ui-slider-horizontal{height:.8em}.ui-slider-horizontal .ui-slider-handle{top:-.3em;margin-left:-.6em}.ui-slider-horizontal .ui-slider-range{top:0;height:100%}.ui-slider-horizontal .ui-slider-range-min{left:0}.ui-slider-horizontal .ui-slider-range-max{right:0}.ui-slider-vertical{width:.8em;height:100px}.ui-slider-vertical .ui-slider-handle{left:-.3em;margin-left:0;margin-bottom:-.6em}.ui-slider-vertical .ui-slider-range{left:0;width:100%}.ui-slider-vertical .ui-slider-range-min{bottom:0}.ui-slider-vertical .ui-slider-range-max{top:0}.ui-spinner{position:relative;display:inline-block;overflow:hidden;padding:0;vertical-align:middle}.ui-spinner-input{border:0;background:0;color:inherit;padding:0;margin:.2em 0;vertical-align:middle;margin-left:.4em;margin-right:22px}.ui-spinner-button{width:16px;height:50%;font-size:.5em;padding:0;margin:0;text-align:center;position:absolute;cursor:default;display:block;overflow:hidden;right:0}.ui-spinner a.ui-spinner-button{border-top:0;border-bottom:0;border-right:0}.ui-spinner .ui-icon{position:absolute;margin-top:-8px;top:50%;left:0}.ui-spinner-up{top:0}.ui-spinner-down{bottom:0}.ui-spinner .ui-icon-triangle-1-s{background-position:-65px -16px}.ui-tabs{position:relative;padding:.2em}.ui-tabs .ui-tabs-nav{margin:0;padding:.2em .2em 0}.ui-tabs .ui-tabs-nav li{list-style:none;float:left;position:relative;top:0;margin:1px .2em 0 0;border-bottom-width:0;padding:0;white-space:nowrap}.ui-tabs .ui-tabs-nav .ui-tabs-anchor{float:left;padding:.5em 1em;text-decoration:none}.ui-tabs .ui-tabs-nav li.ui-tabs-active{margin-bottom:-1px;padding-bottom:1px}.ui-tabs .ui-tabs-nav li.ui-tabs-active .ui-tabs-anchor,.ui-tabs .ui-tabs-nav li.ui-state-disabled .ui-tabs-anchor,.ui-tabs .ui-tabs-nav li.ui-tabs-loading .ui-tabs-anchor{cursor:text}.ui-tabs-collapsible .ui-tabs-nav li.ui-tabs-active .ui-tabs-anchor{cursor:pointer}.ui-tabs .ui-tabs-panel{display:block;border-width:0;padding:1em 1.4em;background:0}.ui-tooltip{padding:8px;position:absolute;z-index:9999;max-width:300px;-webkit-box-shadow:0 0 5px #aaa;box-shadow:0 0 5px #aaa}body .ui-tooltip{border-width:2px}.ui-widget{font-family:Verdana,Arial,sans-serif;font-size:1.1em}.ui-widget .ui-widget{font-size:1em}.ui-widget input,.ui-widget select,.ui-widget textarea,.ui-widget button{font-family:Verdana,Arial,sans-serif;font-size:1em}.ui-widget-content{border:1px solid #aaa;background:#fff url(images/ui-bg_flat_75_ffffff_40x100.png) 50% 50% repeat-x;color:#222}.ui-widget-content a{color:#222}.ui-widget-header{border:1px solid #aaa;background:#ccc url(images/ui-bg_highlight-soft_75_cccccc_1x100.png) 50% 50% repeat-x;color:#222;font-weight:700}.ui-widget-header a{color:#222}.ui-state-default,.ui-widget-content .ui-state-default,.ui-widget-header .ui-state-default{border:1px solid #d3d3d3;background:#e6e6e6 url(images/ui-bg_glass_75_e6e6e6_1x400.png) 50% 50% repeat-x;font-weight:400;color:#555}.ui-state-default a,.ui-state-default a:link,.ui-state-default a:visited{color:#555;text-decoration:none}.ui-state-hover,.ui-widget-content .ui-state-hover,.ui-widget-header .ui-state-hover,.ui-state-focus,.ui-widget-content .ui-state-focus,.ui-widget-header .ui-state-focus{border:1px solid #999;background:#dadada url(images/ui-bg_glass_75_dadada_1x400.png) 50% 50% repeat-x;font-weight:400;color:#212121}.ui-state-hover a,.ui-state-hover a:hover,.ui-state-hover a:link,.ui-state-hover a:visited,.ui-state-focus a,.ui-state-focus a:hover,.ui-state-focus a:link,.ui-state-focus a:visited{color:#212121;text-decoration:none}.ui-state-active,.ui-widget-content .ui-state-active,.ui-widget-header .ui-state-active{border:1px solid #aaa;background:#fff url(images/ui-bg_glass_65_ffffff_1x400.png) 50% 50% repeat-x;font-weight:400;color:#212121}.ui-state-active a,.ui-state-active a:link,.ui-state-active a:visited{color:#212121;text-decoration:none}.ui-state-highlight,.ui-widget-content .ui-state-highlight,.ui-widget-header .ui-state-highlight{border:1px solid #fcefa1;background:#fbf9ee url(images/ui-bg_glass_55_fbf9ee_1x400.png) 50% 50% repeat-x;color:#363636}.ui-state-highlight a,.ui-widget-content .ui-state-highlight a,.ui-widget-header .ui-state-highlight a{color:#363636}.ui-state-error,.ui-widget-content .ui-state-error,.ui-widget-header .ui-state-error{border:1px solid #cd0a0a;background:#fef1ec url(images/ui-bg_glass_95_fef1ec_1x400.png) 50% 50% repeat-x;color:#cd0a0a}.ui-state-error a,.ui-widget-content .ui-state-error a,.ui-widget-header .ui-state-error a{color:#cd0a0a}.ui-state-error-text,.ui-widget-content .ui-state-error-text,.ui-widget-header .ui-state-error-text{color:#cd0a0a}.ui-priority-primary,.ui-widget-content .ui-priority-primary,.ui-widget-header .ui-priority-primary{font-weight:700}.ui-priority-secondary,.ui-widget-content .ui-priority-secondary,.ui-widget-header .ui-priority-secondary{opacity:.7;filter:Alpha(Opacity=70);font-weight:400}.ui-state-disabled,.ui-widget-content .ui-state-disabled,.ui-widget-header .ui-state-disabled{opacity:.35;filter:Alpha(Opacity=35);background-image:none}.ui-state-disabled .ui-icon{filter:Alpha(Opacity=35)}.ui-icon{width:16px;height:16px}.ui-icon,.ui-widget-content .ui-icon{background-image:url(images/ui-icons_222222_256x240.png)}.ui-widget-header .ui-icon{background-image:url(images/ui-icons_222222_256x240.png)}.ui-state-default .ui-icon{background-image:url(images/ui-icons_888888_256x240.png)}.ui-state-hover .ui-icon,.ui-state-focus .ui-icon{background-image:url(images/ui-icons_454545_256x240.png)}.ui-state-active .ui-icon{background-image:url(images/ui-icons_454545_256x240.png)}.ui-state-highlight .ui-icon{background-image:url(images/ui-icons_2e83ff_256x240.png)}.ui-state-error .ui-icon,.ui-state-error-text .ui-icon{background-image:url(images/ui-icons_cd0a0a_256x240.png)}.ui-icon-blank{background-position:16px 16px}.ui-icon-carat-1-n{background-position:0 0}.ui-icon-carat-1-ne{background-position:-16px 0}.ui-icon-carat-1-e{background-position:-32px 0}.ui-icon-carat-1-se{background-position:-48px 0}.ui-icon-carat-1-s{background-position:-64px 0}.ui-icon-carat-1-sw{background-position:-80px 0}.ui-icon-carat-1-w{background-position:-96px 0}.ui-icon-carat-1-nw{background-position:-112px 0}.ui-icon-carat-2-n-s{background-position:-128px 0}.ui-icon-carat-2-e-w{background-position:-144px 0}.ui-icon-triangle-1-n{background-position:0 -16px}.ui-icon-triangle-1-ne{background-position:-16px -16px}.ui-icon-triangle-1-e{background-position:-32px -16px}.ui-icon-triangle-1-se{background-position:-48px -16px}.ui-icon-triangle-1-s{background-position:-64px -16px}.ui-icon-triangle-1-sw{background-position:-80px -16px}.ui-icon-triangle-1-w{background-position:-96px -16px}.ui-icon-triangle-1-nw{background-position:-112px -16px}.ui-icon-triangle-2-n-s{background-position:-128px -16px}.ui-icon-triangle-2-e-w{background-position:-144px -16px}.ui-icon-arrow-1-n{background-position:0 -32px}.ui-icon-arrow-1-ne{background-position:-16px -32px}.ui-icon-arrow-1-e{background-position:-32px -32px}.ui-icon-arrow-1-se{background-position:-48px -32px}.ui-icon-arrow-1-s{background-position:-64px -32px}.ui-icon-arrow-1-sw{background-position:-80px -32px}.ui-icon-arrow-1-w{background-position:-96px -32px}.ui-icon-arrow-1-nw{background-position:-112px -32px}.ui-icon-arrow-2-n-s{background-position:-128px -32px}.ui-icon-arrow-2-ne-sw{background-position:-144px -32px}.ui-icon-arrow-2-e-w{background-position:-160px -32px}.ui-icon-arrow-2-se-nw{background-position:-176px -32px}.ui-icon-arrowstop-1-n{background-position:-192px -32px}.ui-icon-arrowstop-1-e{background-position:-208px -32px}.ui-icon-arrowstop-1-s{background-position:-224px -32px}.ui-icon-arrowstop-1-w{background-position:-240px -32px}.ui-icon-arrowthick-1-n{background-position:0 -48px}.ui-icon-arrowthick-1-ne{background-position:-16px -48px}.ui-icon-arrowthick-1-e{background-position:-32px -48px}.ui-icon-arrowthick-1-se{background-position:-48px -48px}.ui-icon-arrowthick-1-s{background-position:-64px -48px}.ui-icon-arrowthick-1-sw{background-position:-80px -48px}.ui-icon-arrowthick-1-w{background-position:-96px -48px}.ui-icon-arrowthick-1-nw{background-position:-112px -48px}.ui-icon-arrowthick-2-n-s{background-position:-128px -48px}.ui-icon-arrowthick-2-ne-sw{background-position:-144px -48px}.ui-icon-arrowthick-2-e-w{background-position:-160px -48px}.ui-icon-arrowthick-2-se-nw{background-position:-176px -48px}.ui-icon-arrowthickstop-1-n{background-position:-192px -48px}.ui-icon-arrowthickstop-1-e{background-position:-208px -48px}.ui-icon-arrowthickstop-1-s{background-position:-224px -48px}.ui-icon-arrowthickstop-1-w{background-position:-240px -48px}.ui-icon-arrowreturnthick-1-w{background-position:0 -64px}.ui-icon-arrowreturnthick-1-n{background-position:-16px -64px}.ui-icon-arrowreturnthick-1-e{background-position:-32px -64px}.ui-icon-arrowreturnthick-1-s{background-position:-48px -64px}.ui-icon-arrowreturn-1-w{background-position:-64px -64px}.ui-icon-arrowreturn-1-n{background-position:-80px -64px}.ui-icon-arrowreturn-1-e{background-position:-96px -64px}.ui-icon-arrowreturn-1-s{background-position:-112px -64px}.ui-icon-arrowrefresh-1-w{background-position:-128px -64px}.ui-icon-arrowrefresh-1-n{background-position:-144px -64px}.ui-icon-arrowrefresh-1-e{background-position:-160px -64px}.ui-icon-arrowrefresh-1-s{background-position:-176px -64px}.ui-icon-arrow-4{background-position:0 -80px}.ui-icon-arrow-4-diag{background-position:-16px -80px}.ui-icon-extlink{background-position:-32px -80px}.ui-icon-newwin{background-position:-48px -80px}.ui-icon-refresh{background-position:-64px -80px}.ui-icon-shuffle{background-position:-80px -80px}.ui-icon-transfer-e-w{background-position:-96px -80px}.ui-icon-transferthick-e-w{background-position:-112px -80px}.ui-icon-folder-collapsed{background-position:0 -96px}.ui-icon-folder-open{background-position:-16px -96px}.ui-icon-document{background-position:-32px -96px}.ui-icon-document-b{background-position:-48px -96px}.ui-icon-note{background-position:-64px -96px}.ui-icon-mail-closed{background-position:-80px -96px}.ui-icon-mail-open{background-position:-96px -96px}.ui-icon-suitcase{background-position:-112px -96px}.ui-icon-comment{background-position:-128px -96px}.ui-icon-person{background-position:-144px -96px}.ui-icon-print{background-position:-160px -96px}.ui-icon-trash{background-position:-176px -96px}.ui-icon-locked{background-position:-192px -96px}.ui-icon-unlocked{background-position:-208px -96px}.ui-icon-bookmark{background-position:-224px -96px}.ui-icon-tag{background-position:-240px -96px}.ui-icon-home{background-position:0 -112px}.ui-icon-flag{background-position:-16px -112px}.ui-icon-calendar{background-position:-32px -112px}.ui-icon-cart{background-position:-48px -112px}.ui-icon-pencil{background-position:-64px -112px}.ui-icon-clock{background-position:-80px -112px}.ui-icon-disk{background-position:-96px -112px}.ui-icon-calculator{background-position:-112px -112px}.ui-icon-zoomin{background-position:-128px -112px}.ui-icon-zoomout{background-position:-144px -112px}.ui-icon-search{background-position:-160px -112px}.ui-icon-wrench{background-position:-176px -112px}.ui-icon-gear{background-position:-192px -112px}.ui-icon-heart{background-position:-208px -112px}.ui-icon-star{background-position:-224px -112px}.ui-icon-link{background-position:-240px -112px}.ui-icon-cancel{background-position:0 -128px}.ui-icon-plus{background-position:-16px -128px}.ui-icon-plusthick{background-position:-32px -128px}.ui-icon-minus{background-position:-48px -128px}.ui-icon-minusthick{background-position:-64px -128px}.ui-icon-close{background-position:-80px -128px}.ui-icon-closethick{background-position:-96px -128px}.ui-icon-key{background-position:-112px -128px}.ui-icon-lightbulb{background-position:-128px -128px}.ui-icon-scissors{background-position:-144px -128px}.ui-icon-clipboard{background-position:-160px -128px}.ui-icon-copy{background-position:-176px -128px}.ui-icon-contact{background-position:-192px -128px}.ui-icon-image{background-position:-208px -128px}.ui-icon-video{background-position:-224px -128px}.ui-icon-script{background-position:-240px -128px}.ui-icon-alert{background-position:0 -144px}.ui-icon-info{background-position:-16px -144px}.ui-icon-notice{background-position:-32px -144px}.ui-icon-help{background-position:-48px -144px}.ui-icon-check{background-position:-64px -144px}.ui-icon-bullet{background-position:-80px -144px}.ui-icon-radio-on{background-position:-96px -144px}.ui-icon-radio-off{background-position:-112px -144px}.ui-icon-pin-w{background-position:-128px -144px}.ui-icon-pin-s{background-position:-144px -144px}.ui-icon-play{background-position:0 -160px}.ui-icon-pause{background-position:-16px -160px}.ui-icon-seek-next{background-position:-32px -160px}.ui-icon-seek-prev{background-position:-48px -160px}.ui-icon-seek-end{background-position:-64px -160px}.ui-icon-seek-start{background-position:-80px -160px}.ui-icon-seek-first{background-position:-80px -160px}.ui-icon-stop{background-position:-96px -160px}.ui-icon-eject{background-position:-112px -160px}.ui-icon-volume-off{background-position:-128px -160px}.ui-icon-volume-on{background-position:-144px -160px}.ui-icon-power{background-position:0 -176px}.ui-icon-signal-diag{background-position:-16px -176px}.ui-icon-signal{background-position:-32px -176px}.ui-icon-battery-0{background-position:-48px -176px}.ui-icon-battery-1{background-position:-64px -176px}.ui-icon-battery-2{background-position:-80px -176px}.ui-icon-battery-3{background-position:-96px -176px}.ui-icon-circle-plus{background-position:0 -192px}.ui-icon-circle-minus{background-position:-16px -192px}.ui-icon-circle-close{background-position:-32px -192px}.ui-icon-circle-triangle-e{background-position:-48px -192px}.ui-icon-circle-triangle-s{background-position:-64px -192px}.ui-icon-circle-triangle-w{background-position:-80px -192px}.ui-icon-circle-triangle-n{background-position:-96px -192px}.ui-icon-circle-arrow-e{background-position:-112px -192px}.ui-icon-circle-arrow-s{background-position:-128px -192px}.ui-icon-circle-arrow-w{background-position:-144px -192px}.ui-icon-circle-arrow-n{background-position:-160px -192px}.ui-icon-circle-zoomin{background-position:-176px -192px}.ui-icon-circle-zoomout{background-position:-192px -192px}.ui-icon-circle-check{background-position:-208px -192px}.ui-icon-circlesmall-plus{background-position:0 -208px}.ui-icon-circlesmall-minus{background-position:-16px -208px}.ui-icon-circlesmall-close{background-position:-32px -208px}.ui-icon-squaresmall-plus{background-position:-48px -208px}.ui-icon-squaresmall-minus{background-position:-64px -208px}.ui-icon-squaresmall-close{background-position:-80px -208px}.ui-icon-grip-dotted-vertical{background-position:0 -224px}.ui-icon-grip-dotted-horizontal{background-position:-16px -224px}.ui-icon-grip-solid-vertical{background-position:-32px -224px}.ui-icon-grip-solid-horizontal{background-position:-48px -224px}.ui-icon-gripsmall-diagonal-se{background-position:-64px -224px}.ui-icon-grip-diagonal-se{background-position:-80px -224px}.ui-corner-all,.ui-corner-top,.ui-corner-left,.ui-corner-tl{border-top-left-radius:4px}.ui-corner-all,.ui-corner-top,.ui-corner-right,.ui-corner-tr{border-top-right-radius:4px}.ui-corner-all,.ui-corner-bottom,.ui-corner-left,.ui-corner-bl{border-bottom-left-radius:4px}.ui-corner-all,.ui-corner-bottom,.ui-corner-right,.ui-corner-br{border-bottom-right-radius:4px}.ui-widget-overlay{background:#aaa url(images/ui-bg_flat_0_aaaaaa_40x100.png) 50% 50% repeat-x;opacity:.3;filter:Alpha(Opacity=30)}.ui-widget-shadow{margin:-8px 0 0 -8px;padding:8px;background:#aaa url(images/ui-bg_flat_0_aaaaaa_40x100.png) 50% 50% repeat-x;opacity:.3;filter:Alpha(Opacity=30);border-radius:8px}


--------------------------------------------------------------------------------
/static/public/image_regression.html:
--------------------------------------------------------------------------------
  1 | <html>
  2 | <head>
  3 | <title>ConvNetJS demo: Image Painting</title>
  4 | <link href='http://fonts.googleapis.com/css?family=Cabin' rel='stylesheet' type='text/css'>
  5 | 
  6 | <style>
  7 | body {
  8 |   font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
  9 |   color: #333;
 10 |   padding: 20px;
 11 | }
 12 | </style>
 13 | 
 14 | <link rel="stylesheet" href="css/jquery-ui.min.css">
 15 | <script src="js/jquery-1.8.3.min.js"></script>
 16 | <script src="js/jquery-ui.min.js"></script>
 17 | 
 18 | <script src="js/convnet.js"></script>
 19 | <script type="text/javascript">
 20 | 
 21 | var data, labels;
 22 | var layer_defs, net, trainer;
 23 | 
 24 | // create neural net
 25 | var t = "layer_defs = [];\n\
 26 | layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:2}); // 2 inputs: x, y \n\
 27 | layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
 28 | layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
 29 | layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
 30 | layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
 31 | layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
 32 | layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
 33 | layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\
 34 | layer_defs.push({type:'regression', num_neurons:3}); // 3 outputs: r,g,b \n\
 35 | \n\
 36 | net = new convnetjs.Net();\n\
 37 | net.makeLayers(layer_defs);\n\
 38 | \n\
 39 | trainer = new convnetjs.SGDTrainer(net, {learning_rate:0.01, momentum:0.9, batch_size:5, l2_decay:0.0});\n\
 40 | ";
 41 | 
 42 | var batches_per_iteration = 100;
 43 | var mod_skip_draw = 100;
 44 | var smooth_loss = -1;
 45 | 
 46 | function update(){
 47 |   // forward prop the data
 48 |   var W = nn_canvas.width;
 49 |   var H = nn_canvas.height;
 50 | 
 51 |   var p = oridata.data;
 52 | 
 53 |   var v = new convnetjs.Vol(1,1,2);
 54 |   var loss = 0;
 55 |   var lossi = 0;
 56 |   var N = batches_per_iteration;
 57 |   for(var iters=0;iters<trainer.batch_size;iters++) {
 58 |     for(var i=0;i<N;i++) {
 59 |       // sample a coordinate
 60 |       var x = convnetjs.randi(0, W);
 61 |       var y = convnetjs.randi(0, H);
 62 |       var ix = ((W*y)+x)*4;
 63 |       var r = [p[ix]/255.0, p[ix+1]/255.0, p[ix+2]/255.0]; // r g b
 64 |       v.w[0] = (x-W/2)/W;
 65 |       v.w[1] = (y-H/2)/H;
 66 |       var stats = trainer.train(v, r);
 67 |       loss += stats.loss;
 68 |       lossi += 1;
 69 |     }
 70 |   }
 71 |   loss /= lossi;
 72 | 
 73 |   if(counter === 0) smooth_loss = loss;
 74 |   else smooth_loss = 0.99*smooth_loss + 0.01*loss;
 75 | 
 76 |   var t = '';
 77 |   t += 'loss: ' + smooth_loss;
 78 |   t += '<br>'
 79 |   t += 'iteration: ' + counter;
 80 |   $("#report").html(t);
 81 | }
 82 | 
 83 | function draw() {
 84 |   if(counter % mod_skip_draw !== 0) return;
 85 | 
 86 |   // iterate over all pixels in the target array, evaluate them
 87 |   // and draw
 88 |   var W = nn_canvas.width;
 89 |   var H = nn_canvas.height;
 90 | 
 91 |   var g = nn_ctx.getImageData(0, 0, W, H);
 92 |   var v = new convnetjs.Vol(1, 1, 2);
 93 |   for(var x=0;x<W;x++) {
 94 |     v.w[0] = (x-W/2)/W;
 95 |     for(var y=0;y<H;y++) {
 96 |       v.w[1] = (y-H/2)/H;
 97 | 
 98 |       var ix = ((W*y)+x)*4;
 99 |       var r = net.forward(v);
100 |       g.data[ix+0] = Math.floor(255*r.w[0]);
101 |       g.data[ix+1] = Math.floor(255*r.w[1]);
102 |       g.data[ix+2] = Math.floor(255*r.w[2]);
103 |       g.data[ix+3] = 255; // alpha...
104 |     }
105 |   }
106 |   nn_ctx.putImageData(g, 0, 0);
107 | }
108 | 
109 | function tick() {
110 |   update();
111 |   draw();
112 |   counter += 1;
113 | }
114 | 
115 | function reload() {
116 |   counter = 0;
117 |   eval($("#layerdef").val());
118 |   //$("#slider").slider("value", Math.log(trainer.learning_rate) / Math.LN10);
119 |   //$("#lr").html('Learning rate: ' + trainer.learning_rate);
120 | }
121 | function refreshSwatch() {
122 |   var lr = $("#slider").slider("value");
123 |   trainer.learning_rate = Math.pow(10, lr);
124 |   $("#lr").html('Learning rate: ' + trainer.learning_rate);
125 | }
126 | 
127 | var ori_canvas, nn_canvas, ori_ctx, nn_ctx, oridata;
128 | var sz = 200; // size of our drawing area
129 | var counter = 0;
130 | $(function() {
131 |     // dynamically load lena image into original image canvas
132 |     var image = new Image();
133 |     //image.src = "lena.png";
134 |     image.onload = function() {
135 | 
136 |       ori_canvas = document.getElementById('canv_original');
137 |       nn_canvas = document.getElementById('canv_net');
138 |       ori_canvas.width = sz;
139 |       ori_canvas.height = sz;
140 |       nn_canvas.width = sz;
141 |       nn_canvas.height = sz;
142 | 
143 |       ori_ctx = ori_canvas.getContext("2d");
144 |       nn_ctx = nn_canvas.getContext("2d");
145 |       ori_ctx.drawImage(image, 0, 0, sz, sz);
146 |       oridata = ori_ctx.getImageData(0, 0, sz, sz); // grab the data pointer. Our dataset.
147 | 
148 |       // start the regression!
149 |       setInterval(tick, 1);
150 |     }
151 |     image.src = "imgs/cat.jpg";
152 | 
153 |     // init put text into textarea
154 |     $("#layerdef").val(t);
155 | 
156 |     // load the net
157 |     reload();
158 | 
159 |     // set up slider for learning rate
160 |     $("#slider").slider({
161 |       orientation: "horizontal",
162 |       min: -4,
163 |       max: -1,
164 |       step: 0.05,
165 |       value: Math.log(trainer.learning_rate) / Math.LN10,
166 |       slide: refreshSwatch,
167 |       change: refreshSwatch
168 |     });
169 |     $("#lr").html('Learning rate: ' + trainer.learning_rate);
170 | 
171 |     $("#f").on('change', function(ev) {
172 |       var f = ev.target.files[0];
173 |       var fr = new FileReader();
174 |       fr.onload = function(ev2) {
175 |         var image = new Image();
176 |         image.onload = function(){
177 |           ori_ctx.drawImage(image, 0, 0, sz, sz);
178 |           oridata = ori_ctx.getImageData(0, 0, sz, sz);
179 |           reload();
180 |         }
181 |         image.src = ev2.target.result;
182 |       };
183 |       fr.readAsDataURL(f);
184 |     });
185 | 
186 |     $('.ci').click(function(){
187 |       var src = $(this).attr('src');
188 |       ori_ctx.drawImage(this, 0, 0, sz, sz);
189 |       oridata = ori_ctx.getImageData(0, 0, sz, sz);
190 |       reload();
191 |     });
192 | });
193 | 
194 | </script>
195 | 
196 | <style type="text/css">
197 | canvas { 
198 |     border: 1px solid #555;
199 |     margin-top: 10px;
200 | }
201 | #wrap {
202 |   width: 800px;
203 |   margin-right: auto;
204 |   margin-left: auto;
205 |   margin-bottom: 200px;
206 | }
207 | #gallery img {
208 |   width: 195;
209 |   height: 195px;
210 |   display: inline-block;
211 | }
212 | </style>
213 | 
214 | </head>
215 | 
216 | <body>
217 | 
218 | <div id="wrap">
219 | <h1><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvnetJS</a> demo: Image "Painting"</h1>
220 | 
221 | <p>This demo that treats the pixels of an image as a learning problem: it takes the (x,y) position on a grid and learns to predict the color at that point using regression to (r,g,b). It's a bit like compression, since the image information is encoded in the weights of the network, but almost certainly not of practical kind :)</p>
222 | <p>
223 |   Note that the entire ConvNetJS definition is shown in textbox below and it gets eval()'d to create the network, so feel free to fiddle with the parameters and hit "reload". I found that, empirically and interestingly, deeper networks tend to work much better on this task given a fixed parameter budget.
224 | </p>
225 | 
226 | <p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
227 | 
228 | <textarea id="layerdef" style="width:100%; height:250px;"></textarea>
229 | <br /><br  />
230 | <input id="buttontp" type="submit" value="reload network" onclick="reload();" style="width: 200px; height: 30px;"/>
231 | 
232 | <div style="float: right;">
233 | Choose your own image: 
234 | <input id="f" type="file" />
235 | </div>
236 | 
237 | <br>
238 | 
239 | <div style="background-color: #EEE; padding: 10px; margin-top: 10px;">
240 | <div style="float: left; margin-left: 200px;">
241 |   Original Image<br>
242 |   <canvas id="canv_original"></canvas>
243 | </div>
244 | <div style="float: left; margin-left: 10px;">
245 |   Neural Network output<br>
246 |   <canvas id="canv_net"></canvas>
247 | </div>
248 | 
249 | <div style="clear:both;"></div>
250 | </div>
251 | 
252 | <br>
253 | <div id="report"></div>
254 | <br>
255 | 
256 | <div id="lr">Learning rate:</div>
257 | <div id="slider"></div>
258 | <div>The learning rate should probably be decreased over time (slide left) to let the network better overfit the training data. It's nice to not have to worry about overfitting.</div>
259 | 
260 | <br><br>
261 | <div>
262 |   You can upload your own image above (click Choose File), or you can click on any of the images below to load them.
263 | </div>
264 | <div id="gallery">
265 |   <img src="imgs/cat.jpg" class="ci" />
266 |   <img src="imgs/esher.png"  class="ci" />
267 |   <img src="imgs/pencils.png" class="ci" />
268 |   <img src="imgs/usa.png"  class="ci" />
269 |   <img src="imgs/fractal.jpg"  class="ci" />
270 |   <img src="imgs/dora.jpg" class="ci" />
271 |   <img src="imgs/rubiks.jpg" class="ci" />
272 |   <img src="imgs/earth.jpg" class="ci" />
273 |   <img src="imgs/fox.png" class="ci" />
274 |   <img src="imgs/twitter.png" class="ci" />
275 |   <img src="imgs/reddit.jpg" class="ci" />
276 |   <img src="imgs/chip.jpg" class="ci" />
277 |   <img src="imgs/jitendra.jpg" class="ci" />
278 |   <img src="imgs/tesla.jpg" class="ci" />
279 |   <img src="imgs/rainforest.jpg" class="ci" />
280 |   <img src="imgs/chess.png" class="ci" />
281 |   <img src="imgs/gradient.png" class="ci" />
282 |   <img src="imgs/battery.jpg" class="ci" />
283 |   <img src="imgs/starry.jpg" class="ci" />
284 | </div>
285 | 
286 | <br><br>
287 | <p>Go <a href="http://cs.stanford.edu/people/karpathy/convnetjs/">back to ConvNetJS</a></p>
288 | 
289 | </div>
290 | 
291 | </body>
292 | </html>
293 | 
294 | 


--------------------------------------------------------------------------------
/static/public/imgs/balls.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/balls.jpg


--------------------------------------------------------------------------------
/static/public/imgs/battery.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/battery.jpg


--------------------------------------------------------------------------------
/static/public/imgs/cat.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/cat.jpg


--------------------------------------------------------------------------------
/static/public/imgs/chess.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/chess.png


--------------------------------------------------------------------------------
/static/public/imgs/chip.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/chip.jpg


--------------------------------------------------------------------------------
/static/public/imgs/debug.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/debug.jpg


--------------------------------------------------------------------------------
/static/public/imgs/dora.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/dora.jpg


--------------------------------------------------------------------------------
/static/public/imgs/earth.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/earth.jpg


--------------------------------------------------------------------------------
/static/public/imgs/esher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/esher.png


--------------------------------------------------------------------------------
/static/public/imgs/fox.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/fox.png


--------------------------------------------------------------------------------
/static/public/imgs/fractal.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/fractal.jpg


--------------------------------------------------------------------------------
/static/public/imgs/gradient.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/gradient.png


--------------------------------------------------------------------------------
/static/public/imgs/jitendra.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/jitendra.jpg


--------------------------------------------------------------------------------
/static/public/imgs/lena.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/lena.png


--------------------------------------------------------------------------------
/static/public/imgs/pencils.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/pencils.png


--------------------------------------------------------------------------------
/static/public/imgs/rainforest.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/rainforest.jpg


--------------------------------------------------------------------------------
/static/public/imgs/reddit.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/reddit.jpg


--------------------------------------------------------------------------------
/static/public/imgs/rubiks.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/rubiks.jpg


--------------------------------------------------------------------------------
/static/public/imgs/starry.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/starry.jpg


--------------------------------------------------------------------------------
/static/public/imgs/tesla.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/tesla.jpg


--------------------------------------------------------------------------------
/static/public/imgs/twitter.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/twitter.png


--------------------------------------------------------------------------------
/static/public/imgs/usa.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/imgs/usa.png


--------------------------------------------------------------------------------
/static/public/js/util.js:
--------------------------------------------------------------------------------
 1 | 
 2 | // contains various utility functions 
 3 | var cnnutil = (function(exports){
 4 | 
 5 |   // a window stores _size_ number of values
 6 |   // and returns averages. Useful for keeping running
 7 |   // track of validation or training accuracy during SGD
 8 |   var Window = function(size, minsize) {
 9 |     this.v = [];
10 |     this.size = typeof(size)==='undefined' ? 100 : size;
11 |     this.minsize = typeof(minsize)==='undefined' ? 20 : minsize;
12 |     this.sum = 0;
13 |   }
14 |   Window.prototype = {
15 |     add: function(x) {
16 |       this.v.push(x);
17 |       this.sum += x;
18 |       if(this.v.length>this.size) {
19 |         var xold = this.v.shift();
20 |         this.sum -= xold;
21 |       }
22 |     },
23 |     get_average: function() {
24 |       if(this.v.length < this.minsize) return -1;
25 |       else return this.sum/this.v.length;
26 |     },
27 |     reset: function(x) {
28 |       this.v = [];
29 |       this.sum = 0;
30 |     }
31 |   }
32 | 
33 |   // returns min, max and indeces of an array
34 |   var maxmin = function(w) {
35 |     if(w.length === 0) { return {}; } // ... ;s
36 | 
37 |     var maxv = w[0];
38 |     var minv = w[0];
39 |     var maxi = 0;
40 |     var mini = 0;
41 |     for(var i=1;i<w.length;i++) {
42 |       if(w[i] > maxv) { maxv = w[i]; maxi = i; } 
43 |       if(w[i] < minv) { minv = w[i]; mini = i; } 
44 |     }
45 |     return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv};
46 |   }
47 | 
48 |   // returns string representation of float
49 |   // but truncated to length of d digits
50 |   var f2t = function(x, d) {
51 |     if(typeof(d)==='undefined') { var d = 5; }
52 |     var dd = 1.0 * Math.pow(10, d);
53 |     return '' + Math.floor(x*dd)/dd;
54 |   }
55 | 
56 |   exports = exports || {};
57 |   exports.Window = Window;
58 |   exports.maxmin = maxmin;
59 |   exports.f2t = f2t;
60 |   return exports;
61 | 
62 | })(typeof module != 'undefined' && module.exports);  // add exports to module.exports if in node.js
63 | 
64 | 
65 | 


--------------------------------------------------------------------------------
/static/public/js/vis.js:
--------------------------------------------------------------------------------
  1 | 
  2 | // contains various utility functions 
  3 | var cnnvis = (function(exports){
  4 | 
  5 |   // can be used to graph loss, or accuract over time
  6 |   var Graph = function(options) {
  7 |     var options = options || {};
  8 |     this.step_horizon = options.step_horizon || 1000;
  9 |     
 10 |     this.pts = [];
 11 |     
 12 |     this.maxy = -9999;
 13 |     this.miny = 9999;
 14 |   }
 15 | 
 16 |   Graph.prototype = {
 17 |     // canv is the canvas we wish to update with this new datapoint
 18 |     add: function(step, y) {
 19 |       var time = new Date().getTime(); // in ms
 20 |       if(y>this.maxy*0.99) this.maxy = y*1.05;
 21 |       if(y<this.miny*1.01) this.miny = y*0.95;
 22 | 
 23 |       this.pts.push({step: step, time: time, y: y});
 24 |       if(step > this.step_horizon) this.step_horizon *= 2;
 25 |     },
 26 |     // elt is a canvas we wish to draw into
 27 |     drawSelf: function(canv) {
 28 |       
 29 |       var pad = 25;
 30 |       var H = canv.height;
 31 |       var W = canv.width;
 32 |       var ctx = canv.getContext('2d');
 33 | 
 34 |       ctx.clearRect(0, 0, W, H);
 35 |       ctx.font="10px Georgia";
 36 | 
 37 |       var f2t = function(x) {
 38 |         var dd = 1.0 * Math.pow(10, 2);
 39 |         return '' + Math.floor(x*dd)/dd;
 40 |       }
 41 | 
 42 |       // draw guidelines and values
 43 |       ctx.strokeStyle = "#999";
 44 |       ctx.beginPath();
 45 |       var ng = 10;
 46 |       for(var i=0;i<=ng;i++) {
 47 |         var xpos = i/ng*(W-2*pad)+pad;
 48 |         ctx.moveTo(xpos, pad);
 49 |         ctx.lineTo(xpos, H-pad);
 50 |         ctx.fillText(f2t(i/ng*this.step_horizon/1000)+'k',xpos,H-pad+14);
 51 |       }
 52 |       for(var i=0;i<=ng;i++) {
 53 |         var ypos = i/ng*(H-2*pad)+pad;
 54 |         ctx.moveTo(pad, ypos);
 55 |         ctx.lineTo(W-pad, ypos);
 56 |         ctx.fillText(f2t((ng-i)/ng*(this.maxy-this.miny) + this.miny), 0, ypos);
 57 |       }
 58 |       ctx.stroke();
 59 | 
 60 |       var N = this.pts.length;
 61 |       if(N<2) return;
 62 | 
 63 |       // draw the actual curve
 64 |       var t = function(x, y, s) {
 65 |         var tx = x / s.step_horizon * (W-pad*2) + pad;
 66 |         var ty = H - ((y-s.miny) / (s.maxy-s.miny) * (H-pad*2) + pad);
 67 |         return {tx:tx, ty:ty}
 68 |       }
 69 | 
 70 |       ctx.strokeStyle = "red";
 71 |       ctx.beginPath()
 72 |       for(var i=0;i<N;i++) {
 73 |         // draw line from i-1 to i
 74 |         var p = this.pts[i];
 75 |         var pt = t(p.step, p.y, this);
 76 |         if(i===0) ctx.moveTo(pt.tx, pt.ty);
 77 |         else ctx.lineTo(pt.tx, pt.ty);
 78 |       }
 79 |       ctx.stroke();
 80 |     }
 81 |   }
 82 | 
 83 |   // same as graph but draws multiple lines. For now I'm lazy and duplicating
 84 |   // the code, but in future I will merge these two more nicely.
 85 |   var MultiGraph = function(legend, options) {
 86 |     var options = options || {};
 87 |     this.step_horizon = options.step_horizon || 1000;
 88 | 
 89 |     if(typeof options.maxy !== 'undefined') this.maxy_forced = options.maxy;
 90 |     if(typeof options.miny !== 'undefined') this.miny_forced = options.miny;
 91 | 
 92 |     this.pts = [];
 93 |     
 94 |     this.maxy = -9999;
 95 |     this.miny = 9999;
 96 |     this.numlines = 0;
 97 | 
 98 |     this.numlines = legend.length;
 99 |     this.legend = legend;
100 |     this.styles = ["red", "blue", "green", "black", "magenta", "cyan", "purple", "aqua", "olive", "lime", "navy"];
101 |     // 17 basic colors: aqua, black, blue, fuchsia, gray, green, lime, maroon, navy, olive, orange, purple, red, silver, teal, white, and yellow
102 |   }
103 | 
104 |   MultiGraph.prototype = {
105 |     // canv is the canvas we wish to update with this new datapoint
106 |     add: function(step, yl) {
107 |       var time = new Date().getTime(); // in ms
108 |       var n = yl.length;
109 |       for(var k=0;k<n;k++) {
110 |         var y = yl[k];
111 |         if(y>this.maxy*0.99) this.maxy = y*1.05;
112 |         if(y<this.miny*1.01) this.miny = y*0.95;
113 |       }
114 | 
115 |       if(typeof this.maxy_forced !== 'undefined') this.maxy = this.maxy_forced;
116 |       if(typeof this.miny_forced !== 'undefined') this.miny = this.miny_forced;
117 | 
118 |       this.pts.push({step: step, time: time, yl: yl});
119 |       if(step > this.step_horizon) this.step_horizon *= 2;
120 |     },
121 |     // elt is a canvas we wish to draw into
122 |     drawSelf: function(canv) {
123 |       
124 |       var pad = 25;
125 |       var H = canv.height;
126 |       var W = canv.width;
127 |       var ctx = canv.getContext('2d');
128 | 
129 |       ctx.clearRect(0, 0, W, H);
130 |       ctx.font="10px Georgia";
131 | 
132 |       var f2t = function(x) {
133 |         var dd = 1.0 * Math.pow(10, 2);
134 |         return '' + Math.floor(x*dd)/dd;
135 |       }
136 | 
137 |       // draw guidelines and values
138 |       ctx.strokeStyle = "#999";
139 |       ctx.beginPath();
140 |       var ng = 10;
141 |       for(var i=0;i<=ng;i++) {
142 |         var xpos = i/ng*(W-2*pad)+pad;
143 |         ctx.moveTo(xpos, pad);
144 |         ctx.lineTo(xpos, H-pad);
145 |         ctx.fillText(f2t(i/ng*this.step_horizon/1000)+'k',xpos,H-pad+14);
146 |       }
147 |       for(var i=0;i<=ng;i++) {
148 |         var ypos = i/ng*(H-2*pad)+pad;
149 |         ctx.moveTo(pad, ypos);
150 |         ctx.lineTo(W-pad, ypos);
151 |         ctx.fillText(f2t((ng-i)/ng*(this.maxy-this.miny) + this.miny), 0, ypos);
152 |       }
153 |       ctx.stroke();
154 | 
155 |       var N = this.pts.length;
156 |       if(N<2) return;
157 | 
158 |       // draw legend
159 |       for(var k=0;k<this.numlines;k++) {
160 |         ctx.fillStyle = this.styles[k % this.styles.length];
161 |         ctx.fillText(this.legend[k], W-pad-100, pad+20+k*16);
162 |       }
163 |       ctx.fillStyle = "black";
164 | 
165 |       // draw the actual curve
166 |       var t = function(x, y, s) {
167 |         var tx = x / s.step_horizon * (W-pad*2) + pad;
168 |         var ty = H - ((y-s.miny) / (s.maxy-s.miny) * (H-pad*2) + pad);
169 |         return {tx:tx, ty:ty}
170 |       }
171 |       for(var k=0;k<this.numlines;k++) {
172 | 
173 |         ctx.strokeStyle = this.styles[k % this.styles.length];
174 |         ctx.beginPath()
175 |         for(var i=0;i<N;i++) {
176 |           // draw line from i-1 to i
177 |           var p = this.pts[i];
178 |           var pt = t(p.step, p.yl[k], this);
179 |           if(i===0) ctx.moveTo(pt.tx, pt.ty);
180 |           else ctx.lineTo(pt.tx, pt.ty);
181 |         }
182 |         ctx.stroke();
183 |       }
184 | 
185 |     }
186 |   }
187 | 
188 |   exports = exports || {};
189 |   exports.Graph = Graph;
190 |   exports.MultiGraph = MultiGraph;
191 |   return exports;
192 | 
193 | })(typeof module != 'undefined' && module.exports);  // add exports to module.exports if in node.js
194 | 
195 | 
196 | 


--------------------------------------------------------------------------------
/static/public/mnist.html:
--------------------------------------------------------------------------------
  1 | <!doctype html>
  2 | <html lang="en">
  3 | <head>
  4 |   <meta charset="utf-8">
  5 |   <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  6 |   <title>ConvNetJS MNIST demo</title>
  7 |   <meta name="description" content="">
  8 |   <meta name="author" content="">
  9 | 
 10 | <style>
 11 | .layer {
 12 |   border: 1px solid #999;
 13 |   margin-bottom: 5px;
 14 |   text-align: left;
 15 |   padding: 10px;
 16 | }
 17 | .layer_act {
 18 |   width: 450px;
 19 |   float: right;
 20 | }
 21 | .ltconv {
 22 |   background-color: #FDD;
 23 | }
 24 | .ltrelu {
 25 |   background-color: #FDF;
 26 | }
 27 | .ltpool {
 28 |   background-color: #DDF;
 29 | }
 30 | .ltsoftmax {
 31 |   background-color: #FFD;
 32 | }
 33 | .ltfc {
 34 |   background-color: #DFF;
 35 | }
 36 | .ltlrn {
 37 |   background-color: #DFD; 
 38 | }
 39 | .ltdropout {
 40 |   background-color: #AAA;  
 41 | }
 42 | .ltitle {
 43 |   color: #333;
 44 |   font-size: 18px;
 45 | }
 46 | .actmap {
 47 |   margin: 1px;
 48 | }
 49 | #trainstats {
 50 |   text-align: left;
 51 | }
 52 | .clear {
 53 |   clear: both;
 54 | }
 55 | #wrap {
 56 |   width: 800px;
 57 |   margin-left: auto;
 58 |   margin-right: auto;
 59 | }
 60 | h1 {
 61 |   font-size: 16px;
 62 |   color: #333;
 63 |   background-color: #DDD;
 64 |   border-bottom: 1px #999 solid;
 65 |   text-align: center;
 66 | }
 67 | .secpart {
 68 |   width: 400px;
 69 |   float: left;
 70 | }
 71 | #lossgraph {
 72 |   /*border: 1px solid #F0F;*/
 73 |   width: 100%;
 74 | }
 75 | .probsdiv canvas {
 76 |   float: left;
 77 | }
 78 | .probsdiv {
 79 |   height: 60px;
 80 |   width: 180px;
 81 |   display: inline-block;
 82 |   font-size: 12px;
 83 |   box-shadow: 0px 0px 2px 2px #EEE;
 84 |   margin: 5px;
 85 |   padding: 5px;
 86 |   color: black;
 87 | }
 88 | .pp {
 89 |   margin: 1px;
 90 |   padding: 1px;
 91 | }
 92 | #testset_acc {
 93 |   margin-bottom: 200px;
 94 | }
 95 | body {
 96 |   font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
 97 | }
 98 | </style>
 99 | 
100 | <script src="js/jquery-1.8.3.min.js"></script>
101 | <script src="js/vis.js"></script>
102 | <script src="js/util.js"></script>
103 | <script src="js/convnet.js"></script>
104 | 
105 | <script src="js/mnist_labels.js"></script>
106 | 
107 | <script>
108 | 
109 | var layer_defs, net, trainer;
110 | var t = "layer_defs = [];\n\
111 | layer_defs.push({type:'input', out_sx:24, out_sy:24, out_depth:1});\n\
112 | layer_defs.push({type:'conv', sx:5, filters:8, stride:1, pad:2, activation:'relu'});\n\
113 | layer_defs.push({type:'pool', sx:2, stride:2});\n\
114 | layer_defs.push({type:'conv', sx:5, filters:16, stride:1, pad:2, activation:'relu'});\n\
115 | layer_defs.push({type:'pool', sx:3, stride:3});\n\
116 | layer_defs.push({type:'softmax', num_classes:10});\n\
117 | \n\
118 | net = new convnetjs.Net();\n\
119 | net.makeLayers(layer_defs);\n\
120 | \n\
121 | trainer = new convnetjs.SGDTrainer(net, {method:'adadelta', batch_size:20, l2_decay:0.001});\n\
122 | ";
123 | 
124 | // ------------------------
125 | // BEGIN MNIST SPECIFIC STUFF
126 | // ------------------------
127 | classes_txt = ['0','1','2','3','4','5','6','7','8','9'];
128 | 
129 | var use_validation_data = true;
130 | var sample_training_instance = function() {
131 | 
132 |   // find an unloaded batch
133 |   var bi = Math.floor(Math.random()*loaded_train_batches.length);
134 |   var b = loaded_train_batches[bi];
135 |   var k = Math.floor(Math.random()*3000); // sample within the batch
136 |   var n = b*3000+k;
137 | 
138 |   // load more batches over time
139 |   if(step_num%5000===0 && step_num>0) {
140 |     for(var i=0;i<num_batches;i++) {
141 |       if(!loaded[i]) {
142 |         // load it
143 |         load_data_batch(i);
144 |         break; // okay for now
145 |       }
146 |     }
147 |   }
148 | 
149 |   // fetch the appropriate row of the training image and reshape into a Vol
150 |   var p = img_data[b].data;
151 |   var x = new convnetjs.Vol(28,28,1,0.0);
152 |   var W = 28*28;
153 |   for(var i=0;i<W;i++) {
154 |     var ix = ((W * k) + i) * 4;
155 |     x.w[i] = p[ix]/255.0;
156 |   }
157 |   x = convnetjs.augment(x, 24);
158 | 
159 |   var isval = use_validation_data && n%10===0 ? true : false;
160 |   return {x:x, label:labels[n], isval:isval};
161 | }
162 | 
163 | // sample a random testing instance
164 | var sample_test_instance = function() {
165 |   var b = 20;
166 |   var k = Math.floor(Math.random()*3000);
167 |   var n = b*3000+k;
168 | 
169 |   var p = img_data[b].data;
170 |   var x = new convnetjs.Vol(28,28,1,0.0);
171 |   var W = 28*28;
172 |   for(var i=0;i<W;i++) {
173 |     var ix = ((W * k) + i) * 4;
174 |     x.w[i] = p[ix]/255.0;
175 |   }
176 |   var xs = [];
177 |   for(var i=0;i<4;i++) {
178 |     xs.push(convnetjs.augment(x, 24));
179 |   }
180 |   // return multiple augmentations, and we will average the network over them
181 |   // to increase performance
182 |   return {x:xs, label:labels[n]};
183 | }
184 | 
185 | var num_batches = 21; // 20 training batches, 1 test
186 | var data_img_elts = new Array(num_batches);
187 | var img_data = new Array(num_batches);
188 | var loaded = new Array(num_batches);
189 | var loaded_train_batches = [];
190 | 
191 | // int main
192 | $(window).load(function() {
193 | 
194 |   $("#newnet").val(t);
195 |   eval($("#newnet").val());
196 | 
197 |   update_net_param_display();
198 | 
199 |   for(var k=0;k<loaded.length;k++) { loaded[k] = false; }
200 | 
201 |   load_data_batch(0); // async load train set batch 0 (6 total train batches)
202 |   load_data_batch(20); // async load test set (batch 6)
203 |   start_fun();
204 | });
205 | 
206 | var start_fun = function() {
207 |   if(loaded[0] && loaded[20]) { 
208 |     console.log('starting!'); 
209 |     setInterval(load_and_step, 0); // lets go!
210 |   }
211 |   else { setTimeout(start_fun, 200); } // keep checking
212 | }
213 | 
214 | var load_data_batch = function(batch_num) {
215 |   // Load the dataset with JS in background
216 |   data_img_elts[batch_num] = new Image();
217 |   var data_img_elt = data_img_elts[batch_num];
218 |   data_img_elt.onload = function() { 
219 |     var data_canvas = document.createElement('canvas');
220 |     data_canvas.width = data_img_elt.width;
221 |     data_canvas.height = data_img_elt.height;
222 |     var data_ctx = data_canvas.getContext("2d");
223 |     data_ctx.drawImage(data_img_elt, 0, 0); // copy it over... bit wasteful :(
224 |     img_data[batch_num] = data_ctx.getImageData(0, 0, data_canvas.width, data_canvas.height);
225 |     loaded[batch_num] = true;
226 |     if(batch_num < 20) { loaded_train_batches.push(batch_num); }
227 |     console.log('finished loading data batch ' + batch_num);
228 |   };
229 |   data_img_elt.src = "mnist/mnist_batch_" + batch_num + ".png";
230 | }
231 | 
232 | // ------------------------
233 | // END MNIST SPECIFIC STUFF
234 | // ------------------------
235 | 
236 | var maxmin = cnnutil.maxmin;
237 | var f2t = cnnutil.f2t;
238 | 
239 | // elt is the element to add all the canvas activation drawings into
240 | // A is the Vol() to use
241 | // scale is a multiplier to make the visualizations larger. Make higher for larger pictures
242 | // if grads is true then gradients are used instead
243 | var draw_activations = function(elt, A, scale, grads) {
244 | 
245 |   var s = scale || 2; // scale
246 |   var draw_grads = false;
247 |   if(typeof(grads) !== 'undefined') draw_grads = grads;
248 | 
249 |   // get max and min activation to scale the maps automatically
250 |   var w = draw_grads ? A.dw : A.w;
251 |   var mm = maxmin(w);
252 | 
253 |   // create the canvas elements, draw and add to DOM
254 |   for(var d=0;d<A.depth;d++) {
255 | 
256 |     var canv = document.createElement('canvas');
257 |     canv.className = 'actmap';
258 |     var W = A.sx * s;
259 |     var H = A.sy * s;
260 |     canv.width = W;
261 |     canv.height = H;
262 |     var ctx = canv.getContext('2d');
263 |     var g = ctx.createImageData(W, H);
264 | 
265 |     for(var x=0;x<A.sx;x++) {
266 |       for(var y=0;y<A.sy;y++) {
267 |         if(draw_grads) {
268 |           var dval = Math.floor((A.get_grad(x,y,d)-mm.minv)/mm.dv*255);
269 |         } else {
270 |           var dval = Math.floor((A.get(x,y,d)-mm.minv)/mm.dv*255);  
271 |         }
272 |         for(var dx=0;dx<s;dx++) {
273 |           for(var dy=0;dy<s;dy++) {
274 |             var pp = ((W * (y*s+dy)) + (dx + x*s)) * 4;
275 |             for(var i=0;i<3;i++) { g.data[pp + i] = dval; } // rgb
276 |             g.data[pp+3] = 255; // alpha channel
277 |           }
278 |         }
279 |       }
280 |     }
281 |     ctx.putImageData(g, 0, 0);
282 |     elt.appendChild(canv);
283 |   }  
284 | }
285 | 
286 | var visualize_activations = function(net, elt) {
287 | 
288 |   // clear the element
289 |   elt.innerHTML = "";
290 | 
291 |   // show activations in each layer
292 |   var N = net.layers.length;
293 |   for(var i=0;i<N;i++) {
294 |     var L = net.layers[i];
295 | 
296 |     var layer_div = document.createElement('div');
297 | 
298 |     // visualize activations
299 |     var activations_div = document.createElement('div');
300 |     activations_div.appendChild(document.createTextNode('Activations:'));
301 |     activations_div.appendChild(document.createElement('br'));
302 |     activations_div.className = 'layer_act';
303 |     var scale = 2;
304 |     if(L.layer_type==='softmax' || L.layer_type==='fc') scale = 10; // for softmax
305 |     draw_activations(activations_div, L.out_act, scale);
306 | 
307 |     // visualize data gradients
308 |     if(L.layer_type !== 'softmax') {
309 |       var grad_div = document.createElement('div');
310 |       grad_div.appendChild(document.createTextNode('Data Gradients:'));
311 |       grad_div.appendChild(document.createElement('br'));
312 |       grad_div.className = 'layer_grad';
313 |       var scale = 2;
314 |       if(L.layer_type==='softmax' || L.layer_type==='fc') scale = 10; // for softmax
315 |       draw_activations(grad_div, L.out_act, scale, true);
316 |       activations_div.appendChild(grad_div);
317 |     }
318 | 
319 |     // visualize filters if they are of reasonable size
320 |     if(L.layer_type === 'conv') {
321 |       var filters_div = document.createElement('div');
322 |       if(L.filters[0].sx>3) {
323 |         // actual weights
324 |         filters_div.appendChild(document.createTextNode('Weights:'));
325 |         filters_div.appendChild(document.createElement('br'));
326 |         for(var j=0;j<L.filters.length;j++) {
327 |           filters_div.appendChild(document.createTextNode('('));
328 |           draw_activations(filters_div, L.filters[j], 2);
329 |           filters_div.appendChild(document.createTextNode(')'));
330 |         }
331 |         // gradients
332 |         filters_div.appendChild(document.createElement('br'));
333 |         filters_div.appendChild(document.createTextNode('Weight Gradients:'));
334 |         filters_div.appendChild(document.createElement('br'));
335 |         for(var j=0;j<L.filters.length;j++) {
336 |           filters_div.appendChild(document.createTextNode('('));
337 |           draw_activations(filters_div, L.filters[j], 2, true);
338 |           filters_div.appendChild(document.createTextNode(')'));
339 |         }
340 |       } else {
341 |         filters_div.appendChild(document.createTextNode('Weights hidden, too small'));
342 |       }
343 |       activations_div.appendChild(filters_div);
344 |     }
345 |     layer_div.appendChild(activations_div);
346 | 
347 |     // print some stats on left of the layer
348 |     layer_div.className = 'layer ' + 'lt' + L.layer_type;
349 |     var title_div = document.createElement('div');
350 |     title_div.className = 'ltitle'
351 |     var t = L.layer_type + ' (' + L.out_sx + 'x' + L.out_sy + 'x' + L.out_depth + ')';
352 |     title_div.appendChild(document.createTextNode(t));
353 |     layer_div.appendChild(title_div);
354 | 
355 |     if(L.layer_type==='conv') {
356 |       var t = 'filter size ' + L.filters[0].sx + 'x' + L.filters[0].sy + 'x' + L.filters[0].depth + ', stride ' + L.stride;
357 |       layer_div.appendChild(document.createTextNode(t));
358 |       layer_div.appendChild(document.createElement('br'));
359 |     }
360 |     if(L.layer_type==='pool') {
361 |       var t = 'pooling size ' + L.sx + 'x' + L.sy + ', stride ' + L.stride;
362 |       layer_div.appendChild(document.createTextNode(t));
363 |       layer_div.appendChild(document.createElement('br'));
364 |     }
365 | 
366 |     // find min, max activations and display them
367 |     var mma = maxmin(L.out_act.w);
368 |     var t = 'max activation: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
369 |     layer_div.appendChild(document.createTextNode(t));
370 |     layer_div.appendChild(document.createElement('br'));
371 | 
372 |     var mma = maxmin(L.out_act.dw);
373 |     var t = 'max gradient: ' + f2t(mma.maxv) + ', min: ' + f2t(mma.minv);
374 |     layer_div.appendChild(document.createTextNode(t));
375 |     layer_div.appendChild(document.createElement('br'));
376 | 
377 |     // number of parameters
378 |     if(L.layer_type==='conv') {
379 |       var tot_params = L.sx*L.sy*L.in_depth*L.filters.length + L.filters.length;
380 |       var t = 'parameters: ' + L.filters.length + 'x' + L.sx + 'x' + L.sy + 'x' + L.in_depth + '+' + L.filters.length + ' = ' + tot_params;
381 |       layer_div.appendChild(document.createTextNode(t));
382 |       layer_div.appendChild(document.createElement('br'));
383 |     }
384 |     if(L.layer_type==='fc') {
385 |       var tot_params = L.num_inputs*L.filters.length + L.filters.length;
386 |       var t = 'parameters: ' + L.filters.length + 'x' + L.num_inputs + '+' + L.filters.length + ' = ' + tot_params;
387 |       layer_div.appendChild(document.createTextNode(t));
388 |       layer_div.appendChild(document.createElement('br'));
389 |     }
390 | 
391 |     // css madness needed here...
392 |     var clear = document.createElement('div');
393 |     clear.className = 'clear';
394 |     layer_div.appendChild(clear);
395 | 
396 |     elt.appendChild(layer_div);
397 |   }
398 | }
399 | 
400 | // loads a training image and trains on it with the network
401 | var paused = false;
402 | var load_and_step = function() {
403 |   if(paused) return; 
404 | 
405 |   var sample = sample_training_instance();
406 |   step(sample); // process this image
407 | }
408 | 
409 | // evaluate current network on test set
410 | var test_predict = function() {
411 |   var num_classes = net.layers[net.layers.length-1].out_depth;
412 | 
413 |   document.getElementById('testset_acc').innerHTML = '';
414 |   // grab a random test image
415 |   for(num=0;num<50;num++) {
416 |     var sample = sample_test_instance();
417 |     var y = sample.label;  // ground truth label
418 | 
419 |     // forward prop it through the network
420 |     var aavg = new convnetjs.Vol(1,1,num_classes,0.0);
421 |     // ensures we always have a list, regardless if above returns single item or list
422 |     var xs = [].concat(sample.x);
423 |     var n = xs.length;
424 |     for(var i=0;i<n;i++) {
425 |       var a = net.forward(xs[i]);
426 |       aavg.addFrom(a);
427 |     }
428 |     var preds = [];
429 |     for(var k=0;k<aavg.w.length;k++) { preds.push({k:k,p:aavg.w[k]}); }
430 |     preds.sort(function(a,b){return a.p<b.p ? 1:-1;});
431 |     
432 |     var div = document.createElement('div');
433 |     div.className = 'testdiv';
434 | 
435 |     // draw the image into a canvas
436 |     draw_activations(div, xs[0], 2); // draw Vol into canv
437 | 
438 |     // add predictions
439 |     var probsdiv = document.createElement('div');
440 |     div.className = 'probsdiv';
441 |     var t = '';
442 |     for(var k=0;k<3;k++) {
443 |       var col = preds[k].k===y ? 'rgb(85,187,85)' : 'rgb(187,85,85)';
444 |       t += '<div class=\"pp\" style=\"width:' + Math.floor(preds[k].p/n*100) + 'px; margin-left: 60px; background-color:' + col + ';\">' + classes_txt[preds[k].k] + '</div>'
445 |     }
446 |     probsdiv.innerHTML = t;
447 |     div.appendChild(probsdiv);
448 | 
449 |     // add it into DOM
450 |     $("#testset_acc").append(div).fadeIn(1000);
451 |   }
452 | }
453 | 
454 | 
455 | var lossGraph = new cnnvis.Graph();
456 | var xLossWindow = new cnnutil.Window(100);
457 | var wLossWindow = new cnnutil.Window(100);
458 | var trainAccWindow = new cnnutil.Window(100);
459 | var valAccWindow = new cnnutil.Window(100);
460 | var step_num = 0;
461 | var step = function(sample) {
462 | 
463 |   var x = sample.x;
464 |   var y = sample.label;
465 | 
466 |   if(sample.isval) {
467 |     // use x to build our estimate of validation error
468 |     net.forward(x);
469 |     var yhat = net.getPrediction();
470 |     var val_acc = yhat === y ? 1.0 : 0.0;
471 |     valAccWindow.add(val_acc);
472 |     return; // get out
473 |   }
474 | 
475 |   // train on it with network
476 |   var stats = trainer.train(x, y);
477 |   var lossx = stats.cost_loss;
478 |   var lossw = stats.l2_decay_loss;
479 | 
480 |   // keep track of stats such as the average training error and loss
481 |   var yhat = net.getPrediction();
482 |   var train_acc = yhat === y ? 1.0 : 0.0;
483 |   xLossWindow.add(lossx);
484 |   wLossWindow.add(lossw);
485 |   trainAccWindow.add(train_acc);
486 | 
487 |   // visualize training status
488 |   var train_elt = document.getElementById("trainstats");
489 |   train_elt.innerHTML = '';
490 |   var t = 'Forward time per example: ' + stats.fwd_time + 'ms';
491 |   train_elt.appendChild(document.createTextNode(t));
492 |   train_elt.appendChild(document.createElement('br'));
493 |   var t = 'Backprop time per example: ' + stats.bwd_time + 'ms';
494 |   train_elt.appendChild(document.createTextNode(t));
495 |   train_elt.appendChild(document.createElement('br'));
496 |   var t = 'Classification loss: ' + f2t(xLossWindow.get_average());
497 |   train_elt.appendChild(document.createTextNode(t));
498 |   train_elt.appendChild(document.createElement('br'));
499 |   var t = 'L2 Weight decay loss: ' + f2t(wLossWindow.get_average());
500 |   train_elt.appendChild(document.createTextNode(t));
501 |   train_elt.appendChild(document.createElement('br'));
502 |   var t = 'Training accuracy: ' + f2t(trainAccWindow.get_average());
503 |   train_elt.appendChild(document.createTextNode(t));
504 |   train_elt.appendChild(document.createElement('br'));
505 |   var t = 'Validation accuracy: ' + f2t(valAccWindow.get_average());
506 |   train_elt.appendChild(document.createTextNode(t));
507 |   train_elt.appendChild(document.createElement('br'));
508 |   var t = 'Examples seen: ' + step_num;
509 |   train_elt.appendChild(document.createTextNode(t));
510 |   train_elt.appendChild(document.createElement('br'));
511 | 
512 |   // visualize activations
513 |   if(step_num % 100 === 0) {
514 |     var vis_elt = document.getElementById("visnet");
515 |     visualize_activations(net, vis_elt);
516 |   }
517 | 
518 |   // log progress to graph, (full loss)
519 |   if(step_num % 200 === 0) {
520 |     var xa = xLossWindow.get_average();
521 |     var xw = wLossWindow.get_average();
522 |     if(xa >= 0 && xw >= 0) { // if they are -1 it means not enough data was accumulated yet for estimates
523 |       lossGraph.add(step_num, xa + xw);
524 |       lossGraph.drawSelf(document.getElementById("lossgraph"));
525 |     }
526 |   }
527 | 
528 |   // run prediction on test set
529 |   if(step_num % 1000 === 0) {
530 |     test_predict();
531 |   }
532 |   step_num++;
533 | }
534 | 
535 | // user settings 
536 | var change_lr = function() {
537 |   trainer.learning_rate = parseFloat(document.getElementById("lr_input").value);
538 |   update_net_param_display();
539 | }
540 | var change_momentum = function() {
541 |   trainer.momentum = parseFloat(document.getElementById("momentum_input").value);
542 |   update_net_param_display();
543 | }
544 | var change_batch_size = function() {
545 |   trainer.batch_size = parseFloat(document.getElementById("batch_size_input").value);
546 |   update_net_param_display();
547 | }
548 | var change_decay = function() {
549 |   trainer.l2_decay = parseFloat(document.getElementById("decay_input").value);
550 |   update_net_param_display();
551 | }
552 | var update_net_param_display = function() {
553 |   document.getElementById('lr_input').value = trainer.learning_rate;
554 |   document.getElementById('momentum_input').value = trainer.momentum;
555 |   document.getElementById('batch_size_input').value = trainer.batch_size;
556 |   document.getElementById('decay_input').value = trainer.l2_decay;
557 | }
558 | var toggle_pause = function() {
559 |   paused = !paused;
560 |   var btn = document.getElementById('buttontp');
561 |   if(paused) { btn.value = 'resume' }
562 |   else { btn.value = 'pause'; }
563 | }
564 | var dump_json = function() {
565 |   document.getElementById("dumpjson").value = JSON.stringify(net.toJSON());
566 | }
567 | var clear_graph = function() {
568 |   lossGraph = new cnnvis.Graph(); // reinit graph too 
569 | }
570 | var reset_all = function() {
571 |   update_net_param_display();
572 | 
573 |   // reinit windows that keep track of val/train accuracies
574 |   xLossWindow.reset();
575 |   wLossWindow.reset();
576 |   trainAccWindow.reset();
577 |   valAccWindow.reset();
578 |   lossGraph = new cnnvis.Graph(); // reinit graph too
579 |   step_num = 0;
580 | }
581 | var load_from_json = function() {
582 |   var jsonString = document.getElementById("dumpjson").value;
583 |   var json = JSON.parse(jsonString);
584 |   net = new convnetjs.Net();
585 |   net.fromJSON(json);
586 |   reset_all();
587 | }
588 | var change_net = function() {
589 |   eval($("#newnet").val());
590 |   reset_all();
591 | }
592 | 
593 | </script>
594 | </head>
595 | <body>
596 |   <div id="wrap">
597 |   <h2 style="text-align: center;"><a href="http://cs.stanford.edu/people/karpathy/convnetjs/">ConvNetJS</a> MNIST demo</h2>
598 |   <h1>Description</h1>
599 |   <p>
600 |     This demo trains a Convolutional Neural Network on the <a href="http://yann.lecun.com/exdb/mnist/">MNIST digits dataset</a> in your browser, with nothing but Javascript. The dataset is fairly easy and one should expect to get somewhere around 99% accuracy within few minutes. I used <a href="mnist_parse.zip">this python script</a> to parse the <a href="http://deeplearning.net/tutorial/gettingstarted.html">original files</a> into batches of images that can be easily loaded into page DOM with img tags.
601 |   </p>
602 |   <p>
603 |     This network takes a 28x28 MNIST image and crops a random 24x24 window before training on it (this technique is called data augmentation and improves generalization). Similarly to do prediction, 4 random crops are sampled and the probabilities across all crops are averaged to produce final predictions. The network runs at about 5ms for both forward and backward pass on my reasonably decent Ubuntu+Chrome machine.
604 |   </p>
605 |   <p>
606 |     By default, in this demo we're using Adadelta which is one of per-parameter adaptive step size methods, so we don't have to worry about changing learning rates or momentum over time. However, I still included the text fields for changing these if you'd like to play around with SGD+Momentum trainer.
607 |   </p>
608 |   <p>Report questions/bugs/suggestions to <a href="https://twitter.com/karpathy">@karpathy</a>.</p>
609 |   <h1>Training Stats</h1>
610 |   <div class="divsec" style="270px;">
611 |     <div class="secpart">
612 |       <input id="buttontp" type="submit" value="pause" onclick="toggle_pause();" style="width: 100px; height:30px; background-color: #FCC;"/>
613 |       <div id="trainstats"></div>
614 | 
615 |       <div id="controls">
616 |         Learning rate: <input name="lri" type="text" maxlength="20" id="lr_input"/>
617 |         <input id="buttonlr" type="submit" value="change" onclick="change_lr();"/>
618 |         <br />
619 | 
620 |         Momentum: <input name="momi" type="text" maxlength="20" id="momentum_input"/>
621 |         <input id="buttonmom" type="submit" value="change" onclick="change_momentum();"/>
622 |         <br />
623 | 
624 |         Batch size: <input name="bsi" type="text" maxlength="20" id="batch_size_input"/>
625 |         <input id="buttonbs" type="submit" value="change" onclick="change_batch_size();"/>
626 |         <br />
627 | 
628 |         Weight decay: <input name="wdi" type="text" maxlength="20" id="decay_input"/>
629 |         <input id="buttonwd" type="submit" value="change" onclick="change_decay();"/>
630 |       </div>
631 | 
632 |       <input id="buttondj" type="submit" value="save network snapshot as JSON" onclick="dump_json();"/><br />
633 |       <input id="buttonlfj" type="submit" value="init network from JSON snapshot" onclick="load_from_json();"/><br />
634 |       <textarea id="dumpjson"></textarea>
635 |     </div>
636 |     <div class="secpart">
637 |       
638 |       <div>
639 |         Loss:<br />
640 |         <canvas id="lossgraph">
641 |         </canvas>
642 |         <br />
643 |         <input id="buttoncg" type="submit" value="clear graph" onclick="clear_graph();"/>
644 |       </div>
645 |     </div>
646 |     <div style="clear:both;"></div>
647 |   </div>
648 | 
649 |   <h1>Instantiate a Network and Trainer</h1>
650 |   <div>
651 |     <textarea id="newnet" style="width:100%; height:200px;"></textarea><br />
652 |     <input id="buttonnn" type="submit" value="change network" onclick="change_net();" style="width:200px;height:30px;"/>
653 |   </div>
654 | 
655 |   <div class="divsec">
656 |   <h1>Network Visualization</h1>
657 |     <div id="visnet"></div>
658 |   </div>
659 |   
660 |   <div class="divsec">
661 |   <h1>Example predictions on Test set</h1>
662 |     <div id="testset_acc"></div>
663 |   </div>
664 | 
665 |   </div>  
666 | </body>
667 | </html>
668 | 
669 | 
670 | 
671 | 


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_0.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_1.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_2.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_20.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_20.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_3.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_4.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_5.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_6.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_6.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_7.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_7.png


--------------------------------------------------------------------------------
/static/public/mnist/mnist_batch_8.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajlopez/JavaScriptAI/18e6c91e90f56c49e40cc4ec3574fe1afecff6ea/static/public/mnist/mnist_batch_8.png


--------------------------------------------------------------------------------
/static/server.js:
--------------------------------------------------------------------------------
 1 | var static = require('node-static');
 2 | 
 3 | //
 4 | // Create a node-static server instance to serve the './public' folder
 5 | //
 6 | var file = new static.Server('./public');
 7 | 
 8 | require('http').createServer(function (request, response) {
 9 |     request.addListener('end', function () {
10 |         //
11 |         // Serve files!
12 |         //
13 |         file.serve(request, response);
14 |     }).resume();
15 | }).listen(3000);


--------------------------------------------------------------------------------