From 33c56567719ed37ed009ad40320fe21e2ccb516c Mon Sep 17 00:00:00 2001
From: Shaunak Kishore <kshaunak@gmail.com>
Date: Wed, 30 Sep 2015 01:58:22 -0400
Subject: [PATCH] Restore trained stroke extractor

---
 lib/classifier.js                     |   19 +
 lib/external/convnet/1.1.0/LICENSE    |   22 +
 lib/external/convnet/1.1.0/convnet.js | 2115 +++++++++++++++++++++++++
 lib/net.js                            |    1 +
 lib/stroke_extractor.js               |    9 +-
 server/training.js                    |   78 +
 6 files changed, 2241 insertions(+), 3 deletions(-)
 create mode 100644 lib/classifier.js
 create mode 100644 lib/external/convnet/1.1.0/LICENSE
 create mode 100644 lib/external/convnet/1.1.0/convnet.js
 create mode 100644 lib/net.js
 create mode 100644 server/training.js

diff --git a/lib/classifier.js b/lib/classifier.js
new file mode 100644
index 00000000..4affbcd9
--- /dev/null
+++ b/lib/classifier.js
@@ -0,0 +1,19 @@
+"use strict";
+
+Meteor.startup(() => {
+  const input = new convnetjs.Vol(1, 1, 8 /* feature vector dimensions */);
+  const net = new convnetjs.Net();
+  net.fromJSON(NEURAL_NET_TRAINED_FOR_STROKE_EXTRACTION);
+  const weight = 0.8;
+
+  const trainedClassifier = (features) => {
+    input.w = features;
+    const softmax = net.forward(input).w;
+    return softmax[1] - softmax[0];
+  }
+
+  stroke_extractor.combinedClassifier = (features) => {
+    return stroke_extractor.handTunedClassifier(features) +
+           weight*trainedClassifier(features);
+  }
+});
diff --git a/lib/external/convnet/1.1.0/LICENSE b/lib/external/convnet/1.1.0/LICENSE
new file mode 100644
index 00000000..736b9862
--- /dev/null
+++ b/lib/external/convnet/1.1.0/LICENSE
@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2014 Andrej Karpathy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
diff --git a/lib/external/convnet/1.1.0/convnet.js b/lib/external/convnet/1.1.0/convnet.js
new file mode 100644
index 00000000..55205fd1
--- /dev/null
+++ b/lib/external/convnet/1.1.0/convnet.js
@@ -0,0 +1,2115 @@
+var convnetjs = convnetjs || { REVISION: 'ALPHA' };
+(function(global) {
+  "use strict";
+
+  // Random number utilities
+  var return_v = false;
+  var v_val = 0.0;
+  var gaussRandom = function() {
+    if(return_v) { 
+      return_v = false;
+      return v_val; 
+    }
+    var u = 2*Math.random()-1;
+    var v = 2*Math.random()-1;
+    var r = u*u + v*v;
+    if(r == 0 || r > 1) return gaussRandom();
+    var c = Math.sqrt(-2*Math.log(r)/r);
+    v_val = v*c; // cache this
+    return_v = true;
+    return u*c;
+  }
+  var randf = function(a, b) { return Math.random()*(b-a)+a; }
+  var randi = function(a, b) { return Math.floor(Math.random()*(b-a)+a); }
+  var randn = function(mu, std){ return mu+gaussRandom()*std; }
+
+  // Array utilities
+  var zeros = function(n) {
+    if(typeof(n)==='undefined' || isNaN(n)) { return []; }
+    if(typeof ArrayBuffer === 'undefined') {
+      // lacking browser support
+      var arr = new Array(n);
+      for(var i=0;i<n;i++) { arr[i]= 0; }
+      return arr;
+    } else {
+      return new Float64Array(n);
+    }
+  }
+
+  var arrContains = function(arr, elt) {
+    for(var i=0,n=arr.length;i<n;i++) {
+      if(arr[i]===elt) return true;
+    }
+    return false;
+  }
+
+  var arrUnique = function(arr) {
+    var b = [];
+    for(var i=0,n=arr.length;i<n;i++) {
+      if(!arrContains(b, arr[i])) {
+        b.push(arr[i]);
+      }
+    }
+    return b;
+  }
+
+  // return max and min of a given non-empty array.
+  var maxmin = function(w) {
+    if(w.length === 0) { return {}; } // ... ;s
+    var maxv = w[0];
+    var minv = w[0];
+    var maxi = 0;
+    var mini = 0;
+    var n = w.length;
+    for(var i=1;i<n;i++) {
+      if(w[i] > maxv) { maxv = w[i]; maxi = i; } 
+      if(w[i] < minv) { minv = w[i]; mini = i; } 
+    }
+    return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv};
+  }
+
+  // create random permutation of numbers, in range [0...n-1]
+  var randperm = function(n) {
+    var i = n,
+        j = 0,
+        temp;
+    var array = [];
+    for(var q=0;q<n;q++)array[q]=q;
+    while (i--) {
+        j = Math.floor(Math.random() * (i+1));
+        temp = array[i];
+        array[i] = array[j];
+        array[j] = temp;
+    }
+    return array;
+  }
+
+  // sample from list lst according to probabilities in list probs
+  // the two lists are of same size, and probs adds up to 1
+  var weightedSample = function(lst, probs) {
+    var p = randf(0, 1.0);
+    var cumprob = 0.0;
+    for(var k=0,n=lst.length;k<n;k++) {
+      cumprob += probs[k];
+      if(p < cumprob) { return lst[k]; }
+    }
+  }
+
+  // syntactic sugar function for getting default parameter values
+  var getopt = function(opt, field_name, default_value) {
+    return typeof opt[field_name] !== 'undefined' ? opt[field_name] : default_value;
+  }
+
+  global.randf = randf;
+  global.randi = randi;
+  global.randn = randn;
+  global.zeros = zeros;
+  global.maxmin = maxmin;
+  global.randperm = randperm;
+  global.weightedSample = weightedSample;
+  global.arrUnique = arrUnique;
+  global.arrContains = arrContains;
+  global.getopt = getopt;
+  
+})(convnetjs);
+(function(global) {
+  "use strict";
+
+  // Vol is the basic building block of all data in a net.
+  // it is essentially just a 3D volume of numbers, with a
+  // width (sx), height (sy), and depth (depth).
+  // it is used to hold data for all filters, all volumes,
+  // all weights, and also stores all gradients w.r.t. 
+  // the data. c is optionally a value to initialize the volume
+  // with. If c is missing, fills the Vol with random numbers.
+  var Vol = function(sx, sy, depth, c) {
+    // this is how you check if a variable is an array. Oh, Javascript :)
+    if(Object.prototype.toString.call(sx) === '[object Array]') {
+      // we were given a list in sx, assume 1D volume and fill it up
+      this.sx = 1;
+      this.sy = 1;
+      this.depth = sx.length;
+      // we have to do the following copy because we want to use
+      // fast typed arrays, not an ordinary javascript array
+      this.w = global.zeros(this.depth);
+      this.dw = global.zeros(this.depth);
+      for(var i=0;i<this.depth;i++) {
+        this.w[i] = sx[i];
+      }
+    } else {
+      // we were given dimensions of the vol
+      this.sx = sx;
+      this.sy = sy;
+      this.depth = depth;
+      var n = sx*sy*depth;
+      this.w = global.zeros(n);
+      this.dw = global.zeros(n);
+      if(typeof c === 'undefined') {
+        // weight normalization is done to equalize the output
+        // variance of every neuron, otherwise neurons with a lot
+        // of incoming connections have outputs of larger variance
+        var scale = Math.sqrt(1.0/(sx*sy*depth));
+        for(var i=0;i<n;i++) { 
+          this.w[i] = global.randn(0.0, scale);
+        }
+      } else {
+        for(var i=0;i<n;i++) { 
+          this.w[i] = c;
+        }
+      }
+    }
+  }
+
+  Vol.prototype = {
+    get: function(x, y, d) { 
+      var ix=((this.sx * y)+x)*this.depth+d;
+      return this.w[ix];
+    },
+    set: function(x, y, d, v) { 
+      var ix=((this.sx * y)+x)*this.depth+d;
+      this.w[ix] = v; 
+    },
+    add: function(x, y, d, v) { 
+      var ix=((this.sx * y)+x)*this.depth+d;
+      this.w[ix] += v; 
+    },
+    get_grad: function(x, y, d) { 
+      var ix = ((this.sx * y)+x)*this.depth+d;
+      return this.dw[ix]; 
+    },
+    set_grad: function(x, y, d, v) { 
+      var ix = ((this.sx * y)+x)*this.depth+d;
+      this.dw[ix] = v; 
+    },
+    add_grad: function(x, y, d, v) { 
+      var ix = ((this.sx * y)+x)*this.depth+d;
+      this.dw[ix] += v; 
+    },
+    cloneAndZero: function() { return new Vol(this.sx, this.sy, this.depth, 0.0)},
+    clone: function() {
+      var V = new Vol(this.sx, this.sy, this.depth, 0.0);
+      var n = this.w.length;
+      for(var i=0;i<n;i++) { V.w[i] = this.w[i]; }
+      return V;
+    },
+    addFrom: function(V) { for(var k=0;k<this.w.length;k++) { this.w[k] += V.w[k]; }},
+    addFromScaled: function(V, a) { for(var k=0;k<this.w.length;k++) { this.w[k] += a*V.w[k]; }},
+    setConst: function(a) { for(var k=0;k<this.w.length;k++) { this.w[k] = a; }},
+
+    toJSON: function() {
+      // todo: we may want to only save d most significant digits to save space
+      var json = {}
+      json.sx = this.sx; 
+      json.sy = this.sy;
+      json.depth = this.depth;
+      json.w = this.w;
+      return json;
+      // we wont back up gradients to save space
+    },
+    fromJSON: function(json) {
+      this.sx = json.sx;
+      this.sy = json.sy;
+      this.depth = json.depth;
+
+      var n = this.sx*this.sy*this.depth;
+      this.w = global.zeros(n);
+      this.dw = global.zeros(n);
+      // copy over the elements.
+      for(var i=0;i<n;i++) {
+        this.w[i] = json.w[i];
+      }
+    }
+  }
+
+  global.Vol = Vol;
+})(convnetjs);
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+
+  // Volume utilities
+  // intended for use with data augmentation
+  // crop is the size of output
+  // dx,dy are offset wrt incoming volume, of the shift
+  // fliplr is boolean on whether we also want to flip left<->right
+  var augment = function(V, crop, dx, dy, fliplr) {
+    // note assumes square outputs of size crop x crop
+    if(typeof(fliplr)==='undefined') var fliplr = false;
+    if(typeof(dx)==='undefined') var dx = global.randi(0, V.sx - crop);
+    if(typeof(dy)==='undefined') var dy = global.randi(0, V.sy - crop);
+    
+    // randomly sample a crop in the input volume
+    var W;
+    if(crop !== V.sx || dx!==0 || dy!==0) {
+      W = new Vol(crop, crop, V.depth, 0.0);
+      for(var x=0;x<crop;x++) {
+        for(var y=0;y<crop;y++) {
+          if(x+dx<0 || x+dx>=V.sx || y+dy<0 || y+dy>=V.sy) continue; // oob
+          for(var d=0;d<V.depth;d++) {
+           W.set(x,y,d,V.get(x+dx,y+dy,d)); // copy data over
+          }
+        }
+      }
+    } else {
+      W = V;
+    }
+
+    if(fliplr) {
+      // flip volume horziontally
+      var W2 = W.cloneAndZero();
+      for(var x=0;x<W.sx;x++) {
+        for(var y=0;y<W.sy;y++) {
+          for(var d=0;d<W.depth;d++) {
+           W2.set(x,y,d,W.get(W.sx - x - 1,y,d)); // copy data over
+          }
+        }
+      }
+      W = W2; //swap
+    }
+    return W;
+  }
+
+  // img is a DOM element that contains a loaded image
+  // returns a Vol of size (W, H, 4). 4 is for RGBA
+  var img_to_vol = function(img, convert_grayscale) {
+
+    if(typeof(convert_grayscale)==='undefined') var convert_grayscale = false;
+
+    var canvas = document.createElement('canvas');
+    canvas.width = img.width;
+    canvas.height = img.height;
+    var ctx = canvas.getContext("2d");
+
+    // due to a Firefox bug
+    try {
+      ctx.drawImage(img, 0, 0);
+    } catch (e) {
+      if (e.name === "NS_ERROR_NOT_AVAILABLE") {
+        // sometimes happens, lets just abort
+        return false;
+      } else {
+        throw e;
+      }
+    }
+
+    try {
+      var img_data = ctx.getImageData(0, 0, canvas.width, canvas.height);
+    } catch (e) {
+      if(e.name === 'IndexSizeError') {
+        return false; // not sure what causes this sometimes but okay abort
+      } else {
+        throw e;
+      }
+    }
+
+    // prepare the input: get pixels and normalize them
+    var p = img_data.data;
+    var W = img.width;
+    var H = img.height;
+    var pv = []
+    for(var i=0;i<p.length;i++) {
+      pv.push(p[i]/255.0-0.5); // normalize image pixels to [-0.5, 0.5]
+    }
+    var x = new Vol(W, H, 4, 0.0); //input volume (image)
+    x.w = pv;
+
+    if(convert_grayscale) {
+      // flatten into depth=1 array
+      var x1 = new Vol(W, H, 1, 0.0);
+      for(var i=0;i<W;i++) {
+        for(var j=0;j<H;j++) {
+          x1.set(i,j,0,x.get(i,j,0));
+        }
+      }
+      x = x1;
+    }
+
+    return x;
+  }
+  
+  global.augment = augment;
+  global.img_to_vol = img_to_vol;
+
+})(convnetjs);
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+
+  // This file contains all layers that do dot products with input,
+  // but usually in a different connectivity pattern and weight sharing
+  // schemes: 
+  // - FullyConn is fully connected dot products 
+  // - ConvLayer does convolutions (so weight sharing spatially)
+  // putting them together in one file because they are very similar
+  var ConvLayer = function(opt) {
+    var opt = opt || {};
+
+    // required
+    this.out_depth = opt.filters;
+    this.sx = opt.sx; // filter size. Should be odd if possible, it's cleaner.
+    this.in_depth = opt.in_depth;
+    this.in_sx = opt.in_sx;
+    this.in_sy = opt.in_sy;
+    
+    // optional
+    this.sy = typeof opt.sy !== 'undefined' ? opt.sy : this.sx;
+    this.stride = typeof opt.stride !== 'undefined' ? opt.stride : 1; // stride at which we apply filters to input volume
+    this.pad = typeof opt.pad !== 'undefined' ? opt.pad : 0; // amount of 0 padding to add around borders of input volume
+    this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0;
+    this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0;
+
+    // computed
+    // note we are doing floor, so if the strided convolution of the filter doesnt fit into the input
+    // volume exactly, the output volume will be trimmed and not contain the (incomplete) computed
+    // final application.
+    this.out_sx = Math.floor((this.in_sx + this.pad * 2 - this.sx) / this.stride + 1);
+    this.out_sy = Math.floor((this.in_sy + this.pad * 2 - this.sy) / this.stride + 1);
+    this.layer_type = 'conv';
+
+    // initializations
+    var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0;
+    this.filters = [];
+    for(var i=0;i<this.out_depth;i++) { this.filters.push(new Vol(this.sx, this.sy, this.in_depth)); }
+    this.biases = new Vol(1, 1, this.out_depth, bias);
+  }
+  ConvLayer.prototype = {
+    forward: function(V, is_training) {
+      // optimized code by @mdda that achieves 2x speedup over previous version
+
+      this.in_act = V;
+      var A = new Vol(this.out_sx |0, this.out_sy |0, this.out_depth |0, 0.0);
+      
+      var V_sx = V.sx |0;
+      var V_sy = V.sy |0;
+      var xy_stride = this.stride |0;
+
+      for(var d=0;d<this.out_depth;d++) {
+        var f = this.filters[d];
+        var x = -this.pad |0;
+        var y = -this.pad |0;
+        for(var ay=0; ay<this.out_sy; y+=xy_stride,ay++) {  // xy_stride
+          x = -this.pad |0;
+          for(var ax=0; ax<this.out_sx; x+=xy_stride,ax++) {  // xy_stride
+
+            // convolve centered at this particular location
+            var a = 0.0;
+            for(var fy=0;fy<f.sy;fy++) {
+              var oy = y+fy; // coordinates in the original input array coordinates
+              for(var fx=0;fx<f.sx;fx++) {
+                var ox = x+fx;
+                if(oy>=0 && oy<V_sy && ox>=0 && ox<V_sx) {
+                  for(var fd=0;fd<f.depth;fd++) {
+                    // avoid function call overhead (x2) for efficiency, compromise modularity :(
+                    a += f.w[((f.sx * fy)+fx)*f.depth+fd] * V.w[((V_sx * oy)+ox)*V.depth+fd];
+                  }
+                }
+              }
+            }
+            a += this.biases.w[d];
+            A.set(ax, ay, d, a);
+          }
+        }
+      }
+      this.out_act = A;
+      return this.out_act;
+    },
+    backward: function() {
+
+      var V = this.in_act;
+      V.dw = global.zeros(V.w.length); // zero out gradient wrt bottom data, we're about to fill it
+
+      var V_sx = V.sx |0;
+      var V_sy = V.sy |0;
+      var xy_stride = this.stride |0;
+
+      for(var d=0;d<this.out_depth;d++) {
+        var f = this.filters[d];
+        var x = -this.pad |0;
+        var y = -this.pad |0;
+        for(var ay=0; ay<this.out_sy; y+=xy_stride,ay++) {  // xy_stride
+          x = -this.pad |0;
+          for(var ax=0; ax<this.out_sx; x+=xy_stride,ax++) {  // xy_stride
+
+            // convolve centered at this particular location
+            var chain_grad = this.out_act.get_grad(ax,ay,d); // gradient from above, from chain rule
+            for(var fy=0;fy<f.sy;fy++) {
+              var oy = y+fy; // coordinates in the original input array coordinates
+              for(var fx=0;fx<f.sx;fx++) {
+                var ox = x+fx;
+                if(oy>=0 && oy<V_sy && ox>=0 && ox<V_sx) {
+                  for(var fd=0;fd<f.depth;fd++) {
+                    // avoid function call overhead (x2) for efficiency, compromise modularity :(
+                    var ix1 = ((V_sx * oy)+ox)*V.depth+fd;
+                    var ix2 = ((f.sx * fy)+fx)*f.depth+fd;
+                    f.dw[ix2] += V.w[ix1]*chain_grad;
+                    V.dw[ix1] += f.w[ix2]*chain_grad;
+                  }
+                }
+              }
+            }
+            this.biases.dw[d] += chain_grad;
+          }
+        }
+      }
+    },
+    getParamsAndGrads: function() {
+      var response = [];
+      for(var i=0;i<this.out_depth;i++) {
+        response.push({params: this.filters[i].w, grads: this.filters[i].dw, l2_decay_mul: this.l2_decay_mul, l1_decay_mul: this.l1_decay_mul});
+      }
+      response.push({params: this.biases.w, grads: this.biases.dw, l1_decay_mul: 0.0, l2_decay_mul: 0.0});
+      return response;
+    },
+    toJSON: function() {
+      var json = {};
+      json.sx = this.sx; // filter size in x, y dims
+      json.sy = this.sy;
+      json.stride = this.stride;
+      json.in_depth = this.in_depth;
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.l1_decay_mul = this.l1_decay_mul;
+      json.l2_decay_mul = this.l2_decay_mul;
+      json.pad = this.pad;
+      json.filters = [];
+      for(var i=0;i<this.filters.length;i++) {
+        json.filters.push(this.filters[i].toJSON());
+      }
+      json.biases = this.biases.toJSON();
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.sx = json.sx; // filter size in x, y dims
+      this.sy = json.sy;
+      this.stride = json.stride;
+      this.in_depth = json.in_depth; // depth of input volume
+      this.filters = [];
+      this.l1_decay_mul = typeof json.l1_decay_mul !== 'undefined' ? json.l1_decay_mul : 1.0;
+      this.l2_decay_mul = typeof json.l2_decay_mul !== 'undefined' ? json.l2_decay_mul : 1.0;
+      this.pad = typeof json.pad !== 'undefined' ? json.pad : 0;
+      for(var i=0;i<json.filters.length;i++) {
+        var v = new Vol(0,0,0,0);
+        v.fromJSON(json.filters[i]);
+        this.filters.push(v);
+      }
+      this.biases = new Vol(0,0,0,0);
+      this.biases.fromJSON(json.biases);
+    }
+  }
+
+  var FullyConnLayer = function(opt) {
+    var opt = opt || {};
+
+    // required
+    // ok fine we will allow 'filters' as the word as well
+    this.out_depth = typeof opt.num_neurons !== 'undefined' ? opt.num_neurons : opt.filters;
+
+    // optional 
+    this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0;
+    this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0;
+
+    // computed
+    this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
+    this.out_sx = 1;
+    this.out_sy = 1;
+    this.layer_type = 'fc';
+
+    // initializations
+    var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0;
+    this.filters = [];
+    for(var i=0;i<this.out_depth ;i++) { this.filters.push(new Vol(1, 1, this.num_inputs)); }
+    this.biases = new Vol(1, 1, this.out_depth, bias);
+  }
+
+  FullyConnLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var A = new Vol(1, 1, this.out_depth, 0.0);
+      var Vw = V.w;
+      for(var i=0;i<this.out_depth;i++) {
+        var a = 0.0;
+        var wi = this.filters[i].w;
+        for(var d=0;d<this.num_inputs;d++) {
+          a += Vw[d] * wi[d]; // for efficiency use Vols directly for now
+        }
+        a += this.biases.w[i];
+        A.w[i] = a;
+      }
+      this.out_act = A;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act;
+      V.dw = global.zeros(V.w.length); // zero out the gradient in input Vol
+      
+      // compute gradient wrt weights and data
+      for(var i=0;i<this.out_depth;i++) {
+        var tfi = this.filters[i];
+        var chain_grad = this.out_act.dw[i];
+        for(var d=0;d<this.num_inputs;d++) {
+          V.dw[d] += tfi.w[d]*chain_grad; // grad wrt input data
+          tfi.dw[d] += V.w[d]*chain_grad; // grad wrt params
+        }
+        this.biases.dw[i] += chain_grad;
+      }
+    },
+    getParamsAndGrads: function() {
+      var response = [];
+      for(var i=0;i<this.out_depth;i++) {
+        response.push({params: this.filters[i].w, grads: this.filters[i].dw, l1_decay_mul: this.l1_decay_mul, l2_decay_mul: this.l2_decay_mul});
+      }
+      response.push({params: this.biases.w, grads: this.biases.dw, l1_decay_mul: 0.0, l2_decay_mul: 0.0});
+      return response;
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.num_inputs = this.num_inputs;
+      json.l1_decay_mul = this.l1_decay_mul;
+      json.l2_decay_mul = this.l2_decay_mul;
+      json.filters = [];
+      for(var i=0;i<this.filters.length;i++) {
+        json.filters.push(this.filters[i].toJSON());
+      }
+      json.biases = this.biases.toJSON();
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.num_inputs = json.num_inputs;
+      this.l1_decay_mul = typeof json.l1_decay_mul !== 'undefined' ? json.l1_decay_mul : 1.0;
+      this.l2_decay_mul = typeof json.l2_decay_mul !== 'undefined' ? json.l2_decay_mul : 1.0;
+      this.filters = [];
+      for(var i=0;i<json.filters.length;i++) {
+        var v = new Vol(0,0,0,0);
+        v.fromJSON(json.filters[i]);
+        this.filters.push(v);
+      }
+      this.biases = new Vol(0,0,0,0);
+      this.biases.fromJSON(json.biases);
+    }
+  }
+
+  global.ConvLayer = ConvLayer;
+  global.FullyConnLayer = FullyConnLayer;
+  
+})(convnetjs);
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  var PoolLayer = function(opt) {
+
+    var opt = opt || {};
+
+    // required
+    this.sx = opt.sx; // filter size
+    this.in_depth = opt.in_depth;
+    this.in_sx = opt.in_sx;
+    this.in_sy = opt.in_sy;
+
+    // optional
+    this.sy = typeof opt.sy !== 'undefined' ? opt.sy : this.sx;
+    this.stride = typeof opt.stride !== 'undefined' ? opt.stride : 2;
+    this.pad = typeof opt.pad !== 'undefined' ? opt.pad : 0; // amount of 0 padding to add around borders of input volume
+
+    // computed
+    this.out_depth = this.in_depth;
+    this.out_sx = Math.floor((this.in_sx + this.pad * 2 - this.sx) / this.stride + 1);
+    this.out_sy = Math.floor((this.in_sy + this.pad * 2 - this.sy) / this.stride + 1);
+    this.layer_type = 'pool';
+    // store switches for x,y coordinates for where the max comes from, for each output neuron
+    this.switchx = global.zeros(this.out_sx*this.out_sy*this.out_depth);
+    this.switchy = global.zeros(this.out_sx*this.out_sy*this.out_depth);
+  }
+
+  PoolLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+
+      var A = new Vol(this.out_sx, this.out_sy, this.out_depth, 0.0);
+      
+      var n=0; // a counter for switches
+      for(var d=0;d<this.out_depth;d++) {
+        var x = -this.pad;
+        var y = -this.pad;
+        for(var ax=0; ax<this.out_sx; x+=this.stride,ax++) {
+          y = -this.pad;
+          for(var ay=0; ay<this.out_sy; y+=this.stride,ay++) {
+
+            // convolve centered at this particular location
+            var a = -99999; // hopefully small enough ;\
+            var winx=-1,winy=-1;
+            for(var fx=0;fx<this.sx;fx++) {
+              for(var fy=0;fy<this.sy;fy++) {
+                var oy = y+fy;
+                var ox = x+fx;
+                if(oy>=0 && oy<V.sy && ox>=0 && ox<V.sx) {
+                  var v = V.get(ox, oy, d);
+                  // perform max pooling and store pointers to where
+                  // the max came from. This will speed up backprop 
+                  // and can help make nice visualizations in future
+                  if(v > a) { a = v; winx=ox; winy=oy;}
+                }
+              }
+            }
+            this.switchx[n] = winx;
+            this.switchy[n] = winy;
+            n++;
+            A.set(ax, ay, d, a);
+          }
+        }
+      }
+      this.out_act = A;
+      return this.out_act;
+    },
+    backward: function() { 
+      // pooling layers have no parameters, so simply compute 
+      // gradient wrt data here
+      var V = this.in_act;
+      V.dw = global.zeros(V.w.length); // zero out gradient wrt data
+      var A = this.out_act; // computed in forward pass 
+
+      var n = 0;
+      for(var d=0;d<this.out_depth;d++) {
+        var x = -this.pad;
+        var y = -this.pad;
+        for(var ax=0; ax<this.out_sx; x+=this.stride,ax++) {
+          y = -this.pad;
+          for(var ay=0; ay<this.out_sy; y+=this.stride,ay++) {
+
+            var chain_grad = this.out_act.get_grad(ax,ay,d);
+            V.add_grad(this.switchx[n], this.switchy[n], d, chain_grad);
+            n++;
+
+          }
+        }
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.sx = this.sx;
+      json.sy = this.sy;
+      json.stride = this.stride;
+      json.in_depth = this.in_depth;
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.pad = this.pad;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.sx = json.sx;
+      this.sy = json.sy;
+      this.stride = json.stride;
+      this.in_depth = json.in_depth;
+      this.pad = typeof json.pad !== 'undefined' ? json.pad : 0; // backwards compatibility
+      this.switchx = global.zeros(this.out_sx*this.out_sy*this.out_depth); // need to re-init these appropriately
+      this.switchy = global.zeros(this.out_sx*this.out_sy*this.out_depth);
+    }
+  }
+
+  global.PoolLayer = PoolLayer;
+
+})(convnetjs);
+
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  var InputLayer = function(opt) {
+    var opt = opt || {};
+
+    // this is a bit silly but lets allow people to specify either ins or outs
+    this.out_sx = typeof opt.out_sx !== 'undefined' ? opt.out_sx : opt.in_sx;
+    this.out_sy = typeof opt.out_sy !== 'undefined' ? opt.out_sy : opt.in_sy;
+    this.out_depth = typeof opt.out_depth !== 'undefined' ? opt.out_depth : opt.in_depth;
+    this.layer_type = 'input';
+  }
+  InputLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      this.out_act = V;
+      return this.out_act; // dummy identity function for now
+    },
+    backward: function() { },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+    }
+  }
+
+  global.InputLayer = InputLayer;
+})(convnetjs);
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  // Layers that implement a loss. Currently these are the layers that 
+  // can initiate a backward() pass. In future we probably want a more 
+  // flexible system that can accomodate multiple losses to do multi-task
+  // learning, and stuff like that. But for now, one of the layers in this
+  // file must be the final layer in a Net.
+
+  // This is a classifier, with N discrete classes from 0 to N-1
+  // it gets a stream of N incoming numbers and computes the softmax
+  // function (exponentiate and normalize to sum to 1 as probabilities should)
+  var SoftmaxLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
+    this.out_depth = this.num_inputs;
+    this.out_sx = 1;
+    this.out_sy = 1;
+    this.layer_type = 'softmax';
+  }
+
+  SoftmaxLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+
+      var A = new Vol(1, 1, this.out_depth, 0.0);
+
+      // compute max activation
+      var as = V.w;
+      var amax = V.w[0];
+      for(var i=1;i<this.out_depth;i++) {
+        if(as[i] > amax) amax = as[i];
+      }
+
+      // compute exponentials (carefully to not blow up)
+      var es = global.zeros(this.out_depth);
+      var esum = 0.0;
+      for(var i=0;i<this.out_depth;i++) {
+        var e = Math.exp(as[i] - amax);
+        esum += e;
+        es[i] = e;
+      }
+
+      // normalize and output to sum to one
+      for(var i=0;i<this.out_depth;i++) {
+        es[i] /= esum;
+        A.w[i] = es[i];
+      }
+
+      this.es = es; // save these for backprop
+      this.out_act = A;
+      return this.out_act;
+    },
+    backward: function(y) {
+
+      // compute and accumulate gradient wrt weights and bias of this layer
+      var x = this.in_act;
+      x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
+
+      for(var i=0;i<this.out_depth;i++) {
+        var indicator = i === y ? 1.0 : 0.0;
+        var mul = -(indicator - this.es[i]);
+        x.dw[i] = mul;
+      }
+
+      // loss is the class negative log likelihood
+      return -Math.log(this.es[y]);
+    },
+    getParamsAndGrads: function() { 
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.num_inputs = this.num_inputs;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.num_inputs = json.num_inputs;
+    }
+  }
+
+  // implements an L2 regression cost layer,
+  // so penalizes \sum_i(||x_i - y_i||^2), where x is its input
+  // and y is the user-provided array of "correct" values.
+  var RegressionLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
+    this.out_depth = this.num_inputs;
+    this.out_sx = 1;
+    this.out_sy = 1;
+    this.layer_type = 'regression';
+  }
+
+  RegressionLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      this.out_act = V;
+      return V; // identity function
+    },
+    // y is a list here of size num_inputs
+    backward: function(y) { 
+
+      // compute and accumulate gradient wrt weights and bias of this layer
+      var x = this.in_act;
+      x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
+      var loss = 0.0;
+      if(y instanceof Array || y instanceof Float64Array) {
+        for(var i=0;i<this.out_depth;i++) {
+          var dy = x.w[i] - y[i];
+          x.dw[i] = dy;
+          loss += 2*dy*dy;
+        }
+      } else {
+        // assume it is a struct with entries .dim and .val
+        // and we pass gradient only along dimension dim to be equal to val
+        var i = y.dim;
+        var yi = y.val;
+        var dy = x.w[i] - yi;
+        x.dw[i] = dy;
+        loss += 2*dy*dy;
+      }
+      return loss;
+    },
+    getParamsAndGrads: function() { 
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.num_inputs = this.num_inputs;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.num_inputs = json.num_inputs;
+    }
+  }
+
+  var SVMLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.num_inputs = opt.in_sx * opt.in_sy * opt.in_depth;
+    this.out_depth = this.num_inputs;
+    this.out_sx = 1;
+    this.out_sy = 1;
+    this.layer_type = 'svm';
+  }
+
+  SVMLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      this.out_act = V; // nothing to do, output raw scores
+      return V;
+    },
+    backward: function(y) {
+
+      // compute and accumulate gradient wrt weights and bias of this layer
+      var x = this.in_act;
+      x.dw = global.zeros(x.w.length); // zero out the gradient of input Vol
+
+      var yscore = x.w[y]; // score of ground truth
+      var margin = 1.0;
+      var loss = 0.0;
+      for(var i=0;i<this.out_depth;i++) {
+        if(-yscore + x.w[i] + margin > 0) {
+          // violating example, apply loss
+          // I love hinge loss, by the way. Truly.
+          // Seriously, compare this SVM code with Softmax forward AND backprop code above
+          // it's clear which one is superior, not only in code, simplicity
+          // and beauty, but also in practice.
+          x.dw[i] += 1;
+          x.dw[y] -= 1;
+          loss += -yscore + x.w[i] + margin;
+        }
+      }
+
+      return loss;
+    },
+    getParamsAndGrads: function() { 
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.num_inputs = this.num_inputs;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type;
+      this.num_inputs = json.num_inputs;
+    }
+  }
+  
+  global.RegressionLayer = RegressionLayer;
+  global.SoftmaxLayer = SoftmaxLayer;
+  global.SVMLayer = SVMLayer;
+
+})(convnetjs);
+
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  // Implements ReLU nonlinearity elementwise
+  // x -> max(0, x)
+  // the output is in [0, inf)
+  var ReluLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'relu';
+  }
+  ReluLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var V2 = V.clone();
+      var N = V.w.length;
+      var V2w = V2.w;
+      for(var i=0;i<N;i++) { 
+        if(V2w[i] < 0) V2w[i] = 0; // threshold at 0
+      }
+      this.out_act = V2;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var V2 = this.out_act;
+      var N = V.w.length;
+      V.dw = global.zeros(N); // zero out gradient wrt data
+      for(var i=0;i<N;i++) {
+        if(V2.w[i] <= 0) V.dw[i] = 0; // threshold
+        else V.dw[i] = V2.dw[i];
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+    }
+  }
+
+  // Implements Sigmoid nnonlinearity elementwise
+  // x -> 1/(1+e^(-x))
+  // so the output is between 0 and 1.
+  var SigmoidLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'sigmoid';
+  }
+  SigmoidLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var V2 = V.cloneAndZero();
+      var N = V.w.length;
+      var V2w = V2.w;
+      var Vw = V.w;
+      for(var i=0;i<N;i++) { 
+        V2w[i] = 1.0/(1.0+Math.exp(-Vw[i]));
+      }
+      this.out_act = V2;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var V2 = this.out_act;
+      var N = V.w.length;
+      V.dw = global.zeros(N); // zero out gradient wrt data
+      for(var i=0;i<N;i++) {
+        var v2wi = V2.w[i];
+        V.dw[i] =  v2wi * (1.0 - v2wi) * V2.dw[i];
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+    }
+  }
+
+  // Implements Maxout nnonlinearity that computes
+  // x -> max(x)
+  // where x is a vector of size group_size. Ideally of course,
+  // the input size should be exactly divisible by group_size
+  var MaxoutLayer = function(opt) {
+    var opt = opt || {};
+
+    // required
+    this.group_size = typeof opt.group_size !== 'undefined' ? opt.group_size : 2;
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = Math.floor(opt.in_depth / this.group_size);
+    this.layer_type = 'maxout';
+
+    this.switches = global.zeros(this.out_sx*this.out_sy*this.out_depth); // useful for backprop
+  }
+  MaxoutLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var N = this.out_depth; 
+      var V2 = new Vol(this.out_sx, this.out_sy, this.out_depth, 0.0);
+
+      // optimization branch. If we're operating on 1D arrays we dont have
+      // to worry about keeping track of x,y,d coordinates inside
+      // input volumes. In convnets we do :(
+      if(this.out_sx === 1 && this.out_sy === 1) {
+        for(var i=0;i<N;i++) {
+          var ix = i * this.group_size; // base index offset
+          var a = V.w[ix];
+          var ai = 0;
+          for(var j=1;j<this.group_size;j++) {
+            var a2 = V.w[ix+j];
+            if(a2 > a) {
+              a = a2;
+              ai = j;
+            }
+          }
+          V2.w[i] = a;
+          this.switches[i] = ix + ai;
+        }
+      } else {
+        var n=0; // counter for switches
+        for(var x=0;x<V.sx;x++) {
+          for(var y=0;y<V.sy;y++) {
+            for(var i=0;i<N;i++) {
+              var ix = i * this.group_size;
+              var a = V.get(x, y, ix);
+              var ai = 0;
+              for(var j=1;j<this.group_size;j++) {
+                var a2 = V.get(x, y, ix+j);
+                if(a2 > a) {
+                  a = a2;
+                  ai = j;
+                }
+              }
+              V2.set(x,y,i,a);
+              this.switches[n] = ix + ai;
+              n++;
+            }
+          }
+        }
+
+      }
+      this.out_act = V2;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var V2 = this.out_act;
+      var N = this.out_depth;
+      V.dw = global.zeros(V.w.length); // zero out gradient wrt data
+
+      // pass the gradient through the appropriate switch
+      if(this.out_sx === 1 && this.out_sy === 1) {
+        for(var i=0;i<N;i++) {
+          var chain_grad = V2.dw[i];
+          V.dw[this.switches[i]] = chain_grad;
+        }
+      } else {
+        // bleh okay, lets do this the hard way
+        var n=0; // counter for switches
+        for(var x=0;x<V2.sx;x++) {
+          for(var y=0;y<V2.sy;y++) {
+            for(var i=0;i<N;i++) {
+              var chain_grad = V2.get_grad(x,y,i);
+              V.set_grad(x,y,this.switches[n],chain_grad);
+              n++;
+            }
+          }
+        }
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.group_size = this.group_size;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+      this.group_size = json.group_size;
+      this.switches = global.zeros(this.group_size);
+    }
+  }
+
+  // a helper function, since tanh is not yet part of ECMAScript. Will be in v6.
+  function tanh(x) {
+    var y = Math.exp(2 * x);
+    return (y - 1) / (y + 1);
+  }
+  // Implements Tanh nnonlinearity elementwise
+  // x -> tanh(x) 
+  // so the output is between -1 and 1.
+  var TanhLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'tanh';
+  }
+  TanhLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      var V2 = V.cloneAndZero();
+      var N = V.w.length;
+      for(var i=0;i<N;i++) { 
+        V2.w[i] = tanh(V.w[i]);
+      }
+      this.out_act = V2;
+      return this.out_act;
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var V2 = this.out_act;
+      var N = V.w.length;
+      V.dw = global.zeros(N); // zero out gradient wrt data
+      for(var i=0;i<N;i++) {
+        var v2wi = V2.w[i];
+        V.dw[i] = (1.0 - v2wi * v2wi) * V2.dw[i];
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+    }
+  }
+  
+  global.TanhLayer = TanhLayer;
+  global.MaxoutLayer = MaxoutLayer;
+  global.ReluLayer = ReluLayer;
+  global.SigmoidLayer = SigmoidLayer;
+
+})(convnetjs);
+
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+
+  // An inefficient dropout layer
+  // Note this is not most efficient implementation since the layer before
+  // computed all these activations and now we're just going to drop them :(
+  // same goes for backward pass. Also, if we wanted to be efficient at test time
+  // we could equivalently be clever and upscale during train and copy pointers during test
+  // todo: make more efficient.
+  var DropoutLayer = function(opt) {
+    var opt = opt || {};
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'dropout';
+    this.drop_prob = typeof opt.drop_prob !== 'undefined' ? opt.drop_prob : 0.5;
+    this.dropped = global.zeros(this.out_sx*this.out_sy*this.out_depth);
+  }
+  DropoutLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+      if(typeof(is_training)==='undefined') { is_training = false; } // default is prediction mode
+      var V2 = V.clone();
+      var N = V.w.length;
+      if(is_training) {
+        // do dropout
+        for(var i=0;i<N;i++) {
+          if(Math.random()<this.drop_prob) { V2.w[i]=0; this.dropped[i] = true; } // drop!
+          else {this.dropped[i] = false;}
+        }
+      } else {
+        // scale the activations during prediction
+        for(var i=0;i<N;i++) { V2.w[i]*=this.drop_prob; }
+      }
+      this.out_act = V2;
+      return this.out_act; // dummy identity function for now
+    },
+    backward: function() {
+      var V = this.in_act; // we need to set dw of this
+      var chain_grad = this.out_act;
+      var N = V.w.length;
+      V.dw = global.zeros(N); // zero out gradient wrt data
+      for(var i=0;i<N;i++) {
+        if(!(this.dropped[i])) { 
+          V.dw[i] = chain_grad.dw[i]; // copy over the gradient
+        }
+      }
+    },
+    getParamsAndGrads: function() {
+      return [];
+    },
+    toJSON: function() {
+      var json = {};
+      json.out_depth = this.out_depth;
+      json.out_sx = this.out_sx;
+      json.out_sy = this.out_sy;
+      json.layer_type = this.layer_type;
+      json.drop_prob = this.drop_prob;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.out_depth = json.out_depth;
+      this.out_sx = json.out_sx;
+      this.out_sy = json.out_sy;
+      this.layer_type = json.layer_type; 
+      this.drop_prob = json.drop_prob;
+    }
+  }
+  
+
+  global.DropoutLayer = DropoutLayer;
+})(convnetjs);
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  // a bit experimental layer for now. I think it works but I'm not 100%
+  // the gradient check is a bit funky. I'll look into this a bit later.
+  // Local Response Normalization in window, along depths of volumes
+  var LocalResponseNormalizationLayer = function(opt) {
+    var opt = opt || {};
+
+    // required
+    this.k = opt.k;
+    this.n = opt.n;
+    this.alpha = opt.alpha;
+    this.beta = opt.beta;
+
+    // computed
+    this.out_sx = opt.in_sx;
+    this.out_sy = opt.in_sy;
+    this.out_depth = opt.in_depth;
+    this.layer_type = 'lrn';
+
+    // checks
+    if(this.n%2 === 0) { console.log('WARNING n should be odd for LRN layer'); }
+  }
+  LocalResponseNormalizationLayer.prototype = {
+    forward: function(V, is_training) {
+      this.in_act = V;
+
+      var A = V.cloneAndZero();
+      this.S_cache_ = V.cloneAndZero();
+      var n2 = Math.floor(this.n/2);
+      for(var x=0;x<V.sx;x++) {
+        for(var y=0;y<V.sy;y++) {
+          for(var i=0;i<V.depth;i++) {
+
+            var ai = V.get(x,y,i);
+
+            // normalize in a window of size n
+            var den = 0.0;
+            for(var j=Math.max(0,i-n2);j<=Math.min(i+n2,V.depth-1);j++) {
+              var aa = V.get(x,y,j);
+              den += aa*aa;
+            }
+            den *= this.alpha / this.n;
+            den += this.k;
+            this.S_cache_.set(x,y,i,den); // will be useful for backprop
+            den = Math.pow(den, this.beta);
+            A.set(x,y,i,ai/den);
+          }
+        }
+      }
+
+      this.out_act = A;
+      return this.out_act; // dummy identity function for now
+    },
+    backward: function() { 
+      // evaluate gradient wrt data
+      var V = this.in_act; // we need to set dw of this
+      V.dw = global.zeros(V.w.length); // zero out gradient wrt data
+      var A = this.out_act; // computed in forward pass 
+
+      var n2 = Math.floor(this.n/2);
+      for(var x=0;x<V.sx;x++) {
+        for(var y=0;y<V.sy;y++) {
+          for(var i=0;i<V.depth;i++) {
+
+            var chain_grad = this.out_act.get_grad(x,y,i);
+            var S = this.S_cache_.get(x,y,i);
+            var SB = Math.pow(S, this.beta);
+            var SB2 = SB*SB;
+
+            // normalize in a window of size n
+            for(var j=Math.max(0,i-n2);j<=Math.min(i+n2,V.depth-1);j++) {              
+              var aj = V.get(x,y,j); 
+              var g = -aj*this.beta*Math.pow(S,this.beta-1)*this.alpha/this.n*2*aj;
+              if(j===i) g+= SB;
+              g /= SB2;
+              g *= chain_grad;
+              V.add_grad(x,y,j,g);
+            }
+
+          }
+        }
+      }
+    },
+    getParamsAndGrads: function() { return []; },
+    toJSON: function() {
+      var json = {};
+      json.k = this.k;
+      json.n = this.n;
+      json.alpha = this.alpha; // normalize by size
+      json.beta = this.beta;
+      json.out_sx = this.out_sx; 
+      json.out_sy = this.out_sy;
+      json.out_depth = this.out_depth;
+      json.layer_type = this.layer_type;
+      return json;
+    },
+    fromJSON: function(json) {
+      this.k = json.k;
+      this.n = json.n;
+      this.alpha = json.alpha; // normalize by size
+      this.beta = json.beta;
+      this.out_sx = json.out_sx; 
+      this.out_sy = json.out_sy;
+      this.out_depth = json.out_depth;
+      this.layer_type = json.layer_type;
+    }
+  }
+  
+
+  global.LocalResponseNormalizationLayer = LocalResponseNormalizationLayer;
+})(convnetjs);
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+  
+  // Net manages a set of layers
+  // For now constraints: Simple linear order of layers, first layer input last layer a cost layer
+  var Net = function(options) {
+    this.layers = [];
+  }
+
+  Net.prototype = {
+    
+    // takes a list of layer definitions and creates the network layer objects
+    makeLayers: function(defs) {
+
+      // few checks for now
+      if(defs.length<2) {console.log('ERROR! For now at least have input and softmax layers.');}
+      if(defs[0].type !== 'input') {console.log('ERROR! For now first layer should be input.');}
+
+      // desugar syntactic for adding activations and dropouts
+      var desugar = function() {
+        var new_defs = [];
+        for(var i=0;i<defs.length;i++) {
+          var def = defs[i];
+          
+          if(def.type==='softmax' || def.type==='svm') {
+            // add an fc layer here, there is no reason the user should
+            // have to worry about this and we almost always want to
+            new_defs.push({type:'fc', num_neurons: def.num_classes});
+          }
+
+          if(def.type==='regression') {
+            // add an fc layer here, there is no reason the user should
+            // have to worry about this and we almost always want to
+            new_defs.push({type:'fc', num_neurons: def.num_neurons});
+          }
+
+          if((def.type==='fc' || def.type==='conv') 
+              && typeof(def.bias_pref) === 'undefined'){
+            def.bias_pref = 0.0;
+            if(typeof def.activation !== 'undefined' && def.activation === 'relu') {
+              def.bias_pref = 0.1; // relus like a bit of positive bias to get gradients early
+              // otherwise it's technically possible that a relu unit will never turn on (by chance)
+              // and will never get any gradient and never contribute any computation. Dead relu.
+            }
+          }
+          
+          if(typeof def.tensor !== 'undefined') {
+            // apply quadratic transform so that the upcoming multiply will include
+            // quadratic terms, equivalent to doing a tensor product
+            if(def.tensor) {
+              new_defs.push({type: 'quadtransform'});
+            }
+          }
+
+          new_defs.push(def);
+
+          if(typeof def.activation !== 'undefined') {
+            if(def.activation==='relu') { new_defs.push({type:'relu'}); }
+            else if (def.activation==='sigmoid') { new_defs.push({type:'sigmoid'}); }
+            else if (def.activation==='tanh') { new_defs.push({type:'tanh'}); }
+            else if (def.activation==='maxout') {
+              // create maxout activation, and pass along group size, if provided
+              var gs = def.group_size !== 'undefined' ? def.group_size : 2;
+              new_defs.push({type:'maxout', group_size:gs});
+            }
+            else { console.log('ERROR unsupported activation ' + def.activation); }
+          }
+          if(typeof def.drop_prob !== 'undefined' && def.type !== 'dropout') {
+            new_defs.push({type:'dropout', drop_prob: def.drop_prob});
+          }
+
+        }
+        return new_defs;
+      }
+      defs = desugar(defs);
+
+      // create the layers
+      this.layers = [];
+      for(var i=0;i<defs.length;i++) {
+        var def = defs[i];
+        if(i>0) {
+          var prev = this.layers[i-1];
+          def.in_sx = prev.out_sx;
+          def.in_sy = prev.out_sy;
+          def.in_depth = prev.out_depth;
+        }
+
+        switch(def.type) {
+          case 'fc': this.layers.push(new global.FullyConnLayer(def)); break;
+          case 'lrn': this.layers.push(new global.LocalResponseNormalizationLayer(def)); break;
+          case 'dropout': this.layers.push(new global.DropoutLayer(def)); break;
+          case 'input': this.layers.push(new global.InputLayer(def)); break;
+          case 'softmax': this.layers.push(new global.SoftmaxLayer(def)); break;
+          case 'regression': this.layers.push(new global.RegressionLayer(def)); break;
+          case 'conv': this.layers.push(new global.ConvLayer(def)); break;
+          case 'pool': this.layers.push(new global.PoolLayer(def)); break;
+          case 'relu': this.layers.push(new global.ReluLayer(def)); break;
+          case 'sigmoid': this.layers.push(new global.SigmoidLayer(def)); break;
+          case 'tanh': this.layers.push(new global.TanhLayer(def)); break;
+          case 'maxout': this.layers.push(new global.MaxoutLayer(def)); break;
+          case 'quadtransform': this.layers.push(new global.QuadTransformLayer(def)); break;
+          case 'svm': this.layers.push(new global.SVMLayer(def)); break;
+          default: console.log('ERROR: UNRECOGNIZED LAYER TYPE!');
+        }
+      }
+    },
+
+    // forward prop the network. A trainer will pass in is_training = true
+    forward: function(V, is_training) {
+      if(typeof(is_training)==='undefined') is_training = false;
+      var act = this.layers[0].forward(V, is_training);
+      for(var i=1;i<this.layers.length;i++) {
+        act = this.layers[i].forward(act, is_training);
+      }
+      return act;
+    },
+
+    getCostLoss: function(V, y) {
+      this.forward(V, false);
+      var N = this.layers.length;
+      var loss = this.layers[N-1].backward(y);
+      return loss;
+    },
+    
+    // backprop: compute gradients wrt all parameters
+    backward: function(y) {
+      var N = this.layers.length;
+      var loss = this.layers[N-1].backward(y); // last layer assumed softmax
+      for(var i=N-2;i>=0;i--) { // first layer assumed input
+        this.layers[i].backward();
+      }
+      return loss;
+    },
+    getParamsAndGrads: function() {
+      // accumulate parameters and gradients for the entire network
+      var response = [];
+      for(var i=0;i<this.layers.length;i++) {
+        var layer_reponse = this.layers[i].getParamsAndGrads();
+        for(var j=0;j<layer_reponse.length;j++) {
+          response.push(layer_reponse[j]);
+        }
+      }
+      return response;
+    },
+    getPrediction: function() {
+      var S = this.layers[this.layers.length-1]; // softmax layer
+      var p = S.out_act.w;
+      var maxv = p[0];
+      var maxi = 0;
+      for(var i=1;i<p.length;i++) {
+        if(p[i] > maxv) { maxv = p[i]; maxi = i;}
+      }
+      return maxi;
+    },
+    toJSON: function() {
+      var json = {};
+      json.layers = [];
+      for(var i=0;i<this.layers.length;i++) {
+        json.layers.push(this.layers[i].toJSON());
+      }
+      return json;
+    },
+    fromJSON: function(json) {
+      this.layers = [];
+      for(var i=0;i<json.layers.length;i++) {
+        var Lj = json.layers[i]
+        var t = Lj.layer_type;
+        var L;
+        if(t==='input') { L = new global.InputLayer(); }
+        if(t==='relu') { L = new global.ReluLayer(); }
+        if(t==='sigmoid') { L = new global.SigmoidLayer(); }
+        if(t==='tanh') { L = new global.TanhLayer(); }
+        if(t==='dropout') { L = new global.DropoutLayer(); }
+        if(t==='conv') { L = new global.ConvLayer(); }
+        if(t==='pool') { L = new global.PoolLayer(); }
+        if(t==='lrn') { L = new global.LocalResponseNormalizationLayer(); }
+        if(t==='softmax') { L = new global.SoftmaxLayer(); }
+        if(t==='regression') { L = new global.RegressionLayer(); }
+        if(t==='fc') { L = new global.FullyConnLayer(); }
+        if(t==='maxout') { L = new global.MaxoutLayer(); }
+        if(t==='quadtransform') { L = new global.QuadTransformLayer(); }
+        if(t==='svm') { L = new global.SVMLayer(); }
+        L.fromJSON(Lj);
+        this.layers.push(L);
+      }
+    }
+  }
+  
+
+  global.Net = Net;
+})(convnetjs);
+(function(global) {
+  "use strict";
+  var Vol = global.Vol; // convenience
+
+  var Trainer = function(net, options) {
+
+    this.net = net;
+
+    var options = options || {};
+    this.learning_rate = typeof options.learning_rate !== 'undefined' ? options.learning_rate : 0.01;
+    this.l1_decay = typeof options.l1_decay !== 'undefined' ? options.l1_decay : 0.0;
+    this.l2_decay = typeof options.l2_decay !== 'undefined' ? options.l2_decay : 0.0;
+    this.batch_size = typeof options.batch_size !== 'undefined' ? options.batch_size : 1;
+    this.method = typeof options.method !== 'undefined' ? options.method : 'sgd'; // sgd/adagrad/adadelta/windowgrad
+
+    this.momentum = typeof options.momentum !== 'undefined' ? options.momentum : 0.9;
+    this.ro = typeof options.ro !== 'undefined' ? options.ro : 0.95; // used in adadelta
+    this.eps = typeof options.eps !== 'undefined' ? options.eps : 1e-6; // used in adadelta
+
+    this.k = 0; // iteration counter
+    this.gsum = []; // last iteration gradients (used for momentum calculations)
+    this.xsum = []; // used in adadelta
+  }
+
+  Trainer.prototype = {
+    train: function(x, y) {
+
+      var start = new Date().getTime();
+      this.net.forward(x, true); // also set the flag that lets the net know we're just training
+      var end = new Date().getTime();
+      var fwd_time = end - start;
+
+      var start = new Date().getTime();
+      var cost_loss = this.net.backward(y);
+      var l2_decay_loss = 0.0;
+      var l1_decay_loss = 0.0;
+      var end = new Date().getTime();
+      var bwd_time = end - start;
+      
+      this.k++;
+      if(this.k % this.batch_size === 0) {
+
+        var pglist = this.net.getParamsAndGrads();
+
+        // initialize lists for accumulators. Will only be done once on first iteration
+        if(this.gsum.length === 0 && (this.method !== 'sgd' || this.momentum > 0.0)) {
+          // only vanilla sgd doesnt need either lists
+          // momentum needs gsum
+          // adagrad needs gsum
+          // adadelta needs gsum and xsum
+          for(var i=0;i<pglist.length;i++) {
+            this.gsum.push(global.zeros(pglist[i].params.length));
+            if(this.method === 'adadelta') {
+              this.xsum.push(global.zeros(pglist[i].params.length));
+            } else {
+              this.xsum.push([]); // conserve memory
+            }
+          }
+        }
+
+        // perform an update for all sets of weights
+        for(var i=0;i<pglist.length;i++) {
+          var pg = pglist[i]; // param, gradient, other options in future (custom learning rate etc)
+          var p = pg.params;
+          var g = pg.grads;
+
+          // learning rate for some parameters.
+          var l2_decay_mul = typeof pg.l2_decay_mul !== 'undefined' ? pg.l2_decay_mul : 1.0;
+          var l1_decay_mul = typeof pg.l1_decay_mul !== 'undefined' ? pg.l1_decay_mul : 1.0;
+          var l2_decay = this.l2_decay * l2_decay_mul;
+          var l1_decay = this.l1_decay * l1_decay_mul;
+
+          var plen = p.length;
+          for(var j=0;j<plen;j++) {
+            l2_decay_loss += l2_decay*p[j]*p[j]/2; // accumulate weight decay loss
+            l1_decay_loss += l1_decay*Math.abs(p[j]);
+            var l1grad = l1_decay * (p[j] > 0 ? 1 : -1);
+            var l2grad = l2_decay * (p[j]);
+
+            var gij = (l2grad + l1grad + g[j]) / this.batch_size; // raw batch gradient
+
+            var gsumi = this.gsum[i];
+            var xsumi = this.xsum[i];
+            if(this.method === 'adagrad') {
+              // adagrad update
+              gsumi[j] = gsumi[j] + gij * gij;
+              var dx = - this.learning_rate / Math.sqrt(gsumi[j] + this.eps) * gij;
+              p[j] += dx;
+            } else if(this.method === 'windowgrad') {
+              // this is adagrad but with a moving window weighted average
+              // so the gradient is not accumulated over the entire history of the run. 
+              // it's also referred to as Idea #1 in Zeiler paper on Adadelta. Seems reasonable to me!
+              gsumi[j] = this.ro * gsumi[j] + (1-this.ro) * gij * gij;
+              var dx = - this.learning_rate / Math.sqrt(gsumi[j] + this.eps) * gij; // eps added for better conditioning
+              p[j] += dx;
+            } else if(this.method === 'adadelta') {
+              // assume adadelta if not sgd or adagrad
+              gsumi[j] = this.ro * gsumi[j] + (1-this.ro) * gij * gij;
+              var dx = - Math.sqrt((xsumi[j] + this.eps)/(gsumi[j] + this.eps)) * gij;
+              xsumi[j] = this.ro * xsumi[j] + (1-this.ro) * dx * dx; // yes, xsum lags behind gsum by 1.
+              p[j] += dx;
+            } else {
+              // assume SGD
+              if(this.momentum > 0.0) {
+                // momentum update
+                var dx = this.momentum * gsumi[j] - this.learning_rate * gij; // step
+                gsumi[j] = dx; // back this up for next iteration of momentum
+                p[j] += dx; // apply corrected gradient
+              } else {
+                // vanilla sgd
+                p[j] +=  - this.learning_rate * gij;
+              }
+            }
+            g[j] = 0.0; // zero out gradient so that we can begin accumulating anew
+          }
+        }
+      }
+
+      // appending softmax_loss for backwards compatibility, but from now on we will always use cost_loss
+      // in future, TODO: have to completely redo the way loss is done around the network as currently 
+      // loss is a bit of a hack. Ideally, user should specify arbitrary number of loss functions on any layer
+      // and it should all be computed correctly and automatically. 
+      return {fwd_time: fwd_time, bwd_time: bwd_time, 
+              l2_decay_loss: l2_decay_loss, l1_decay_loss: l1_decay_loss,
+              cost_loss: cost_loss, softmax_loss: cost_loss, 
+              loss: cost_loss + l1_decay_loss + l2_decay_loss}
+    }
+  }
+  
+  global.Trainer = Trainer;
+  global.SGDTrainer = Trainer; // backwards compatibility
+})(convnetjs);
+
+(function(global) {
+  "use strict";
+
+  // used utilities, make explicit local references
+  var randf = global.randf;
+  var randi = global.randi;
+  var Net = global.Net;
+  var Trainer = global.Trainer;
+  var maxmin = global.maxmin;
+  var randperm = global.randperm;
+  var weightedSample = global.weightedSample;
+  var getopt = global.getopt;
+  var arrUnique = global.arrUnique;
+
+  /*
+  A MagicNet takes data: a list of convnetjs.Vol(), and labels
+  which for now are assumed to be class indeces 0..K. MagicNet then:
+  - creates data folds for cross-validation
+  - samples candidate networks
+  - evaluates candidate networks on all data folds
+  - produces predictions by model-averaging the best networks
+  */
+  var MagicNet = function(data, labels, opt) {
+    var opt = opt || {};
+    if(typeof data === 'undefined') { data = []; }
+    if(typeof labels === 'undefined') { labels = []; }
+
+    // required inputs
+    this.data = data; // store these pointers to data
+    this.labels = labels;
+
+    // optional inputs
+    this.train_ratio = getopt(opt, 'train_ratio', 0.7);
+    this.num_folds = getopt(opt, 'num_folds', 10);
+    this.num_candidates = getopt(opt, 'num_candidates', 50); // we evaluate several in parallel
+    // how many epochs of data to train every network? for every fold?
+    // higher values mean higher accuracy in final results, but more expensive
+    this.num_epochs = getopt(opt, 'num_epochs', 50); 
+    // number of best models to average during prediction. Usually higher = better
+    this.ensemble_size = getopt(opt, 'ensemble_size', 10);
+
+    // candidate parameters
+    this.batch_size_min = getopt(opt, 'batch_size_min', 10);
+    this.batch_size_max = getopt(opt, 'batch_size_max', 300);
+    this.l2_decay_min = getopt(opt, 'l2_decay_min', -4);
+    this.l2_decay_max = getopt(opt, 'l2_decay_max', 2);
+    this.learning_rate_min = getopt(opt, 'learning_rate_min', -4);
+    this.learning_rate_max = getopt(opt, 'learning_rate_max', 0);
+    this.momentum_min = getopt(opt, 'momentum_min', 0.9);
+    this.momentum_max = getopt(opt, 'momentum_max', 0.9);
+    this.neurons_min = getopt(opt, 'neurons_min', 5);
+    this.neurons_max = getopt(opt, 'neurons_max', 30);
+
+    // computed
+    this.folds = []; // data fold indices, gets filled by sampleFolds()
+    this.candidates = []; // candidate networks that are being currently evaluated
+    this.evaluated_candidates = []; // history of all candidates that were fully evaluated on all folds
+    this.unique_labels = arrUnique(labels);
+    this.iter = 0; // iteration counter, goes from 0 -> num_epochs * num_training_data
+    this.foldix = 0; // index of active fold
+
+    // callbacks
+    this.finish_fold_callback = null;
+    this.finish_batch_callback = null;
+
+    // initializations
+    if(this.data.length > 0) {
+      this.sampleFolds();
+      this.sampleCandidates();
+    }
+  };
+
+  MagicNet.prototype = {
+
+    // sets this.folds to a sampling of this.num_folds folds
+    sampleFolds: function() {
+      var N = this.data.length;
+      var num_train = Math.floor(this.train_ratio * N);
+      this.folds = []; // flush folds, if any
+      for(var i=0;i<this.num_folds;i++) {
+        var p = randperm(N);
+        this.folds.push({train_ix: p.slice(0, num_train), test_ix: p.slice(num_train, N)});
+      }
+    },
+
+    // returns a random candidate network
+    sampleCandidate: function() {
+      var input_depth = this.data[0].w.length;
+      var num_classes = this.unique_labels.length;
+
+      // sample network topology and hyperparameters
+      var layer_defs = [];
+      layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth: input_depth});
+      var nl = weightedSample([0,1,2,3], [0.2, 0.3, 0.3, 0.2]); // prefer nets with 1,2 hidden layers
+      for(var q=0;q<nl;q++) {
+        var ni = randi(this.neurons_min, this.neurons_max);
+        var act = ['tanh','maxout','relu'][randi(0,3)];
+        if(randf(0,1)<0.5) {
+          var dp = Math.random();
+          layer_defs.push({type:'fc', num_neurons: ni, activation: act, drop_prob: dp});
+        } else {
+          layer_defs.push({type:'fc', num_neurons: ni, activation: act});
+        }
+      }
+      layer_defs.push({type:'softmax', num_classes: num_classes});
+      var net = new Net();
+      net.makeLayers(layer_defs);
+
+      // sample training hyperparameters
+      var bs = randi(this.batch_size_min, this.batch_size_max); // batch size
+      var l2 = Math.pow(10, randf(this.l2_decay_min, this.l2_decay_max)); // l2 weight decay
+      var lr = Math.pow(10, randf(this.learning_rate_min, this.learning_rate_max)); // learning rate
+      var mom = randf(this.momentum_min, this.momentum_max); // momentum. Lets just use 0.9, works okay usually ;p
+      var tp = randf(0,1); // trainer type
+      var trainer_def;
+      if(tp<0.33) {
+        trainer_def = {method:'adadelta', batch_size:bs, l2_decay:l2};
+      } else if(tp<0.66) {
+        trainer_def = {method:'adagrad', learning_rate: lr, batch_size:bs, l2_decay:l2};
+      } else {
+        trainer_def = {method:'sgd', learning_rate: lr, momentum: mom, batch_size:bs, l2_decay:l2};
+      }
+      
+      var trainer = new Trainer(net, trainer_def);
+
+      var cand = {};
+      cand.acc = [];
+      cand.accv = 0; // this will maintained as sum(acc) for convenience
+      cand.layer_defs = layer_defs;
+      cand.trainer_def = trainer_def;
+      cand.net = net;
+      cand.trainer = trainer;
+      return cand;
+    },
+
+    // sets this.candidates with this.num_candidates candidate nets
+    sampleCandidates: function() {
+      this.candidates = []; // flush, if any
+      for(var i=0;i<this.num_candidates;i++) {
+        var cand = this.sampleCandidate();
+        this.candidates.push(cand);
+      }
+    },
+
+    step: function() {
+      
+      // run an example through current candidate
+      this.iter++;
+
+      // step all candidates on a random data point
+      var fold = this.folds[this.foldix]; // active fold
+      var dataix = fold.train_ix[randi(0, fold.train_ix.length)];
+      for(var k=0;k<this.candidates.length;k++) {
+        var x = this.data[dataix];
+        var l = this.labels[dataix];
+        this.candidates[k].trainer.train(x, l);
+      }
+
+      // process consequences: sample new folds, or candidates
+      var lastiter = this.num_epochs * fold.train_ix.length;
+      if(this.iter >= lastiter) {
+        // finished evaluation of this fold. Get final validation
+        // accuracies, record them, and go on to next fold.
+        var val_acc = this.evalValErrors();
+        for(var k=0;k<this.candidates.length;k++) {
+          var c = this.candidates[k];
+          c.acc.push(val_acc[k]);
+          c.accv += val_acc[k];
+        }
+        this.iter = 0; // reset step number
+        this.foldix++; // increment fold
+
+        if(this.finish_fold_callback !== null) {
+          this.finish_fold_callback();
+        }
+
+        if(this.foldix >= this.folds.length) {
+          // we finished all folds as well! Record these candidates
+          // and sample new ones to evaluate.
+          for(var k=0;k<this.candidates.length;k++) {
+            this.evaluated_candidates.push(this.candidates[k]);
+          }
+          // sort evaluated candidates according to accuracy achieved
+          this.evaluated_candidates.sort(function(a, b) { 
+            return (a.accv / a.acc.length) 
+                 > (b.accv / b.acc.length) 
+                 ? -1 : 1;
+          });
+          // and clip only to the top few ones (lets place limit at 3*ensemble_size)
+          // otherwise there are concerns with keeping these all in memory 
+          // if MagicNet is being evaluated for a very long time
+          if(this.evaluated_candidates.length > 3 * this.ensemble_size) {
+            this.evaluated_candidates = this.evaluated_candidates.slice(0, 3 * this.ensemble_size);
+          }
+          if(this.finish_batch_callback !== null) {
+            this.finish_batch_callback();
+          }
+          this.sampleCandidates(); // begin with new candidates
+          this.foldix = 0; // reset this
+        } else {
+          // we will go on to another fold. reset all candidates nets
+          for(var k=0;k<this.candidates.length;k++) {
+            var c = this.candidates[k];
+            var net = new Net();
+            net.makeLayers(c.layer_defs);
+            var trainer = new Trainer(net, c.trainer_def);
+            c.net = net;
+            c.trainer = trainer;
+          }
+        }
+      }
+    },
+
+    evalValErrors: function() {
+      // evaluate candidates on validation data and return performance of current networks
+      // as simple list
+      var vals = [];
+      var fold = this.folds[this.foldix]; // active fold
+      for(var k=0;k<this.candidates.length;k++) {
+        var net = this.candidates[k].net;
+        var v = 0.0;
+        for(var q=0;q<fold.test_ix.length;q++) {
+          var x = this.data[fold.test_ix[q]];
+          var l = this.labels[fold.test_ix[q]];
+          net.forward(x);
+          var yhat = net.getPrediction();
+          v += (yhat === l ? 1.0 : 0.0); // 0 1 loss
+        }
+        v /= fold.test_ix.length; // normalize
+        vals.push(v);
+      }
+      return vals;
+    },
+
+    // returns prediction scores for given test data point, as Vol
+    // uses an averaged prediction from the best ensemble_size models
+    // x is a Vol.
+    predict_soft: function(data) {
+      // forward prop the best networks
+      // and accumulate probabilities at last layer into a an output Vol
+      var nv = Math.min(this.ensemble_size, this.evaluated_candidates.length);
+      if(nv === 0) { return new convnetjs.Vol(0,0,0); } // not sure what to do here? we're not ready yet
+      var xout, n;
+      for(var j=0;j<nv;j++) {
+        var net = this.evaluated_candidates[j].net;
+        var x = net.forward(data);
+        if(j===0) { 
+          xout = x; 
+          n = x.w.length; 
+        } else {
+          // add it on
+          for(var d=0;d<n;d++) {
+            xout.w[d] += x.w[d];
+          }
+        }
+      }
+      // produce average
+      for(var d=0;d<n;d++) {
+        xout.w[d] /= n;
+      }
+      return xout;
+    },
+
+    predict: function(data) {
+      var xout = this.predict_soft(data);
+      if(xout.w.length !== 0) {
+        var stats = maxmin(xout.w);
+        var predicted_label = stats.maxi; 
+      } else {
+        var predicted_label = -1; // error out
+      }
+      return predicted_label;
+
+    },
+
+    toJSON: function() {
+      // dump the top ensemble_size networks as a list
+      var nv = Math.min(this.ensemble_size, this.evaluated_candidates.length);
+      var json = {};
+      json.nets = [];
+      for(var i=0;i<nv;i++) {
+        json.nets.push(this.evaluated_candidates[i].net.toJSON());
+      }
+      return json;
+    },
+
+    fromJSON: function(json) {
+      this.ensemble_size = json.nets.length;
+      this.evaluated_candidates = [];
+      for(var i=0;i<this.ensemble_size;i++) {
+        var net = new Net();
+        net.fromJSON(json.nets[i]);
+        var dummy_candidate = {};
+        dummy_candidate.net = net;
+        this.evaluated_candidates.push(dummy_candidate);
+      }
+    },
+
+    // callback functions
+    // called when a fold is finished, while evaluating a batch
+    onFinishFold: function(f) { this.finish_fold_callback = f; },
+    // called when a batch of candidates has finished evaluating
+    onFinishBatch: function(f) { this.finish_batch_callback = f; }
+    
+  };
+
+  global.MagicNet = MagicNet;
+})(convnetjs);
+this.convnetjs = convnetjs;
diff --git a/lib/net.js b/lib/net.js
new file mode 100644
index 00000000..65c2a2c0
--- /dev/null
+++ b/lib/net.js
@@ -0,0 +1 @@
+NEURAL_NET_TRAINED_FOR_STROKE_EXTRACTION = {"layers":[{"out_depth":8,"out_sx":1,"out_sy":1,"layer_type":"input"},{"out_depth":8,"out_sx":1,"out_sy":1,"layer_type":"fc","num_inputs":8,"l1_decay_mul":0,"l2_decay_mul":1,"filters":[{"sx":1,"sy":1,"depth":8,"w":{"0":-0.3044261605666167,"1":-0.14834922423083324,"2":0.20219401661574177,"3":0.5503522616459873,"4":0.45502127328350234,"5":0.2625745186594936,"6":0.012889731022695689,"7":-0.2675923800252626,"BYTES_PER_ELEMENT":8,"buffer":{"0":183,"1":140,"2":235,"3":220,"4":183,"5":123,"6":211,"7":191,"8":214,"9":172,"10":58,"11":125,"12":27,"13":253,"14":194,"15":191,"16":244,"17":222,"18":103,"19":88,"20":126,"21":225,"22":201,"23":63,"24":164,"25":146,"26":161,"27":88,"28":124,"29":156,"30":225,"31":63,"32":183,"33":41,"34":239,"35":139,"36":17,"37":31,"38":221,"39":63,"40":43,"41":243,"42":153,"43":90,"44":5,"45":206,"46":208,"47":63,"48":29,"49":223,"50":146,"51":105,"52":238,"53":101,"54":138,"55":63,"56":243,"57":129,"58":55,"59":202,"60":59,"61":32,"62":209,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":-0.5596639882456166,"1":-0.2794084096442046,"2":-0.1125698422524117,"3":-0.08850676702777903,"4":0.06317601682543969,"5":-0.18247248453514878,"6":0.2940108272454184,"7":0.9861821092760742,"BYTES_PER_ELEMENT":8,"buffer":{"0":121,"1":113,"2":200,"3":115,"4":196,"5":232,"6":225,"7":191,"8":251,"9":140,"10":105,"11":207,"12":211,"13":225,"14":209,"15":191,"16":206,"17":111,"18":253,"19":142,"20":96,"21":209,"22":188,"23":191,"24":143,"25":231,"26":219,"27":37,"28":97,"29":168,"30":182,"31":191,"32":166,"33":36,"34":40,"35":174,"36":77,"37":44,"38":176,"39":63,"40":69,"41":201,"42":191,"43":36,"44":66,"45":91,"46":199,"47":191,"48":54,"49":24,"50":236,"51":201,"52":18,"53":209,"54":210,"55":63,"56":147,"57":182,"58":103,"59":200,"60":205,"61":142,"62":239,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":-0.4293240703659736,"1":0.48799666353430715,"2":-0.011221411170891243,"3":0.016759551491042825,"4":-0.10178241490300455,"5":-0.05938636975897821,"6":-0.7140555216543757,"7":-0.13033896328056724,"BYTES_PER_ELEMENT":8,"buffer":{"0":157,"1":224,"2":102,"3":170,"4":11,"5":122,"6":219,"7":191,"8":60,"9":247,"10":155,"11":91,"12":86,"13":59,"14":223,"15":63,"16":118,"17":154,"18":243,"19":79,"20":64,"21":251,"22":134,"23":191,"24":71,"25":217,"26":50,"27":118,"28":106,"29":41,"30":145,"31":63,"32":136,"33":249,"34":80,"35":143,"36":105,"37":14,"38":186,"39":191,"40":152,"41":205,"42":226,"43":231,"44":227,"45":103,"46":174,"47":191,"48":151,"49":20,"50":33,"51":247,"52":138,"53":217,"54":230,"55":191,"56":91,"57":160,"58":87,"59":120,"60":242,"61":174,"62":192,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":0.004132243204399921,"1":0.31644328046013015,"2":-0.4246219644532225,"3":0.021888719524958525,"4":-0.5736499683744954,"5":-0.1285778687631867,"6":-0.17633637428199953,"7":-0.09184800562849475,"BYTES_PER_ELEMENT":8,"buffer":{"0":192,"1":147,"2":192,"3":150,"4":248,"5":236,"6":112,"7":63,"8":174,"9":95,"10":39,"11":81,"12":155,"13":64,"14":212,"15":63,"16":243,"17":89,"18":159,"19":154,"20":1,"21":45,"22":219,"23":191,"24":33,"25":71,"26":11,"27":26,"28":255,"29":105,"30":150,"31":63,"32":245,"33":162,"34":176,"35":45,"36":87,"37":91,"38":226,"39":191,"40":225,"41":227,"42":169,"43":86,"44":61,"45":117,"46":192,"47":191,"48":171,"49":117,"50":81,"51":184,"52":48,"53":146,"54":198,"55":191,"56":195,"57":144,"58":96,"59":212,"60":89,"61":131,"62":183,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":0.09461892805990832,"1":-0.6329745552844204,"2":-0.7923676394624737,"3":-0.08606445900172546,"4":0.11603124920286288,"5":0.029252054814320392,"6":-0.03747907757038657,"7":-0.6005289047981558,"BYTES_PER_ELEMENT":8,"buffer":{"0":238,"1":145,"2":153,"3":49,"4":242,"5":56,"6":184,"7":63,"8":2,"9":178,"10":196,"11":218,"12":83,"13":65,"14":228,"15":191,"16":40,"17":205,"18":60,"19":97,"20":19,"21":91,"22":233,"23":191,"24":226,"25":165,"26":194,"27":4,"28":82,"29":8,"30":182,"31":191,"32":39,"33":236,"34":163,"35":84,"36":57,"37":180,"38":189,"39":63,"40":169,"41":190,"42":18,"43":43,"44":64,"45":244,"46":157,"47":63,"48":220,"49":234,"50":229,"51":40,"52":117,"53":48,"54":163,"55":191,"56":21,"57":37,"58":205,"59":100,"60":136,"61":55,"62":227,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":0.42277438855180405,"1":1.4138688143060145,"2":-0.3778547415032166,"3":-0.012023049493322651,"4":0.11825224526479829,"5":-0.4434819918251306,"6":-0.6772974432572635,"7":-0.14741163123927092,"BYTES_PER_ELEMENT":8,"buffer":{"0":56,"1":166,"2":26,"3":79,"4":188,"5":14,"6":219,"7":63,"8":68,"9":117,"10":228,"11":231,"12":52,"13":159,"14":246,"15":63,"16":53,"17":69,"18":89,"19":167,"20":197,"21":46,"22":216,"23":191,"24":118,"25":224,"26":249,"27":98,"28":138,"29":159,"30":136,"31":191,"32":225,"33":67,"34":23,"35":118,"36":199,"37":69,"38":190,"39":63,"40":126,"41":63,"42":208,"43":74,"44":2,"45":98,"46":220,"47":191,"48":35,"49":138,"50":14,"51":176,"52":107,"53":172,"54":229,"55":191,"56":204,"57":128,"58":156,"59":99,"60":98,"61":222,"62":194,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":1.1993670420146874,"1":0.3223431096192715,"2":-0.36861400047798204,"3":-0.08038294233717612,"4":0.2711210775205418,"5":0.42879169827918595,"6":0.6376085656983045,"7":0.03756028253259824,"BYTES_PER_ELEMENT":8,"buffer":{"0":72,"1":167,"2":213,"3":126,"4":155,"5":48,"6":243,"7":63,"8":56,"9":247,"10":121,"11":254,"12":68,"13":161,"14":212,"15":63,"16":111,"17":164,"18":57,"19":45,"20":95,"21":151,"22":215,"23":191,"24":167,"25":146,"26":126,"27":252,"28":249,"29":147,"30":180,"31":191,"32":158,"33":63,"34":77,"35":56,"36":12,"37":90,"38":209,"39":63,"40":32,"41":242,"42":57,"43":188,"44":82,"45":113,"46":219,"47":63,"48":161,"49":91,"50":42,"51":20,"52":74,"53":103,"54":228,"55":63,"56":127,"57":73,"58":54,"59":242,"60":25,"61":59,"62":163,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":0.13967402583430144,"1":-0.11357850366802424,"2":-0.5746610827627967,"3":-0.5717582875884522,"4":-0.1898576928922138,"5":-0.18657398702306335,"6":0.7165884005339106,"7":-0.6224249593531741,"BYTES_PER_ELEMENT":8,"buffer":{"0":200,"1":140,"2":135,"3":166,"4":214,"5":224,"6":193,"7":63,"8":72,"9":100,"10":200,"11":22,"12":123,"13":19,"14":189,"15":191,"16":82,"17":1,"18":152,"19":163,"20":159,"21":99,"22":226,"23":191,"24":132,"25":25,"26":77,"27":9,"28":216,"29":75,"30":226,"31":191,"32":97,"33":219,"34":238,"35":194,"36":65,"37":77,"38":200,"39":191,"40":127,"41":49,"42":70,"43":10,"44":168,"45":225,"46":199,"47":191,"48":23,"49":142,"50":31,"51":204,"52":74,"53":238,"54":230,"55":63,"56":54,"57":90,"58":148,"59":191,"60":231,"61":234,"62":227,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}}],"biases":{"sx":1,"sy":1,"depth":8,"w":{"0":0.2562049254190115,"1":-0.5455393081802729,"2":0.10903726980643962,"3":-0.16355954769541572,"4":0.08992117884673975,"5":0.5185622512844232,"6":-0.46073562437071663,"7":0.38509647559811017,"BYTES_PER_ELEMENT":8,"buffer":{"0":232,"1":235,"2":239,"3":87,"4":169,"5":101,"6":208,"7":63,"8":182,"9":34,"10":234,"11":217,"12":14,"13":117,"14":225,"15":191,"16":27,"17":33,"18":221,"19":211,"20":221,"21":233,"22":187,"23":63,"24":73,"25":114,"26":38,"27":238,"28":132,"29":239,"30":196,"31":191,"32":206,"33":80,"34":93,"35":10,"36":19,"37":5,"38":183,"39":63,"40":140,"41":157,"42":198,"43":220,"44":15,"45":152,"46":224,"47":63,"48":65,"49":143,"50":177,"51":69,"52":177,"53":124,"54":221,"55":191,"56":113,"57":235,"58":31,"59":176,"60":107,"61":165,"62":216,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}}},{"out_depth":8,"out_sx":1,"out_sy":1,"layer_type":"tanh"},{"out_depth":8,"out_sx":1,"out_sy":1,"layer_type":"fc","num_inputs":8,"l1_decay_mul":0,"l2_decay_mul":1,"filters":[{"sx":1,"sy":1,"depth":8,"w":{"0":-0.38832824611419614,"1":0.5331207710690121,"2":-0.16958013252471874,"3":-0.03763800230330026,"4":-0.30277152771651167,"5":-0.03899235791753754,"6":0.4840579241426027,"7":-0.5416342032769544,"BYTES_PER_ELEMENT":8,"buffer":{"0":208,"1":26,"2":75,"3":183,"4":94,"5":218,"6":216,"7":191,"8":37,"9":233,"10":145,"11":74,"12":83,"13":15,"14":225,"15":63,"16":149,"17":92,"18":159,"19":65,"20":205,"21":180,"22":197,"23":191,"24":29,"25":122,"26":245,"27":201,"28":73,"29":69,"30":163,"31":191,"32":183,"33":243,"34":108,"35":212,"36":155,"37":96,"38":211,"39":191,"40":226,"41":120,"42":25,"43":108,"44":206,"45":246,"46":163,"47":191,"48":229,"49":249,"50":99,"51":22,"52":206,"53":250,"54":222,"55":63,"56":111,"57":6,"58":175,"59":64,"60":17,"61":85,"62":225,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":-0.11439407632270632,"1":-1.2819890886327963,"2":0.14634106395136273,"3":0.14304395032875164,"4":0.2297327647600765,"5":0.4348725634157742,"6":-0.26416425812412686,"7":0.058453811796899485,"BYTES_PER_ELEMENT":8,"buffer":{"0":196,"1":130,"2":169,"3":32,"4":238,"5":72,"6":189,"7":191,"8":119,"9":27,"10":152,"11":253,"12":6,"13":131,"14":244,"15":191,"16":57,"17":209,"18":221,"19":209,"20":77,"21":187,"22":194,"23":63,"24":205,"25":188,"26":70,"27":160,"28":67,"29":79,"30":194,"31":63,"32":147,"33":106,"34":187,"35":27,"36":226,"37":103,"38":205,"39":63,"40":148,"41":25,"42":115,"43":187,"44":243,"45":212,"46":219,"47":63,"48":21,"49":147,"50":90,"51":52,"52":17,"53":232,"54":208,"55":191,"56":180,"57":10,"58":253,"59":115,"60":168,"61":237,"62":173,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":-0.1406881916489247,"1":0.9405578660965507,"2":0.004901858025518234,"3":-0.2326645372062353,"4":-0.21448421502002576,"5":-1.0893898213683164,"6":-0.3778283066766321,"7":-0.37747385299064595,"BYTES_PER_ELEMENT":8,"buffer":{"0":178,"1":98,"2":8,"3":23,"4":18,"5":2,"6":194,"7":191,"8":221,"9":42,"10":92,"11":207,"12":12,"13":25,"14":238,"15":63,"16":254,"17":58,"18":140,"19":126,"20":248,"21":19,"22":116,"23":63,"24":82,"25":176,"26":30,"27":153,"28":243,"29":199,"30":205,"31":191,"32":139,"33":118,"34":130,"35":0,"36":56,"37":116,"38":203,"39":191,"40":137,"41":244,"42":117,"43":5,"44":36,"45":110,"46":241,"47":191,"48":231,"49":119,"50":43,"51":199,"52":86,"53":46,"54":216,"55":191,"56":234,"57":39,"58":108,"59":23,"60":136,"61":40,"62":216,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":0.3710384010689571,"1":-0.6190881680177,"2":0.41162941225861827,"3":-0.543618129725223,"4":0.6163330090258718,"5":0.7949110806898168,"6":-0.7884090007104152,"7":0.38478012561877223,"BYTES_PER_ELEMENT":8,"buffer":{"0":239,"1":174,"2":137,"3":217,"4":23,"5":191,"6":215,"7":63,"8":25,"9":64,"10":95,"11":253,"12":145,"13":207,"14":227,"15":191,"16":230,"17":60,"18":238,"19":227,"20":34,"21":88,"22":218,"23":63,"24":49,"25":215,"26":21,"27":217,"28":81,"29":101,"30":225,"31":191,"32":186,"33":195,"34":166,"35":0,"36":0,"37":185,"38":227,"39":63,"40":18,"41":78,"42":217,"43":92,"44":233,"45":111,"46":233,"47":63,"48":118,"49":139,"50":61,"51":131,"52":165,"53":58,"54":233,"55":191,"56":205,"57":188,"58":235,"59":209,"60":60,"61":160,"62":216,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":0.2740006754893999,"1":-0.8544168596508195,"2":0.07387149782516346,"3":-0.18854684467760982,"4":0.5306163852118577,"5":0.5217519355682549,"6":-0.16672917486486497,"7":0.23335567893271977,"BYTES_PER_ELEMENT":8,"buffer":{"0":182,"1":196,"2":19,"3":33,"4":58,"5":137,"6":209,"7":63,"8":200,"9":61,"10":171,"11":6,"12":98,"13":87,"14":235,"15":191,"16":41,"17":255,"18":67,"19":19,"20":62,"21":233,"22":178,"23":63,"24":243,"25":192,"26":211,"27":145,"28":77,"29":34,"30":200,"31":191,"32":4,"33":157,"34":166,"35":54,"36":207,"37":250,"38":224,"39":63,"40":200,"41":125,"42":124,"43":29,"44":49,"45":178,"46":224,"47":63,"48":97,"49":181,"50":170,"51":176,"52":97,"53":87,"54":197,"55":191,"56":79,"57":11,"58":173,"59":80,"60":153,"61":222,"62":205,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":-0.1906626781357325,"1":0.4709381707241033,"2":0.24160919622692853,"3":0.6311757769867091,"4":-0.3203913931138507,"5":-0.5879387228046364,"6":0.39346084678734317,"7":-0.3561015790308374,"BYTES_PER_ELEMENT":8,"buffer":{"0":78,"1":147,"2":148,"3":119,"4":162,"5":103,"6":200,"7":191,"8":123,"9":173,"10":108,"11":218,"12":217,"13":35,"14":222,"15":63,"16":131,"17":143,"18":26,"19":214,"20":12,"21":237,"22":206,"23":63,"24":5,"25":238,"26":5,"27":139,"28":151,"29":50,"30":228,"31":63,"32":240,"33":1,"34":214,"35":230,"36":74,"37":129,"38":212,"39":191,"40":252,"41":238,"42":79,"43":222,"44":100,"45":208,"46":226,"47":191,"48":154,"49":81,"50":77,"51":103,"52":118,"53":46,"54":217,"55":63,"56":50,"57":115,"58":255,"59":70,"60":94,"61":202,"62":214,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":-0.052332119821916476,"1":0.5000167096781711,"2":0.26266568911062693,"3":0.26506535297601835,"4":-0.25463287277049923,"5":0.08282611179305391,"6":0.9396179911813585,"7":-0.5542899781400487,"BYTES_PER_ELEMENT":8,"buffer":{"0":102,"1":186,"2":84,"3":142,"4":70,"5":203,"6":170,"7":191,"8":66,"9":174,"10":240,"11":10,"12":35,"13":0,"14":224,"15":63,"16":136,"17":187,"18":32,"19":192,"20":131,"21":207,"22":208,"23":63,"24":165,"25":107,"26":149,"27":171,"28":212,"29":246,"30":208,"31":63,"32":237,"33":74,"34":66,"35":173,"36":231,"37":75,"38":208,"39":191,"40":10,"41":240,"42":103,"43":145,"44":23,"45":52,"46":181,"47":63,"48":200,"49":109,"50":219,"51":191,"52":89,"53":17,"54":238,"55":63,"56":7,"57":150,"58":19,"59":86,"60":190,"61":188,"62":225,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":0.5183573300646674,"1":-0.8084279309930922,"2":0.17564116878772115,"3":-0.4120839198806116,"4":0.5760434856452349,"5":0.35578634913953205,"6":-0.3705548599822078,"7":0.44300177295886806,"BYTES_PER_ELEMENT":8,"buffer":{"0":195,"1":160,"2":136,"3":28,"4":98,"5":150,"6":224,"7":63,"8":142,"9":57,"10":153,"11":64,"12":164,"13":222,"14":233,"15":191,"16":27,"17":34,"18":227,"19":233,"20":104,"21":123,"22":198,"23":63,"24":191,"25":15,"26":198,"27":59,"28":149,"29":95,"30":218,"31":191,"32":179,"33":113,"34":125,"35":191,"36":242,"37":110,"38":226,"39":63,"40":199,"41":184,"42":122,"43":27,"44":52,"45":197,"46":214,"47":63,"48":21,"49":214,"50":63,"51":187,"52":43,"53":183,"54":215,"55":191,"56":45,"57":106,"58":187,"59":27,"60":36,"61":90,"62":220,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}}],"biases":{"sx":1,"sy":1,"depth":8,"w":{"0":0.013187463476726313,"1":-0.007630520124428615,"2":-0.27144190488489406,"3":-0.30429595726968894,"4":-0.05004981783245224,"5":0.07003641753819947,"6":-0.09093686693747323,"7":0.05757614475366814,"BYTES_PER_ELEMENT":8,"buffer":{"0":83,"1":65,"2":204,"3":98,"4":7,"5":2,"6":139,"7":63,"8":200,"9":155,"10":44,"11":38,"12":46,"13":65,"14":127,"15":191,"16":234,"17":166,"18":15,"19":222,"20":77,"21":95,"22":209,"23":191,"24":30,"25":208,"26":49,"27":192,"28":149,"29":121,"30":211,"31":191,"32":36,"33":180,"34":133,"35":53,"36":33,"37":160,"38":169,"39":191,"40":122,"41":6,"42":219,"43":26,"44":232,"45":237,"46":177,"47":63,"48":77,"49":69,"50":127,"51":117,"52":163,"53":71,"54":183,"55":191,"56":50,"57":88,"58":126,"59":213,"60":158,"61":122,"62":173,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}}},{"out_depth":8,"out_sx":1,"out_sy":1,"layer_type":"tanh"},{"out_depth":2,"out_sx":1,"out_sy":1,"layer_type":"fc","num_inputs":8,"l1_decay_mul":0,"l2_decay_mul":1,"filters":[{"sx":1,"sy":1,"depth":8,"w":{"0":1.0172730016739508,"1":-0.13221137581570833,"2":0.9395466221657038,"3":-1.6251860610880569,"4":-1.2388309137808013,"5":0.8533890654636394,"6":0.7649152131278658,"7":-1.7907313802649556,"BYTES_PER_ELEMENT":8,"buffer":{"0":167,"1":177,"2":20,"3":14,"4":192,"5":70,"6":240,"7":63,"8":37,"9":209,"10":164,"11":103,"12":77,"13":236,"14":192,"15":191,"16":96,"17":155,"18":232,"19":19,"20":196,"21":16,"22":238,"23":63,"24":185,"25":156,"26":100,"27":25,"28":195,"29":0,"30":250,"31":191,"32":188,"33":101,"34":63,"35":93,"36":64,"37":210,"38":243,"39":191,"40":76,"41":197,"42":221,"43":149,"44":246,"45":78,"46":235,"47":63,"48":16,"49":27,"50":19,"51":120,"52":47,"53":122,"54":232,"55":63,"56":244,"57":138,"58":162,"59":242,"60":213,"61":166,"62":252,"63":191,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}},{"sx":1,"sy":1,"depth":8,"w":{"0":-1.252270530582403,"1":0.7285270999976606,"2":0.023798576828390303,"3":1.064484519454345,"4":0.246286754788085,"5":-1.2474340354253382,"6":-0.3051491951725839,"7":1.4177785201450572,"BYTES_PER_ELEMENT":8,"buffer":{"0":43,"1":137,"2":233,"3":210,"4":76,"5":9,"6":244,"7":191,"8":105,"9":171,"10":151,"11":16,"12":24,"13":80,"14":231,"15":63,"16":172,"17":87,"18":173,"19":116,"20":167,"21":94,"22":152,"23":63,"24":154,"25":121,"26":98,"27":235,"28":32,"29":8,"30":241,"31":63,"32":163,"33":91,"34":160,"35":10,"36":83,"37":134,"38":207,"39":63,"40":43,"41":27,"42":33,"43":100,"44":125,"45":245,"46":243,"47":191,"48":158,"49":175,"50":106,"51":125,"52":144,"53":135,"54":211,"55":191,"56":165,"57":232,"58":143,"59":135,"60":56,"61":175,"62":246,"63":63,"byteLength":64},"length":8,"byteOffset":0,"byteLength":64}}],"biases":{"sx":1,"sy":1,"depth":2,"w":{"0":-0.008745691297802017,"1":0.00874569129780175,"BYTES_PER_ELEMENT":8,"buffer":{"0":71,"1":251,"2":214,"3":208,"4":66,"5":233,"6":129,"7":191,"8":173,"9":250,"10":214,"11":208,"12":66,"13":233,"14":129,"15":63,"byteLength":16},"length":2,"byteOffset":0,"byteLength":16}}},{"out_depth":2,"out_sx":1,"out_sy":1,"layer_type":"softmax","num_inputs":2}]};
diff --git a/lib/stroke_extractor.js b/lib/stroke_extractor.js
index 5dee927c..8076b029 100644
--- a/lib/stroke_extractor.js
+++ b/lib/stroke_extractor.js
@@ -354,7 +354,7 @@ if (this.stroke_extractor !== undefined) {
 }
 this.stroke_extractor = {};
 
-this.stroke_extractor.getBridges = (glyph, classifier) => {
+stroke_extractor.getBridges = (glyph, classifier) => {
   assert(glyph.stages.path)
   const paths = svg.convertSVGPathToPaths(glyph.stages.path);
   const endpoints = [];
@@ -363,11 +363,12 @@ this.stroke_extractor.getBridges = (glyph, classifier) => {
       endpoints.push(new Endpoint(paths, [i, j]));
     }
   }
-  const bridges = getBridges(endpoints, classifier || handTunedClassifier);
+  classifier = classifier || stroke_extractor.combinedClassifier;
+  const bridges = getBridges(endpoints, classifier);
   return {endpoints: endpoints, bridges: bridges};
 }
 
-this.stroke_extractor.getStrokes = (glyph) => {
+stroke_extractor.getStrokes = (glyph) => {
   assert(glyph.stages.path)
   assert(glyph.stages.bridges)
   const paths = svg.convertSVGPathToPaths(glyph.stages.path);
@@ -383,3 +384,5 @@ this.stroke_extractor.getStrokes = (glyph) => {
   const strokes = stroke_paths.map((x) => svg.convertPathsToSVGPath([x]));
   return {log: log, strokes: strokes};
 }
+
+stroke_extractor.handTunedClassifier = handTunedClassifier;
diff --git a/server/training.js b/server/training.js
new file mode 100644
index 00000000..92cdc75f
--- /dev/null
+++ b/server/training.js
@@ -0,0 +1,78 @@
+"use strict";
+
+function evaluate(glyphs, classifier) {
+  var num_correct = 0;
+  for (var i = 0; i < glyphs.length; i++) {
+    if (check_classifier_on_glyph(glyphs[i], classifier)) {
+      num_correct += 1;
+    }
+  }
+  return num_correct/glyphs.length;
+}
+
+function train_neural_net() {
+  var glyphs = Glyphs.find({'manual.verified': true}).fetch();
+  var sample = _.sample(glyphs, 400);
+  console.log('Hand-tuned accuracy:', evaluate(sample, hand_tuned_classifier));
+
+  var training_data = [];
+  for (var i = 0; i < glyphs.length; i++) {
+    var glyph_data = get_glyph_training_data(glyphs[i]);
+    var positive_data = glyph_data.filter(function(x) { return x[1] > 0; });
+    var negative_data = glyph_data.filter(function(x) { return x[1] === 0; });
+    if (positive_data.length > negative_data.length) {
+      positive_data = _.sample(positive_data, negative_data.length);
+    } else {
+      negative_data = _.sample(negative_data, positive_data.length);
+    }
+    glyph_data = negative_data.concat(positive_data);
+    for (var j = 0; j < glyph_data.length; j++) {
+      training_data.push(glyph_data[j]);
+    }
+  }
+  console.log('Got ' + training_data.length + ' rows of training data.');
+
+  var net = new convnetjs.Net();
+  net.makeLayers([
+    {type: 'input', out_sx: 1, out_sy: 1, out_depth: 8},
+    {type: 'fc', num_neurons: 8, activation: 'tanh'},
+    {type: 'fc', num_neurons: 8, activation: 'tanh'},
+    {type: 'softmax', num_classes: 2},
+  ]);
+  var trainer = new convnetjs.Trainer(
+      net, {method: 'adadelta', l2_decay: 0.001, batch_size: 10});
+  var input = new convnetjs.Vol(1, 1, 8);
+  for (var iteration = 0; iteration < 10; iteration++) {
+    var loss = 0;
+    var round_data = _.sample(training_data, 4000);
+    for (var i = 0; i < round_data.length; i++) {
+      assert(input.w.length === round_data[i][0].length);
+      input.w = round_data[i][0];
+      var stats = trainer.train(input, round_data[i][1]);
+      assert(!isNaN(stats.loss))
+      loss += stats.loss;
+    }
+    console.log('Iteration', iteration, 'mean loss:', loss/round_data.length);
+  }
+  console.log('Trained neural network:', JSON.stringify(net.toJSON()));
+
+  function net_classifier(features) {
+    assert(input.w.length === features.length);
+    input.w = features;
+    var softmax = net.forward(input).w;
+    assert(softmax.length === 2);
+    return softmax[1] - softmax[0];
+  }
+  console.log('Neural-net accuracy:', evaluate(sample, net_classifier));
+
+  function combined_classifier(weight) {
+    return function(features) {
+      return hand_tuned_classifier(features) + weight*net_classifier(features);
+    }
+  }
+  var weights = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1];
+  for (var i = 0; i < weights.length; i++) {
+    console.log('Weight',  weights[i], 'combined accuracy:',
+                evaluate(sample, combined_classifier(weights[i])));
+  }
+}