├── LICENSE ├── README.md ├── criterion.lua ├── layer.lua ├── learn.lua ├── model.lua ├── tensor.lua └── transfer.lua /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Matthew Nichols 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Learn's api is something like a combination between [Torch](http://torch.ch/) and [Scikit Learn](http://scikit-learn.org/stable/). The purpose of Learn is to provide a flexible and portable neural network implementation that only depends on Lua. Learn is *not* multithreaded and does *not* use hardware acceleration, if you are looking for a high performance library I would suggest looking at [Torch](http://torch.ch/) instead. 2 | 3 | ### Start using Learn 4 | Learn can be installed as a submodule for your git project by using the command: 5 | ``` 6 | git submodule add https://github.com/Polkm/learn.git learn 7 | ``` 8 | 9 | After you have Learn installed you can use it in an existing Lua project. 10 | ```lua 11 | require("learn/learn") 12 | ``` 13 | 14 | ### Set up your training data 15 | ```lua 16 | -- XOR training data 17 | local train_features = {{0, 0}, {0, 1}, {1, 0}, {1, 1}} 18 | local train_labels = {{0}, {1}, {1}, {0}} 19 | ``` 20 | 21 | ### Set up your model 22 | ```lua 23 | local n_input = #train_features[1] 24 | local n_output = #train_labels[1] 25 | 26 | local model = learn.model.nnet({modules = { 27 | learn.layer.linear({n_input = n_input, n_output = n_input * 3}), 28 | learn.transfer.sigmoid({}), 29 | learn.layer.linear({n_input = n_input * 3, n_output = n_output}), 30 | learn.transfer.sigmoid({}), 31 | }}) 32 | ``` 33 | By default nnet will use the MSE criterion, appropriate for regression tasks. 34 | 35 | ### Train your model on your data 36 | ```lua 37 | local epochs = 1000 38 | local error = model.fit(train_features, train_labels, epochs) 39 | ``` 40 | 41 | ### Make predictions using your newly trained model 42 | ```lua 43 | for _, prediction in pairs(model.predict(train_features)) do 44 | print(table.concat(prediction, ", ")) 45 | end 46 | ``` 47 | -------------------------------------------------------------------------------- /criterion.lua: -------------------------------------------------------------------------------- 1 | function learn.criterion.mse(p) 2 | function p.forward(predictions, target) 3 | p.output_tensor = predictions.copy().sub(target).pow(2.0).sum().scale(1.0 / predictions.size[1]) 4 | p.output = p.output_tensor.data[1] 5 | return p.output 6 | end 7 | 8 | function p.backward(predictions, target) 9 | p.gradInput = predictions.copy().sub(target).scale(2.0 / predictions.size[1]) 10 | return p.gradInput 11 | end 12 | 13 | return p 14 | end 15 | -------------------------------------------------------------------------------- /layer.lua: -------------------------------------------------------------------------------- 1 | -- Module for linear transformations using a weight tensor 2 | function learn.layer.linear(p) 3 | p.n_input = p.n_input or 1 4 | p.n_output = p.n_output or 1 5 | 6 | p.weight_init = p.weight_init or function() 7 | return learn.gaussian(0.0, 1.0) 8 | end 9 | 10 | p.weights = p.weights or learn.tensor({size = {p.n_output, p.n_input}}).map(p.weight_init) 11 | p.gradients = p.gradients or learn.tensor({size = {p.n_output, p.n_input}}) 12 | 13 | function p.forward(input) 14 | -- print(table.concat(p.weights.data, ", ")) 15 | p.output = p.weights.dot(input) 16 | -- print(table.concat(p.output.data, ", ")) 17 | return p.output 18 | end 19 | 20 | function p.backward(input, gradients) 21 | p.delta = gradients.copy().mul(input) 22 | return input, p.weights.transpose().dot(p.delta) 23 | end 24 | 25 | function p.update(input, learning_rate) 26 | p.weights.sub(p.delta.dot(input.transpose()).scale(learning_rate)) 27 | return p.output 28 | end 29 | 30 | return p 31 | end 32 | -------------------------------------------------------------------------------- /learn.lua: -------------------------------------------------------------------------------- 1 | learn = {} 2 | learn.model = {} 3 | learn.layer = {} 4 | learn.transfer = {} 5 | learn.criterion = {} 6 | 7 | -- Returns a random sample of a gaussian distribution 8 | function learn.gaussian(mean, sd) 9 | return math.sqrt(-2 * math.log(math.random())) * math.cos(2 * math.pi * math.random()) * sd + mean 10 | end 11 | 12 | require("learn/tensor") 13 | require("learn/model") 14 | require("learn/layer") 15 | require("learn/transfer") 16 | require("learn/criterion") 17 | 18 | function learn.normalize(samples) 19 | local max = 0 20 | for i, vector in ipairs(samples) do 21 | for j, v in ipairs(vector) do 22 | max = math.max(max, math.abs(v)) 23 | end 24 | end 25 | if max > 0 then 26 | for i, vector in ipairs(samples) do 27 | for j, v in ipairs(vector) do 28 | vector[j] = vector[j] / max 29 | end 30 | end 31 | end 32 | return max 33 | end 34 | 35 | function learn.unormalize(samples, max) 36 | for i, vector in ipairs(samples) do 37 | for j, v in ipairs(vector) do 38 | vector[j] = vector[j] * max 39 | end 40 | end 41 | end 42 | 43 | -- Runs all unit tests 44 | function learn.test() 45 | local identity = learn.tensor({data = {1, 0, 0, 1}, size = {2, 2}}) 46 | local test = learn.tensor({data = {1, 2, 3, 4, 5, 6}, size = {2, 3}}) 47 | -- identity.dot(test).print() 48 | 49 | -- print(test.string()) 50 | -- print(test.transpose().string()) 51 | 52 | -- XOR training data 53 | -- local train_features = {{0, 0}, {0, 1}, {1, 0}, {1, 1}} 54 | -- local train_labels = {{0}, {1}, {1}, {0}} 55 | 56 | local train_features = {{0, 0}, {0, 1}, {1, 0}, {-1, -1}} 57 | local train_labels = {{0, 0}, {0, 1}, {1, 0}, {3, 3}} 58 | -- local train_labels = {{0}, {0.1}, {0.3}, {-3.0}} 59 | 60 | local n_input = #train_features[1] 61 | local n_output = #train_labels[1] 62 | 63 | local model = learn.model.nnet({modules = { 64 | learn.layer.linear({n_input = n_input, n_output = n_input * 3}), 65 | learn.transfer.tanh({}), 66 | learn.layer.linear({n_input = n_input * 3, n_output = n_output}), 67 | learn.transfer.tanh({}), 68 | -- learn.layer.linear({n_input = n_output, n_output = n_output}), 69 | -- learn.transfer.sigmoid({}), 70 | }}) 71 | -- 72 | -- local epochs = 1000 73 | -- local learning_rate = 0.5 74 | -- local error = model.fit(train_features, train_labels, epochs, learning_rate, true) 75 | -- 76 | -- local predictions = model.predict(train_features) 77 | -- 78 | -- for _, prediction in pairs(predictions) do 79 | -- print(table.concat(prediction, ", ")) 80 | -- end 81 | end 82 | -------------------------------------------------------------------------------- /model.lua: -------------------------------------------------------------------------------- 1 | function learn.model.nnet(p) 2 | p.n_input = p.n_input or 1 3 | p.n_output = p.n_output or 1 4 | 5 | p.criterion = p.criterion or learn.criterion.mse({}) 6 | 7 | function p.forward(input) 8 | for _, mod in ipairs(p.modules) do 9 | input = mod.forward(input) 10 | end 11 | return input 12 | end 13 | 14 | function p.backward(input, gradients) 15 | for i = #p.modules, 1, -1 do 16 | input, gradients = p.modules[i].backward(input, gradients) 17 | end 18 | end 19 | 20 | function p.update(input, learning_rate) 21 | for i, mod in ipairs(p.modules) do 22 | input = mod.update(input, learning_rate) 23 | end 24 | end 25 | 26 | function p.fit(features, labels, epochs, learning_rate, verbose) 27 | local final_error = 1 28 | 29 | p.feature_max, p.label_max = learn.normalize(features), learn.normalize(labels) 30 | 31 | for e = 1, epochs do 32 | local error_sum = 0 33 | 34 | for i, input in ipairs(features) do 35 | input = learn.tensor({data = input}) 36 | local target = learn.tensor({data = labels[i]}) 37 | 38 | local output = p.forward(input) 39 | 40 | error_sum = error_sum + p.criterion.forward(output, target) 41 | 42 | p.backward(input, p.criterion.backward(output, target)) 43 | p.update(input, learning_rate) 44 | end 45 | 46 | final_error = error_sum / #features 47 | 48 | if verbose and (e % (epochs / 10)) == 0 then 49 | print("Error " .. math.floor(final_error / 0.00001) * 0.00001) 50 | end 51 | end 52 | 53 | return final_error 54 | end 55 | 56 | function p.predict(features) 57 | local predictions = {} 58 | 59 | for i, feature_vector in ipairs(features) do 60 | predictions[i] = p.forward(learn.tensor({data = feature_vector})).data 61 | end 62 | 63 | learn.unormalize(predictions, p.label_max) 64 | 65 | return predictions 66 | end 67 | 68 | return p 69 | end 70 | -------------------------------------------------------------------------------- /tensor.lua: -------------------------------------------------------------------------------- 1 | -- Helper class for 2D matrix math and such 2 | function learn.tensor(p) 3 | p.data = p.data or {} 4 | p.size = p.size or {#p.data, 1} 5 | 6 | function p.set(v, x, y) 7 | p.data[x + (y - 1) * p.size[1]] = v 8 | end 9 | 10 | function p.get(x, y) 11 | return p.data[x + (y - 1) * p.size[1]] 12 | end 13 | 14 | function p.copy() 15 | return learn.tensor({size = {p.size[1], p.size[2]}}).map(function(v, x, y) return p.get(x, y) end) 16 | end 17 | 18 | function p.each(func) 19 | for x = 1, p.size[1] do 20 | for y = 1, p.size[2] do 21 | func(p.get(x, y), x, y) 22 | end 23 | end 24 | return p 25 | end 26 | 27 | function p.map(func) 28 | return p.each(function(v, x, y) p.set(func(p.get(x, y), x, y), x, y) end) 29 | end 30 | 31 | function p.add(b) 32 | return p.map(function(v, x, y) return v + b.get(x, y) end) 33 | end 34 | function p.sub(b) 35 | return p.map(function(v, x, y) return v - b.get(x, y) end) 36 | end 37 | function p.div(b) 38 | return p.map(function(v, x, y) return v / b.get(x, y) end) 39 | end 40 | function p.mul(b) 41 | return p.map(function(v, x, y) return v * b.get(x, y) end) 42 | end 43 | function p.scale(s) 44 | return p.map(function(v, x, y) return v * s end) 45 | end 46 | 47 | function p.pow(e) 48 | return p.map(function(v, x, y) return math.pow(v, e) end) 49 | end 50 | 51 | function p.sum(result) 52 | result = result or learn.tensor({size = {1, 1}, data = {0}}) 53 | p.each(function(v) result.data[1] = result.data[1] + v end) 54 | return result 55 | end 56 | 57 | function p.dot(b, result) 58 | assert(p.size[2] == b.size[1], "Invalid dot product tensor size " .. p.size[2] .. " " .. b.size[1]) 59 | 60 | if result then 61 | result.size[1], result.size[2] = p.size[1], b.size[2] 62 | else 63 | result = learn.tensor({size = {p.size[1], b.size[2]}}) 64 | end 65 | 66 | result.map(function(v, x, y) 67 | local sum = 0 68 | for c = 1, p.size[2] do 69 | sum = sum + p.get(x, c) * b.get(c, y) 70 | end 71 | return sum 72 | end) 73 | 74 | return result 75 | end 76 | 77 | function p.transpose() 78 | local q = p.copy() 79 | q.size = {p.size[2], p.size[1]} 80 | p.each(function(v, x, y) 81 | q.set(p.get(x, y), y, x) 82 | end) 83 | return q 84 | end 85 | 86 | function p.string() 87 | local str = "" 88 | for x = 1, p.size[1] do 89 | for y = 1, p.size[2] do 90 | str = str .. (p.get(x, y) or "nil") .. " " 91 | end 92 | str = str .. "\n" 93 | end 94 | return str 95 | end 96 | 97 | return p 98 | end 99 | -------------------------------------------------------------------------------- /transfer.lua: -------------------------------------------------------------------------------- 1 | -- Transfer function module, simply applies the function 2 | function learn.transfer.transfer(p) 3 | function p.forward(input) 4 | p.output = input.copy().map(p.transfer) 5 | return p.output 6 | end 7 | 8 | function p.backward(input, gradients) 9 | return p.output.copy().map(p.derivative), gradients 10 | end 11 | 12 | function p.update(input) 13 | return p.output 14 | end 15 | 16 | return p 17 | end 18 | 19 | -- The sigmoid transfer function in the form of a module 20 | function learn.transfer.sigmoid(p) 21 | p = learn.transfer.transfer(p) 22 | function p.transfer(x) return 1 / (1 + math.exp(-x)) end 23 | function p.derivative(x) return x * (1 - x) end 24 | return p 25 | end 26 | 27 | -- The hyperbolic tangent transfer function in the form of a module 28 | function learn.transfer.tanh(p) 29 | p = learn.transfer.transfer(p) 30 | function p.transfer(x) 31 | local ex, enx = math.exp(x), math.exp(-x) 32 | return (ex - enx) / (ex + enx) 33 | end 34 | function p.derivative(x) 35 | local e2x = math.exp(2.0 * x) 36 | return 4 * e2x / math.pow(e2x + 1, 2.0) 37 | end 38 | return p 39 | end 40 | 41 | -- Applies the rectified linear unit function 42 | function learn.transfer.relu(p) 43 | p = learn.transfer.transfer(p) 44 | function p.transfer(x) return math.max(0, x) end 45 | function p.derivative(x) return x > 0 and 1 or 0 end 46 | return p 47 | end 48 | --------------------------------------------------------------------------------