├── Activator.go ├── Criterion.go ├── LICENSE ├── Linear.go ├── MSE.go ├── Network.go ├── README.md ├── ReLU.go ├── Sigmoid.go ├── Tanh.go ├── Utils.go ├── characters_test.go └── xor_test.go /Activator.go: -------------------------------------------------------------------------------- 1 | package drago 2 | 3 | // Activator represents the activation function for a given layer 4 | type Activator interface { 5 | // Apply calculates the activation of a given layer given 6 | // given previous layer activations 7 | Apply(int, int, float64) float64 8 | // Derivative is the calculation used to update weights during backprop 9 | Derivative(int, int, float64) float64 10 | } 11 | -------------------------------------------------------------------------------- /Criterion.go: -------------------------------------------------------------------------------- 1 | package drago 2 | 3 | import "github.com/gonum/matrix/mat64" 4 | 5 | // Criterion calculates error in predicted value for a given sample 6 | type Criterion interface { 7 | // Apply calculates the error given the predicted (first) and 8 | // actual (second) labels 9 | Apply(*mat64.Dense, *mat64.Dense) float64 10 | // Derivative calculates the derivative of the error function (Apply) 11 | // given the predicted (first) and actual (second) labels 12 | Derivative(*mat64.Dense, *mat64.Dense) *mat64.Dense 13 | } 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Alex Parella 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Linear.go: -------------------------------------------------------------------------------- 1 | package drago 2 | 3 | // Linear represents linear activation function 4 | type Linear struct { 5 | } 6 | 7 | var _ Activator = new(Linear) 8 | 9 | // Apply calculates activation for layer given previous layer value 10 | func (l *Linear) Apply(r, c int, value float64) float64 { 11 | return value 12 | } 13 | 14 | // Derivative returns linear derivative (always 1) 15 | func (l *Linear) Derivative(r, c int, value float64) float64 { 16 | return 1 17 | } 18 | -------------------------------------------------------------------------------- /MSE.go: -------------------------------------------------------------------------------- 1 | package drago 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/gonum/matrix/mat64" 7 | ) 8 | 9 | // MSE implements Mean Squared Error calculations for Criterion interface 10 | type MSE struct { 11 | } 12 | 13 | var _ Criterion = new(MSE) 14 | 15 | // Derivative calculates derivative of MSE 16 | func (m *MSE) Derivative(prediction, actual *mat64.Dense) *mat64.Dense { 17 | mat := &mat64.Dense{} 18 | mat.Sub(prediction, actual) 19 | return mat 20 | } 21 | 22 | // Apply calculates the MSE given predicted and actual labels 23 | func (m *MSE) Apply(prediction, actual *mat64.Dense) float64 { 24 | rows, _ := prediction.Dims() 25 | error := 0.0 26 | for i := 0; i < rows; i++ { 27 | diff := prediction.At(i, 0) - actual.At(i, 0) 28 | error += math.Pow(diff, 2) 29 | } 30 | return error / float64(rows) 31 | } 32 | -------------------------------------------------------------------------------- /Network.go: -------------------------------------------------------------------------------- 1 | // Package Drago provides implementation of feed forward neural network 2 | package drago 3 | 4 | import ( 5 | "fmt" 6 | 7 | "github.com/gonum/matrix/mat64" 8 | ) 9 | 10 | // Network struct represents the neural network 11 | // Values are exported but should not be messed with during training, 12 | // are exported simply for ease of examining the network state 13 | type Network struct { 14 | Activators []Activator 15 | Activations []*mat64.Dense 16 | Weights []*mat64.Dense 17 | Errors []*mat64.Dense 18 | Topology []int 19 | Layers int 20 | LearningRate float64 21 | Iterations int 22 | Loss Criterion 23 | currentErr float64 24 | // Controls logging output during training. 25 | Verbose bool 26 | } 27 | 28 | // New creates a new neural network 29 | // Topology specifies number of hidden layers and nodes in each, as well as 30 | // size of samples and labels (first and last values, respectively). 31 | // Acts array should have one activator for each hidden layer 32 | func New(learnRate float64, iterations int, topology []int, acts []Activator) *Network { 33 | net := &Network{ 34 | LearningRate: learnRate, 35 | Iterations: iterations, 36 | Activators: make([]Activator, len(topology)), 37 | Activations: make([]*mat64.Dense, len(topology)), 38 | Errors: make([]*mat64.Dense, len(topology)), 39 | Weights: make([]*mat64.Dense, len(topology)-1), 40 | Topology: topology, 41 | Layers: len(topology), 42 | Loss: new(MSE), 43 | Verbose: true, 44 | } 45 | 46 | net.initActivations(topology) 47 | net.initWeights(topology) 48 | net.initActivators(acts) 49 | net.initErrors(topology) 50 | 51 | return net 52 | } 53 | 54 | func (n *Network) verbosePrint(a ...interface{}) { 55 | if n.Verbose { 56 | fmt.Println(a...) 57 | } 58 | } 59 | 60 | func (n *Network) initActivations(topology []int) { 61 | for i, nodes := range topology { 62 | n.Activations[i] = mat64.NewDense(nodes, 1, nil) 63 | } 64 | } 65 | 66 | func (n *Network) initErrors(topology []int) { 67 | for i := 0; i < n.Layers; i++ { 68 | n.Errors[i] = mat64.NewDense(topology[i], 1, nil) 69 | } 70 | } 71 | 72 | func (n *Network) initWeights(topology []int) { 73 | for i := 0; i < n.Layers-1; i++ { 74 | n.Weights[i] = randomMatrix(topology[i+1], topology[i]) 75 | } 76 | } 77 | 78 | func (n *Network) initActivators(acts []Activator) { 79 | acts = append([]Activator{new(Linear)}, append(acts, new(Linear))...) 80 | for i := 0; i < len(acts); i++ { 81 | n.Activators[i] = acts[i] 82 | } 83 | } 84 | 85 | // Predict returns the predicted value of the provided sample 86 | // Dimensions must match those from provided topology 87 | // Only use after training the network 88 | func (n *Network) Predict(sample []float64) *mat64.Dense { 89 | n.Forward(sample) 90 | return n.Activations[n.Layers-1] 91 | } 92 | 93 | // Learn trains the network using the provided dataset 94 | // Samples must have number of features and labels as specified by topology 95 | // when constructing the network 96 | func (n *Network) Learn(dataset [][][]float64) { 97 | n.verbosePrint("Learning...") 98 | for i := 0; i < n.Iterations; i++ { 99 | n.verbosePrint("=== Iteration ", i+1, " ===") 100 | n.currentErr = 0 101 | for _, sample := range dataset { 102 | n.Forward(sample[0]) 103 | n.Back(sample[1]) 104 | } 105 | n.verbosePrint("Error : ", n.currentErr/float64(len(dataset))) 106 | } 107 | } 108 | 109 | // Forward calculates activations at each layer for given sample 110 | func (n *Network) Forward(sample []float64) { 111 | n.Activations[0].SetCol(0, sample) 112 | for i := 0; i < len(n.Weights); i++ { 113 | n.activateLayer(i) 114 | } 115 | } 116 | 117 | func (n *Network) activateLayer(layer int) { 118 | n.Activations[layer+1].Mul(n.Weights[layer], n.Activations[layer]) 119 | n.Activations[layer+1].Apply(n.Activators[layer+1].Apply, n.Activations[layer+1]) 120 | } 121 | 122 | // Back performs back propagation to update weights at each layer 123 | func (n *Network) Back(label []float64) { 124 | n.calculateErrors(label) 125 | n.updateWeights() 126 | } 127 | 128 | func (n *Network) calculateErrors(label []float64) { 129 | actual := mat64.NewDense(len(label), 1, label) 130 | n.Errors[n.Layers-1] = n.Loss.Derivative(n.Activations[n.Layers-1], actual) 131 | n.currentErr += n.Loss.Apply(n.Activations[n.Layers-1], actual) 132 | for i := n.Layers - 2; i >= 0; i-- { 133 | n.calculateErrorForLayer(i) 134 | } 135 | } 136 | 137 | func (n *Network) calculateErrorForLayer(layer int) { 138 | n.Errors[layer].Mul(n.Weights[layer].T(), n.Errors[layer+1]) 139 | n.Errors[layer].MulElem(n.Errors[layer], n.Activations[layer]) 140 | mat := &mat64.Dense{} 141 | mat.Apply(n.Activators[layer].Derivative, n.Activations[layer]) 142 | n.Errors[layer].MulElem(mat, n.Errors[layer]) 143 | } 144 | 145 | func (n *Network) updateWeights() { 146 | for i := 0; i < n.Layers-1; i++ { 147 | mat := &mat64.Dense{} 148 | mat.Mul(n.Errors[i+1], n.Activations[i].T()) 149 | mat.Scale(n.LearningRate, mat) 150 | n.Weights[i].Sub(n.Weights[i], mat) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | drago 2 | === 3 | 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/aaparella/drago)](https://goreportcard.com/report/github.com/aaparella/drago) 5 | 6 | Simple feed forward neural network implementation. Still need to add some nice utility functions, the logic can stand to be cleaned up in some places, but the algorithms are implemented and it can be used. 7 | 8 | Usage: 9 | === 10 | 11 | acts := drago.Activator[]{new(drago.Sigmoid), new(drago.Sigmoid)} 12 | net := drago.New(0.1, 25, []int{5, 2, 2, 1}, acts) 13 | net.Learn([][][]float64{ 14 | {{0, 0}, {1}}, 15 | {{0, 1}, {0}}, 16 | {{1, 1}, {0}}, 17 | }) 18 | 19 | // Predict a value 20 | fmt.Println(net.Predict([]float64{1, 1}) 21 | 22 | To add an activation function: 23 | === 24 | 25 | An activation function needs both the function and it's derivative. See Sigmoid.go, Tanh.go, and ReLU.go for examples of this. 26 | 27 | type YourActivationFunction struct { 28 | } 29 | 30 | func (y *YourActivationFunction) Apply(r, c int, val float64) float64 { 31 | // ... 32 | } 33 | 34 | func (y *YourActivationFunction) Derivative(r, c int, val float64) float64 { 35 | // ... 36 | } 37 | -------------------------------------------------------------------------------- /ReLU.go: -------------------------------------------------------------------------------- 1 | package drago 2 | 3 | import "math" 4 | 5 | // ReLU struct represents a rectified linear unit 6 | type ReLU struct { 7 | } 8 | 9 | var _ Activator = new(ReLU) 10 | 11 | // Apply ReLU calculation to val 12 | func (u *ReLU) Apply(r, c int, val float64) float64 { 13 | return math.Log(1 + math.Exp(val)) 14 | } 15 | 16 | // Derivative calculates derivative of ReLU for val 17 | func (u *ReLU) Derivative(r, c int, val float64) float64 { 18 | return 1.0 / (1.0 + math.Exp(-val)) 19 | } 20 | -------------------------------------------------------------------------------- /Sigmoid.go: -------------------------------------------------------------------------------- 1 | package drago 2 | 3 | import "math" 4 | 5 | // Sigmoid represents sigmoid activation function 6 | type Sigmoid struct { 7 | } 8 | 9 | var _ Activator = new(Sigmoid) 10 | 11 | // Apply calculates sigmoid of given value, r and c ignored 12 | func (s *Sigmoid) Apply(r, c int, value float64) float64 { 13 | return 1 / (1 + math.Exp(-value)) 14 | } 15 | 16 | // Derivative calculates sigmoid derivative of given value, r and c ignored 17 | func (s *Sigmoid) Derivative(r, c int, value float64) float64 { 18 | res := s.Apply(r, c, value) 19 | return res * (1 - res) 20 | } 21 | -------------------------------------------------------------------------------- /Tanh.go: -------------------------------------------------------------------------------- 1 | package drago 2 | 3 | import "math" 4 | 5 | // Tanh struct represents hyperbolic tangent activation function 6 | type Tanh struct { 7 | } 8 | 9 | var _ Activator = new(Tanh) 10 | 11 | // Apply calculates hyperbolic tangent of val, r and c ignored 12 | func (t *Tanh) Apply(r, c int, val float64) float64 { 13 | return (1 - math.Exp(-2*val)) / (1 + math.Exp(-2*val)) 14 | } 15 | 16 | // Derivative calculates derivative of hyperbolic tangent for val, r and c ignored 17 | func (t *Tanh) Derivative(r, c int, val float64) float64 { 18 | return 1 - (math.Pow((math.Exp(2*val)-1)/(math.Exp(2*val)+1), 2)) 19 | } 20 | -------------------------------------------------------------------------------- /Utils.go: -------------------------------------------------------------------------------- 1 | package drago 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | 7 | "github.com/gonum/matrix/mat64" 8 | ) 9 | 10 | func randomMatrix(r, c int) *mat64.Dense { 11 | rand.Seed(time.Now().UnixNano()) 12 | data := make([]float64, r*c) 13 | for i := range data { 14 | data[i] = rand.Float64() / 2.0 15 | } 16 | return mat64.NewDense(r, c, data) 17 | } 18 | -------------------------------------------------------------------------------- /characters_test.go: -------------------------------------------------------------------------------- 1 | // Example borrowed from https://github.com/stevenmiller888/go-mind 2 | 3 | package drago_test 4 | 5 | import ( 6 | "log" 7 | 8 | "github.com/aaparella/drago" 9 | ) 10 | 11 | var ( 12 | a = character( 13 | ".#####." + 14 | "#.....#" + 15 | "#.....#" + 16 | "#######" + 17 | "#.....#" + 18 | "#.....#" + 19 | "#.....#") 20 | b = character( 21 | "######." + 22 | "#.....#" + 23 | "#.....#" + 24 | "######." + 25 | "#.....#" + 26 | "#.....#" + 27 | "######.") 28 | c = character( 29 | "#######" + 30 | "#......" + 31 | "#......" + 32 | "#......" + 33 | "#......" + 34 | "#......" + 35 | "#######") 36 | ) 37 | 38 | func ExampleLearn() { 39 | m := drago.New(0.3, 10000, []int{49, 3, 1}, []drago.Activator{new(drago.Sigmoid)}) 40 | m.Learn([][][]float64{ 41 | {c, []float64{.5}}, 42 | {b, []float64{.3}}, 43 | {a, []float64{.1}}, 44 | }) 45 | 46 | result := m.Predict( 47 | character( 48 | "#######" + 49 | "#......" + 50 | "#......" + 51 | "#......" + 52 | "#......" + 53 | "#......" + 54 | "#######")) 55 | log.Println(result) 56 | } 57 | func character(chars string) []float64 { 58 | flt := make([]float64, len(chars)) 59 | for i := 0; i < len(chars); i++ { 60 | if chars[i] == '#' { 61 | flt[i] = 1.0 62 | } else { // if '.' 63 | flt[i] = 0.0 64 | } 65 | } 66 | return flt 67 | } 68 | -------------------------------------------------------------------------------- /xor_test.go: -------------------------------------------------------------------------------- 1 | package drago_test 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aaparella/drago" 7 | ) 8 | 9 | func ExamplePredict() { 10 | acts := []drago.Activator{new(drago.Sigmoid)} 11 | net := drago.New(.7, 1000, []int{2, 3, 1}, acts) 12 | net.Learn([][][]float64{ 13 | {{0, 0}, {0}}, 14 | {{1, 0}, {1}}, 15 | {{0, 1}, {1}}, 16 | {{1, 1}, {0}}, 17 | }) 18 | 19 | fmt.Println("XOR value for {0, 0}, =>", net.Predict([]float64{0, 0})) 20 | fmt.Println("XOR value for {1, 0}, =>", net.Predict([]float64{1, 0})) 21 | fmt.Println("XOR value for {0, 1}, =>", net.Predict([]float64{0, 1})) 22 | fmt.Println("XOR value for {1, 1}, =>", net.Predict([]float64{1, 1})) 23 | } 24 | --------------------------------------------------------------------------------