├── LICENSE ├── README.md ├── activation.go ├── examples ├── evolve.go ├── layers.go ├── persist.go ├── rgb.go └── xor.go ├── go.mod ├── layer.go ├── loss.go ├── neural.go └── neuron.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Lucas Barrena 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # neural-go 2 | 3 | Genetic Neural Networks 4 | 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/LuKks/neural-go?t=0)](https://goreportcard.com/report/github.com/LuKks/neural-go) ![](https://img.shields.io/github/v/release/LuKks/neural-go) [![GoDoc](https://godoc.org/github.com/LuKks/neural-go?status.svg)](https://godoc.org/github.com/LuKks/neural-go) ![](https://img.shields.io/github/license/LuKks/neural-go.svg) 6 | 7 | ```golang 8 | package main 9 | 10 | import ( 11 | "fmt" 12 | "github.com/lukks/neural-go/v3" 13 | ) 14 | 15 | func main() { 16 | xor := neural.NewNeural([]*neural.Layer{ 17 | {Inputs: 2, Units: 16}, 18 | {Units: 16}, 19 | {Units: 1}, 20 | }) 21 | 22 | for i := 0; i <= 5000; i++ { 23 | loss := xor.Learns([][][]float64{ 24 | {{0, 0}, {0}}, 25 | {{1, 0}, {1}}, 26 | {{0, 1}, {1}}, 27 | {{1, 1}, {0}}, 28 | }) 29 | 30 | if i%1000 == 0 { 31 | fmt.Printf("iter %v, loss %f\n", i, loss) 32 | } 33 | } 34 | 35 | fmt.Printf("think some values:\n") 36 | fmt.Printf("0, 0 [0] -> %f\n", xor.Think([]float64{0, 0})) 37 | fmt.Printf("1, 0 [1] -> %f\n", xor.Think([]float64{1, 0})) 38 | fmt.Printf("0, 1 [1] -> %f\n", xor.Think([]float64{0, 1})) 39 | fmt.Printf("1, 1 [0] -> %f\n", xor.Think([]float64{1, 1})) 40 | } 41 | ``` 42 | 43 | ## Install latest version 44 | ``` 45 | go get github.com/lukks/neural-go/v3 46 | ``` 47 | 48 | Also find versions on [releases](https://github.com/LuKks/neural-go/releases). 49 | The changes from v2 to v3 were just for go mod versioning. 50 | 51 | ## Features 52 | #### Range 53 | Set a range of values for every input and output.\ 54 | So you use your values as you know but the neural get it in raw activation.\ 55 | Check [examples/rgb.go](https://github.com/LuKks/neural-go/blob/master/examples/rgb.go) for usage example. 56 | 57 | #### Customizable 58 | Set different activations, rates, momentums, etc at layer level. 59 | - Activation: `linear`, `sigmoid` (default), `tanh` and `relu` 60 | - Learning Rate 61 | - Optimizer by Momentum 62 | - Loss: for output layer, only `mse` for now 63 | - Range: for input and output layer 64 | 65 | Check [examples/layers.go](https://github.com/LuKks/neural-go/blob/master/examples/layers.go) for complete example. 66 | 67 | #### Genetics 68 | Clone, mutate and crossover neurons, layers and neurals.\ 69 | The `Evolve` method internally uses these methods to put this very easy.\ 70 | Check [examples/evolve.go](https://github.com/LuKks/neural-go/blob/master/examples/evolve.go) but it's optional, not always need to use genetics. 71 | 72 | #### Utils 73 | There are several useful methods: Export, Import, Reset, ToFile, FromFile, etc.\ 74 | Check the [documentation here](https://godoc.org/github.com/LuKks/neural-go). 75 | 76 | #### Description 77 | From my previous [neural-amxx](https://github.com/LuKks/neural-amxx). 78 | 79 | ## Examples 80 | Basic XOR [examples/xor.go](https://github.com/LuKks/neural-go/blob/master/examples/xor.go)\ 81 | RGB brightness [examples/rgb.go](https://github.com/LuKks/neural-go/blob/master/examples/rgb.go)\ 82 | Genetics [examples/evolve.go](https://github.com/LuKks/neural-go/blob/master/examples/evolve.go)\ 83 | Layer configs [examples/layers.go](https://github.com/LuKks/neural-go/blob/master/examples/layers.go)\ 84 | Persist [examples/persist.go](https://github.com/LuKks/neural-go/blob/master/examples/persist.go) 85 | 86 | ``` 87 | go run examples/rgb.go 88 | ``` 89 | 90 | ## Tests 91 | ``` 92 | There are no tests yet 93 | ``` 94 | 95 | ## Issues 96 | Feedback, ideas, etc are very welcome so feel free to open an issue. 97 | 98 | ## License 99 | Code released under the [MIT License](https://github.com/LuKks/neural-go/blob/master/LICENSE). 100 | -------------------------------------------------------------------------------- /activation.go: -------------------------------------------------------------------------------- 1 | package neural 2 | 3 | import ( 4 | "math" 5 | ) 6 | 7 | // ForwardFn is used to think 8 | type ForwardFn func(sum float64) float64 9 | 10 | // BackwardFn is used to learn (derivative of forward) 11 | type BackwardFn func(activation float64) float64 12 | 13 | // LinearForward is the linear fn 14 | func LinearForward(sum float64) float64 { 15 | return sum 16 | } 17 | 18 | // LinearBackward is the linear derivative 19 | func LinearBackward(activation float64) float64 { 20 | return 1.0 21 | } 22 | 23 | // SigmoidForward is the sigmoid fn 24 | func SigmoidForward(sum float64) float64 { 25 | return 1.0 / (1.0 + math.Exp(-sum)) 26 | } 27 | 28 | // SigmoidBackward is the sigmoid derivative 29 | func SigmoidBackward(activation float64) float64 { 30 | return activation * (1.0 - activation) 31 | } 32 | 33 | // TanhForward is the tanh fn 34 | func TanhForward(sum float64) float64 { 35 | return math.Tanh(sum) 36 | } 37 | 38 | // TanhBackward is the tanh derivative 39 | func TanhBackward(activation float64) float64 { 40 | return 1 - activation*activation 41 | } 42 | 43 | // ReluForward is the relu fn 44 | func ReluForward(sum float64) float64 { 45 | if sum < 0.0 { 46 | return 0.0 47 | } 48 | return sum 49 | } 50 | 51 | // ReluBackward is the relu derivative 52 | func ReluBackward(activation float64) float64 { 53 | if activation <= 0.0 { 54 | return 0.0 55 | } 56 | return 1.0 57 | } 58 | 59 | // ActivationSet is a forward and backward fn with its range 60 | type ActivationSet struct { 61 | Forward ForwardFn 62 | Backward BackwardFn 63 | // Range of the activation 64 | Ranges []float64 65 | } 66 | 67 | func selectActivation(activation string) ActivationSet { 68 | set := ActivationSet{} 69 | 70 | if activation == "linear" { 71 | set.Forward = LinearForward 72 | set.Backward = LinearBackward 73 | } else if activation == "" || activation == "sigmoid" { 74 | set.Forward = SigmoidForward 75 | set.Backward = SigmoidBackward 76 | set.Ranges = []float64{0.0, 1.0} 77 | } else if activation == "tanh" { 78 | set.Forward = TanhForward 79 | set.Backward = TanhBackward 80 | set.Ranges = []float64{-1.0, 1.0} 81 | } else if activation == "relu" { 82 | set.Forward = ReluForward 83 | set.Backward = ReluBackward 84 | set.Ranges = []float64{0.0, 1.0} 85 | } else { 86 | panic("need a valid activation name") 87 | } 88 | 89 | return set 90 | } 91 | -------------------------------------------------------------------------------- /examples/evolve.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/lukks/neural-go/v3" 6 | "time" 7 | ) 8 | 9 | const fmtColor = "\033[0;36m%s\033[0m" 10 | 11 | func main() { 12 | xor := neural.NewNeural([]*neural.Layer{ 13 | {Inputs: 2, Units: 3}, 14 | {Units: 3}, 15 | {Units: 1, Loss: "mse"}, 16 | }) 17 | 18 | fmt.Printf(fmtColor, "think some values:\n") 19 | fmt.Printf("0, 0 [0] -> %f\n", xor.Think([]float64{0, 0})) 20 | fmt.Printf("1, 0 [1] -> %f\n", xor.Think([]float64{1, 0})) 21 | fmt.Printf("0, 1 [1] -> %f\n", xor.Think([]float64{0, 1})) 22 | fmt.Printf("1, 1 [0] -> %f\n", xor.Think([]float64{1, 1})) 23 | 24 | fmt.Printf(fmtColor, "learning:\n") 25 | start := millis() 26 | 27 | xor = xor.Evolve(neural.Evolve{ 28 | Population: 20, 29 | Mutate: 0.05, 30 | Crossover: 0.5, 31 | Elitism: 5, 32 | Epochs: 100, 33 | Iterations: 50, 34 | Threshold: 0.00005, 35 | Dataset: [][][]float64{ 36 | {{0, 0}, {0}}, 37 | {{1, 0}, {1}}, 38 | {{0, 1}, {1}}, 39 | {{1, 1}, {0}}, 40 | }, 41 | Callback: func(epoch int, loss float64) bool { 42 | if epoch%10 == 0 || epoch == 99 { 43 | fmt.Printf("epoch=%v loss=%f elapsed=%v\n", epoch, loss, millis()-start) 44 | } 45 | 46 | return true 47 | }, 48 | }) 49 | 50 | fmt.Printf(fmtColor, "think some values:\n") 51 | fmt.Printf("0, 0 [0] -> %f\n", xor.Think([]float64{0, 0})) 52 | fmt.Printf("1, 0 [1] -> %f\n", xor.Think([]float64{1, 0})) 53 | fmt.Printf("0, 1 [1] -> %f\n", xor.Think([]float64{0, 1})) 54 | fmt.Printf("1, 1 [0] -> %f\n", xor.Think([]float64{1, 1})) 55 | 56 | fmt.Printf(fmtColor, "export:\n") 57 | json, _ := xor.Export() 58 | fmt.Printf("%s\n", json) 59 | // stream the json over network 60 | 61 | // or just xor.ToFile("./evolve.json") 62 | } 63 | 64 | func millis() int64 { 65 | return time.Now().UnixNano() / 1e6 66 | } 67 | -------------------------------------------------------------------------------- /examples/layers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/lukks/neural-go/v3" 6 | ) 7 | 8 | const fmtColor = "\033[0;36m%s\033[0m" 9 | 10 | func main() { 11 | xor := neural.NewNeural([]*neural.Layer{ 12 | {Inputs: 2, Units: 16, Activation: "sigmoid", Rate: 0.002, Momentum: 0.999}, 13 | {Units: 16, Activation: "tanh", Rate: 0.001}, 14 | {Units: 1, Activation: "sigmoid", Loss: "mse", Rate: 0.0005}, 15 | }) 16 | // that is just to show different configurations 17 | // normally you want same rate and momentum for all layers 18 | 19 | /* 20 | // Change rate or momentum to all layers 21 | xor.Rate(0.002) 22 | xor.Momentum(0.999) 23 | 24 | // Change to specific layer 25 | xor.Layers[0].Rate = 0.002 26 | xor.Layers[0].Momentum = 0.999 27 | */ 28 | 29 | fmt.Printf(fmtColor, "learning:\n") 30 | for i := 0; i <= 5000; i++ { 31 | loss := xor.Learns([][][]float64{ 32 | {{0, 0}, {0}}, 33 | {{1, 0}, {1}}, 34 | {{0, 1}, {1}}, 35 | {{1, 1}, {0}}, 36 | }) 37 | 38 | if i%1000 == 0 { 39 | fmt.Printf("iter %v, loss %f\n", i, loss) 40 | } 41 | } 42 | 43 | fmt.Printf(fmtColor, "think some values:\n") 44 | fmt.Printf("0, 0 [0] -> %f\n", xor.Think([]float64{0, 0})) 45 | fmt.Printf("1, 0 [1] -> %f\n", xor.Think([]float64{1, 0})) 46 | fmt.Printf("0, 1 [1] -> %f\n", xor.Think([]float64{0, 1})) 47 | fmt.Printf("1, 1 [0] -> %f\n", xor.Think([]float64{1, 1})) 48 | } 49 | -------------------------------------------------------------------------------- /examples/persist.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/lukks/neural-go/v3" 6 | ) 7 | 8 | const fmtColor = "\033[0;36m%s\033[0m" 9 | 10 | func main() { 11 | xor := neural.NewNeural([]*neural.Layer{ 12 | {Inputs: 2, Units: 3}, 13 | {Units: 3}, 14 | {Units: 1}, 15 | }) 16 | 17 | fmt.Printf(fmtColor, "learning:\n") 18 | for i := 0; i <= 50000; i++ { 19 | loss := xor.Learns([][][]float64{ 20 | {{0, 0}, {0}}, 21 | {{1, 0}, {1}}, 22 | {{0, 1}, {1}}, 23 | {{1, 1}, {0}}, 24 | }) 25 | 26 | if i%10000 == 0 { 27 | fmt.Printf("iter %v, loss %f\n", i, loss) 28 | } 29 | } 30 | 31 | fmt.Printf(fmtColor, "think some values:\n") 32 | fmt.Printf("0, 0 [0] -> %f\n", xor.Think([]float64{0, 0})) 33 | fmt.Printf("1, 0 [1] -> %f\n", xor.Think([]float64{1, 0})) 34 | fmt.Printf("0, 1 [1] -> %f\n", xor.Think([]float64{0, 1})) 35 | fmt.Printf("1, 1 [0] -> %f\n", xor.Think([]float64{1, 1})) 36 | 37 | fmt.Printf(fmtColor, "to file:\n") 38 | err1 := xor.ToFile("./evolve.json") 39 | fmt.Printf("err? %v\n", err1) 40 | 41 | fmt.Printf(fmtColor, "from file:\n") 42 | err2 := xor.FromFile("./evolve.json") 43 | fmt.Printf("err? %v\n", err2) 44 | 45 | fmt.Printf(fmtColor, "delete file:\n") 46 | err3 := xor.DeleteFile("./evolve.json") 47 | fmt.Printf("err? %v\n", err3) 48 | 49 | fmt.Printf(fmtColor, "to string (export):\n") 50 | json, _ := xor.Export() 51 | fmt.Printf("%s\n", json) 52 | 53 | // can stream the exported json over network 54 | // then in a different process/machine: 55 | fmt.Printf(fmtColor, "from string (import):\n") 56 | xorCopy := neural.NewNeural([]*neural.Layer{}) 57 | err4 := xorCopy.Import(json) 58 | fmt.Printf("err? %v\n", err4) 59 | } 60 | -------------------------------------------------------------------------------- /examples/rgb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/lukks/neural-go/v3" 6 | ) 7 | 8 | const fmtColor = "\033[0;36m%s\033[0m" 9 | 10 | func main() { 11 | rgb := neural.NewNeural([]*neural.Layer{ 12 | {Inputs: 3, Units: 8, Range: [][]float64{{0, 255}, {0, 255}, {0, 255}}}, // r, g, b 0-255 13 | {Units: 8}, 14 | {Units: 1, Range: [][]float64{{0, 100}}}, // brightness 0-100 (percentage) 15 | }) 16 | 17 | fmt.Printf(fmtColor, "think some values:\n") 18 | fmt.Printf("255, 255, 255 [100] -> %f\n", rgb.Think([]float64{255, 255, 255})) 19 | fmt.Printf("0 , 0 , 0 [0] -> %f\n", rgb.Think([]float64{0, 0, 0})) 20 | 21 | fmt.Printf(fmtColor, "learning:\n") 22 | for i := 0; i <= 2000; i++ { 23 | light := []float64{100} 24 | dark := []float64{0} 25 | 26 | // LearnRaw doesn't use Range 27 | loss := rgb.LearnRaw([]float64{1, 0, 0}, []float64{1}) 28 | loss += rgb.Learn([]float64{0, 255, 0}, light) 29 | loss += rgb.Learn([]float64{0, 0, 255}, light) 30 | loss += rgb.Learn([]float64{0, 0, 0}, dark) 31 | loss += rgb.Learn([]float64{100, 100, 100}, light) 32 | loss += rgb.Learn([]float64{107, 181, 255}, light) 33 | loss += rgb.Learn([]float64{0, 53, 105}, dark) 34 | loss += rgb.Learn([]float64{150, 150, 75}, light) 35 | loss += rgb.Learn([]float64{75, 75, 0}, dark) 36 | loss += rgb.Learn([]float64{0, 75, 75}, dark) 37 | loss += rgb.Learn([]float64{150, 74, 142}, light) 38 | loss += rgb.Learn([]float64{50, 50, 75}, dark) 39 | loss += rgb.Learn([]float64{103, 22, 94}, dark) 40 | loss /= 13 41 | 42 | if i%500 == 0 { 43 | fmt.Printf("iter %v, loss %f\n", i, loss) 44 | } 45 | } 46 | 47 | fmt.Printf(fmtColor, "think some values:\n") 48 | fmt.Printf("255, 255, 255 [100] -> %f\n", rgb.Think([]float64{255, 255, 255})) 49 | fmt.Printf("0 , 0 , 0 [0] -> %f\n", rgb.Think([]float64{0, 0, 0})) 50 | 51 | fmt.Printf(fmtColor, "think new values:\n") 52 | fmt.Printf("243, 179, 10 [100] -> %f\n", rgb.Think([]float64{243, 179, 10})) 53 | fmt.Printf("75 , 50 , 50 [0] -> %f\n", rgb.Think([]float64{75, 50, 50})) 54 | fmt.Printf("95 , 99 , 104 [100] -> %f\n", rgb.Think([]float64{95, 99, 104})) 55 | fmt.Printf("65 , 38 , 70 [0] -> %f\n", rgb.Think([]float64{65, 38, 70})) 56 | 57 | // ThinkRaw doesn't use Range 58 | fmt.Printf(fmtColor, "example using ThinkRaw:\n") 59 | fmt.Printf("65 , 38 , 70 [0.0] -> %f\n", rgb.ThinkRaw([]float64{0.254, 0.149, 0.274})) 60 | 61 | } 62 | -------------------------------------------------------------------------------- /examples/xor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/lukks/neural-go/v3" 6 | ) 7 | 8 | const fmtColor = "\033[0;36m%s\033[0m" 9 | 10 | func main() { 11 | xor := neural.NewNeural([]*neural.Layer{ 12 | {Inputs: 2, Units: 16}, // input 13 | {Units: 16}, // hidden/s 14 | {Units: 1}, // output 15 | }) 16 | 17 | fmt.Printf(fmtColor, "think some values:\n") 18 | fmt.Printf("0, 0 [0] -> %f\n", xor.Think([]float64{0, 0})) 19 | fmt.Printf("1, 0 [1] -> %f\n", xor.Think([]float64{1, 0})) 20 | fmt.Printf("0, 1 [1] -> %f\n", xor.Think([]float64{0, 1})) 21 | fmt.Printf("1, 1 [0] -> %f\n", xor.Think([]float64{1, 1})) 22 | 23 | fmt.Printf(fmtColor, "learning:\n") 24 | for i := 0; i <= 5000; i++ { 25 | loss := xor.Learns([][][]float64{ 26 | {{0, 0}, {0}}, 27 | {{1, 0}, {1}}, 28 | {{0, 1}, {1}}, 29 | {{1, 1}, {0}}, 30 | }) 31 | 32 | if i%1000 == 0 { 33 | fmt.Printf("iter %v, loss %f\n", i, loss) 34 | } 35 | } 36 | 37 | fmt.Printf(fmtColor, "think some values:\n") 38 | fmt.Printf("0, 0 [0] -> %f\n", xor.Think([]float64{0, 0})) 39 | fmt.Printf("1, 0 [1] -> %f\n", xor.Think([]float64{1, 0})) 40 | fmt.Printf("0, 1 [1] -> %f\n", xor.Think([]float64{0, 1})) 41 | fmt.Printf("1, 1 [0] -> %f\n", xor.Think([]float64{1, 1})) 42 | } 43 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/lukks/neural-go/v3 2 | 3 | go 1.13 4 | -------------------------------------------------------------------------------- /layer.go: -------------------------------------------------------------------------------- 1 | package neural 2 | 3 | // Layer is a set of neurons + config 4 | type Layer struct { 5 | // Amount of inputs (default is previous layer units) 6 | Inputs int `json:"-"` 7 | Units int `json:"-"` 8 | Neurons []*Neuron `json:"Neurons"` 9 | // Default activation is sigmoid 10 | Activation string `json:"Activation,omitempty"` 11 | Forward ForwardFn `json:"-"` 12 | Backward BackwardFn `json:"-"` 13 | // Default loss is mse 14 | Loss string `json:"Loss,omitempty"` 15 | LossFn LossFn `json:"-"` 16 | // Default rate is 0.001 17 | Rate float64 `json:"-"` 18 | // Default momentum is 0.999 19 | Momentum float64 `json:"-"` 20 | // Range of arbitrary values for input/output layers 21 | Range [][]float64 `json:"Range,omitempty"` 22 | } 23 | 24 | // NewLayer creates a layer based on simple layer definition 25 | func NewLayer(layer *Layer) *Layer { 26 | if layer.Rate == 0.0 { 27 | layer.Rate = 0.001 28 | } 29 | 30 | if layer.Momentum == 0.0 { 31 | layer.Momentum = 0.999 32 | } 33 | 34 | layer.Neurons = make([]*Neuron, layer.Units) 35 | for i := 0; i < layer.Units; i++ { 36 | layer.Neurons[i] = NewNeuron(layer, layer.Inputs) 37 | } 38 | 39 | activation := layer.SetActivation(layer.Activation) 40 | 41 | if len(activation.Ranges) == 0 { 42 | layer.Range = [][]float64{} 43 | } 44 | 45 | for i, total := 0, len(layer.Range); i < total; i++ { 46 | layer.Range[i] = append(layer.Range[i], activation.Ranges[0], activation.Ranges[1]) 47 | } 48 | 49 | return layer 50 | } 51 | 52 | // Think process the layer forward based on inputs 53 | func (layer *Layer) Think(inputs []float64) []float64 { 54 | outs := make([]float64, layer.Units) 55 | 56 | for i := 0; i < layer.Units; i++ { 57 | outs[i] = layer.Neurons[i].Think(inputs) 58 | } 59 | 60 | return outs 61 | } 62 | 63 | // Clone layer with same neurons, activation, range, etc 64 | func (layer *Layer) Clone() *Layer { 65 | clone := NewLayer(&Layer{ 66 | Inputs: layer.Inputs, 67 | Units: layer.Units, 68 | Activation: layer.Activation, 69 | Rate: layer.Rate, 70 | Momentum: layer.Momentum, 71 | }) 72 | 73 | for i := 0; i < clone.Units; i++ { 74 | clone.Neurons[i] = layer.Neurons[i].Clone() 75 | } 76 | 77 | clone.Range = make([][]float64, len(layer.Range)) 78 | copy(clone.Range, layer.Range) 79 | 80 | return clone 81 | } 82 | 83 | // Mutate neurons of layer based on probability 84 | func (layer *Layer) Mutate(probability float64) { 85 | for i := 0; i < layer.Units; i++ { 86 | layer.Neurons[i].Mutate(probability) 87 | } 88 | } 89 | 90 | // Crossover two layers merging neurons 91 | func (layer *Layer) Crossover(layerB *Layer, dominant float64) *Layer { 92 | new := NewLayer(&Layer{ 93 | Inputs: layer.Inputs, 94 | Units: layer.Units, 95 | Activation: layer.Activation, 96 | Rate: layer.Rate, 97 | Momentum: layer.Momentum, 98 | }) 99 | 100 | for i := 0; i < layer.Units; i++ { 101 | new.Neurons[i] = layer.Neurons[i].Crossover(*layerB.Neurons[i], dominant) 102 | } 103 | 104 | new.Range = make([][]float64, len(layer.Range)) 105 | copy(new.Range, layer.Range) 106 | 107 | return new 108 | } 109 | 110 | // Reset every neuron (weights, bias, etc) 111 | func (layer *Layer) Reset() { 112 | for i := 0; i < layer.Units; i++ { 113 | layer.Neurons[i].Reset() 114 | } 115 | } 116 | 117 | // SetActivation set or change the activation functions based on name 118 | func (layer *Layer) SetActivation(activation string) ActivationSet { 119 | set := selectActivation(activation) 120 | layer.Forward = set.Forward 121 | layer.Backward = set.Backward 122 | return set 123 | } 124 | -------------------------------------------------------------------------------- /loss.go: -------------------------------------------------------------------------------- 1 | package neural 2 | 3 | // LossFn is used to calculate the loss 4 | type LossFn func(output float64, current float64) float64 5 | -------------------------------------------------------------------------------- /neural.go: -------------------------------------------------------------------------------- 1 | package neural 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | "sort" 8 | ) 9 | 10 | // Neural is a set of layers 11 | type Neural struct { 12 | MaxLayers int `json:"-"` 13 | Layers []*Layer `json:"Layers"` 14 | // Average of loss (used in Learns, LearnsRaw and Evolve) 15 | Loss float64 `json:"-"` 16 | } 17 | 18 | // Evolve is the config for evolution process 19 | type Evolve struct { 20 | Population int 21 | Mutate float64 22 | Crossover float64 23 | Elitism int 24 | Epochs int 25 | Iterations int 26 | Threshold float64 27 | Dataset [][][]float64 28 | Callback func(epoch int, loss float64) bool 29 | } 30 | 31 | // NewNeural creates a neural based on multiple layers 32 | func NewNeural(layers []*Layer) *Neural { 33 | neural := &Neural{ 34 | MaxLayers: len(layers), 35 | Layers: make([]*Layer, len(layers)), 36 | } 37 | 38 | for i, prevUnits := 0, 0; i < neural.MaxLayers; i++ { 39 | if layers[i].Inputs == 0 { 40 | if prevUnits == 0 { 41 | panic("need the first layer with defined inputs") 42 | } 43 | 44 | layers[i].Inputs = prevUnits 45 | } 46 | 47 | prevUnits = layers[i].Units 48 | neural.Layers[i] = NewLayer(layers[i]) 49 | } 50 | 51 | return neural 52 | } 53 | 54 | // ThinkRaw process the neural forward based on inputs and then based on output of previous layer 55 | func (neural *Neural) ThinkRaw(inputs []float64) []float64 { 56 | outs := neural.Layers[0].Think(inputs) 57 | 58 | for i := 1; i < neural.MaxLayers; i++ { 59 | outs = neural.Layers[i].Think(outs) 60 | } 61 | 62 | return outs 63 | } 64 | 65 | // Think arbitrary values by automatic conversion to raw values and vice versa for output 66 | func (neural *Neural) Think(inputs []float64) []float64 { 67 | return neural.OutputValuesFromRaw(neural.ThinkRaw(neural.InputValuesToRaw(inputs))) 68 | } 69 | 70 | // LearnRaw uses backpropagation 71 | func (neural *Neural) LearnRaw(inputs []float64, outputs []float64) float64 { 72 | loss := 0.0 73 | outputLayer := neural.Layers[neural.MaxLayers-1] 74 | currentOut := neural.ThinkRaw(inputs) 75 | 76 | for o, output := range outputLayer.Neurons { 77 | output.error = outputs[o] - currentOut[o] 78 | output.delta = output.Layer.Backward(output.activation) * output.error 79 | loss += output.error * output.error 80 | } 81 | 82 | for l := neural.MaxLayers - 2; l >= 0; l-- { 83 | layer := neural.Layers[l] 84 | nextLayer := neural.Layers[l+1] 85 | 86 | for h, hidden := range layer.Neurons { 87 | hidden.error = 0.0 88 | for _, next := range nextLayer.Neurons { 89 | hidden.error += next.Weights[h] * next.delta 90 | } 91 | hidden.delta = hidden.Layer.Backward(hidden.activation) * hidden.error 92 | } 93 | 94 | for _, next := range nextLayer.Neurons { 95 | for w := 0; w < next.MaxInputs; w++ { 96 | next.Weights[w] += next.Optimizer(w, next.Inputs[w]*next.delta*next.Layer.Rate) 97 | } 98 | next.Bias += next.Optimizer(next.MaxInputs, next.delta*next.Layer.Rate) 99 | } 100 | } 101 | 102 | return loss / float64(outputLayer.Units) 103 | } 104 | 105 | // LearnsRaw is a shorcut to learn a raw dataset of inputs/outputs backed by LearnRaw method 106 | func (neural *Neural) LearnsRaw(dataset [][][]float64) float64 { 107 | neural.Loss = 0.0 108 | for _, data := range dataset { 109 | neural.Loss += neural.LearnRaw(data[0], data[1]) 110 | } 111 | neural.Loss /= float64(len(dataset)) 112 | return neural.Loss 113 | } 114 | 115 | // Learn arbitrary values by automatic conversion to raw values 116 | func (neural *Neural) Learn(inputs []float64, outputs []float64) float64 { 117 | return neural.LearnRaw(neural.InputValuesToRaw(inputs), neural.OutputValuesToRaw(outputs)) 118 | } 119 | 120 | // Learns is a shorcut to learn dataset of arbitrary inputs/outputs backed by Learn method 121 | func (neural *Neural) Learns(dataset [][][]float64) float64 { 122 | neural.Loss = 0.0 123 | for _, data := range dataset { 124 | neural.Loss += neural.Learn(data[0], data[1]) 125 | } 126 | neural.Loss /= float64(len(dataset)) 127 | return neural.Loss 128 | } 129 | 130 | // Clone neural with same layers 131 | func (neural *Neural) Clone() *Neural { 132 | layers := make([]*Layer, neural.MaxLayers) 133 | 134 | for i := 0; i < neural.MaxLayers; i++ { 135 | layers[i] = &Layer{ 136 | Inputs: neural.Layers[i].Inputs, 137 | Units: neural.Layers[i].Units, 138 | Activation: neural.Layers[i].Activation, 139 | Rate: neural.Layers[i].Rate, 140 | Momentum: neural.Layers[i].Momentum, 141 | } 142 | 143 | layers[i].Range = make([][]float64, len(neural.Layers[i].Range)) 144 | copy(layers[i].Range, neural.Layers[i].Range) 145 | } 146 | 147 | clone := NewNeural(layers) 148 | 149 | for i := 0; i < neural.MaxLayers; i++ { 150 | clone.Layers[i] = neural.Layers[i].Clone() 151 | } 152 | 153 | return clone 154 | } 155 | 156 | // Mutate neurons of all layers based on probability 157 | func (neural *Neural) Mutate(probability float64) { 158 | for i := 0; i < neural.MaxLayers; i++ { 159 | neural.Layers[i].Mutate(probability) 160 | } 161 | } 162 | 163 | // Crossover two neurals merging layers 164 | func (neural *Neural) Crossover(neuralB *Neural, dominant float64) *Neural { 165 | new := NewNeural([]*Layer{}) 166 | new.MaxLayers = neural.MaxLayers 167 | new.Layers = make([]*Layer, neural.MaxLayers) 168 | 169 | for i := 0; i < neural.MaxLayers; i++ { 170 | new.Layers[i] = neural.Layers[i].Crossover(neuralB.Layers[i], dominant) 171 | } 172 | 173 | return new 174 | } 175 | 176 | // Evolve uses Clone, Mutate, Learns and Crossover to create a evolutionary scenario 177 | func (neural *Neural) Evolve(evolve Evolve) *Neural { 178 | if evolve.Population == 0 { 179 | evolve.Population = 20 180 | } 181 | if evolve.Mutate == 0.0 { 182 | evolve.Mutate = 0.01 183 | } 184 | if evolve.Crossover == 0.0 { 185 | evolve.Crossover = 0.5 186 | } 187 | if evolve.Elitism == 0 { 188 | evolve.Elitism = 5 189 | } 190 | if evolve.Epochs == 0 { 191 | panic("need to set epochs in evolve") 192 | } 193 | if evolve.Iterations == 0 { 194 | evolve.Iterations = 1 195 | } 196 | 197 | population := make([]*Neural, evolve.Population) 198 | for p := 0; p < evolve.Population; p++ { 199 | population[p] = neural.Clone() 200 | } 201 | 202 | for e := 0; e < evolve.Epochs; e++ { 203 | for p := 0; p < evolve.Population; p++ { 204 | population[p].Mutate(evolve.Mutate) 205 | 206 | for i := 0; i < evolve.Iterations; i++ { 207 | population[p].Learns(evolve.Dataset) 208 | } 209 | } 210 | 211 | sort.Slice(population, func(a int, b int) bool { 212 | return population[a].Loss < population[b].Loss 213 | }) 214 | 215 | if evolve.Callback(e, population[0].Loss) == false { 216 | break 217 | } 218 | 219 | if population[0].Loss <= evolve.Threshold { 220 | break 221 | } 222 | 223 | if e == evolve.Epochs-1 { 224 | break 225 | } 226 | 227 | for p := 0; p < evolve.Population; p++ { 228 | if p < evolve.Elitism { 229 | randomIndex := randomInt(int64(evolve.Population)) 230 | children := population[p].Crossover(population[randomIndex], evolve.Crossover) 231 | 232 | population[evolve.Population-1-p] = children 233 | } 234 | } 235 | } 236 | 237 | return population[0] 238 | } 239 | 240 | // Reset neurons (weights, bias, etc) of all layers 241 | func (neural *Neural) Reset() { 242 | for i := 0; i < neural.MaxLayers; i++ { 243 | neural.Layers[i].Reset() 244 | } 245 | } 246 | 247 | // Rate set the rate for all layers 248 | func (neural *Neural) Rate(value float64) { 249 | for i := 0; i < neural.MaxLayers; i++ { 250 | neural.Layers[i].Rate = value 251 | } 252 | } 253 | 254 | // Momentum set the momentum for all layers 255 | func (neural *Neural) Momentum(value float64) { 256 | for i := 0; i < neural.MaxLayers; i++ { 257 | neural.Layers[i].Momentum = value 258 | } 259 | } 260 | 261 | // Export neural to json string 262 | func (neural *Neural) Export() ([]byte, error) { 263 | return json.Marshal(&neural) 264 | } 265 | 266 | // Import neural from json string 267 | func (neural *Neural) Import(encoded []byte) error { 268 | json.Unmarshal(encoded, &neural) 269 | 270 | neural.MaxLayers = len(neural.Layers) 271 | 272 | for _, layer := range neural.Layers { 273 | layer.Inputs = len(layer.Neurons[0].Weights) 274 | layer.Units = len(layer.Neurons) 275 | layer.SetActivation(layer.Activation) 276 | 277 | for _, neuron := range layer.Neurons { 278 | neuron.MaxInputs = len(neuron.Weights) 279 | neuron.Layer = layer 280 | neuron.Inputs = make([]float64, neuron.MaxInputs) 281 | } 282 | } 283 | 284 | return nil 285 | } 286 | 287 | // ToFile export neural to file 288 | func (neural *Neural) ToFile(filename string) error { 289 | encoded, err := neural.Export() 290 | if err != nil { 291 | return err 292 | } 293 | return ioutil.WriteFile(filename, encoded, 0644) 294 | } 295 | 296 | // FromFile import neural from file 297 | func (neural *Neural) FromFile(filename string) error { 298 | content, err := ioutil.ReadFile(filename) 299 | if err != nil { 300 | return err 301 | } 302 | return neural.Import(content) 303 | } 304 | 305 | // DeleteFile is a shortcut to delete a file 306 | func (neural *Neural) DeleteFile(filename string) error { 307 | return os.Remove(filename) 308 | } 309 | 310 | // InputValuesToRaw converts arbitrary input values to raw (using layer range property) 311 | func (neural *Neural) InputValuesToRaw(inputs []float64) []float64 { 312 | layer := neural.Layers[0] 313 | total := len(layer.Range) 314 | if total == 0 { 315 | return inputs 316 | } 317 | 318 | raw := make([]float64, total) 319 | for i, ranges := range layer.Range { 320 | raw[i] = rangeToRange(inputs[i], ranges[0], ranges[1], ranges[2], ranges[3]) 321 | } 322 | return raw 323 | } 324 | 325 | // OutputValuesToRaw converts arbitrary output values to raw (using layer range property) 326 | func (neural *Neural) OutputValuesToRaw(outputs []float64) []float64 { 327 | layer := neural.Layers[neural.MaxLayers-1] 328 | total := len(layer.Range) 329 | if total == 0 { 330 | return outputs 331 | } 332 | 333 | raw := make([]float64, total) 334 | for i, ranges := range layer.Range { 335 | raw[i] = rangeToRange(outputs[i], ranges[0], ranges[1], ranges[2], ranges[3]) 336 | } 337 | return raw 338 | } 339 | 340 | // OutputValuesFromRaw converts raw output to arbitrary output values (using layer range property) 341 | func (neural *Neural) OutputValuesFromRaw(outputs []float64) []float64 { 342 | layer := neural.Layers[neural.MaxLayers-1] 343 | total := len(layer.Range) 344 | if total == 0 { 345 | return outputs 346 | } 347 | 348 | values := make([]float64, total) 349 | for i, ranges := range layer.Range { 350 | values[i] = rangeToRange(outputs[i], ranges[2], ranges[3], ranges[0], ranges[1]) 351 | } 352 | return values 353 | } 354 | 355 | func rangeToRange(v float64, fMin float64, fMax float64, tMin float64, tMax float64) float64 { 356 | return (tMax-tMin)/(fMax-fMin)*(v-fMax) + tMax 357 | /* 358 | more efficient with pre alloc 359 | x = (tMax - tMin) / (fMax - fMin) 360 | y = x * fMax - tMax 361 | then just 362 | return v * x + y 363 | */ 364 | } 365 | -------------------------------------------------------------------------------- /neuron.go: -------------------------------------------------------------------------------- 1 | package neural 2 | 3 | import ( 4 | "crypto/rand" 5 | "math/big" 6 | ) 7 | 8 | // Neuron is a set of weights + bias linked to a layer 9 | type Neuron struct { 10 | MaxInputs int `json:"-"` 11 | Weights []float64 `json:"Weights"` 12 | Bias float64 `json:"Bias"` 13 | // Previous momentum of every weight and bias 14 | Momentums []float64 `json:"-"` 15 | // Layer to which neuron is linked 16 | Layer *Layer `json:"-"` 17 | activation float64 18 | delta float64 19 | error float64 20 | Inputs []float64 `json:"-"` 21 | } 22 | 23 | // NewNeuron creates a neuron linked to a layer 24 | func NewNeuron(Layer *Layer, MaxInputs int) *Neuron { 25 | neuron := &Neuron{ 26 | MaxInputs: MaxInputs, 27 | Weights: make([]float64, MaxInputs), 28 | Bias: randomFloat(-1.0, 1.0), 29 | Momentums: make([]float64, MaxInputs+1), 30 | Layer: Layer, 31 | Inputs: make([]float64, MaxInputs), 32 | } 33 | 34 | for i := 0; i < neuron.MaxInputs; i++ { 35 | neuron.Weights[i] = randomFloat(-1.0, 1.0) 36 | } 37 | 38 | return neuron 39 | } 40 | 41 | // Think process the neuron forward based on inputs 42 | func (neuron *Neuron) Think(inputs []float64) float64 { 43 | sum := neuron.Bias 44 | 45 | for i := 0; i < neuron.MaxInputs; i++ { 46 | sum += inputs[i] * neuron.Weights[i] 47 | neuron.Inputs[i] = inputs[i] 48 | } 49 | 50 | neuron.activation = neuron.Layer.Forward(sum) 51 | return neuron.activation 52 | } 53 | 54 | // Optimizer learning by momentum 55 | func (neuron *Neuron) Optimizer(index int, value float64) float64 { 56 | neuron.Momentums[index] = value + (neuron.Layer.Momentum * neuron.Momentums[index]) 57 | return neuron.Momentums[index] 58 | } 59 | 60 | // Clone neuron with same weights, bias, etc 61 | func (neuron *Neuron) Clone() *Neuron { 62 | clone := NewNeuron(neuron.Layer, neuron.MaxInputs) 63 | 64 | for i := 0; i < neuron.MaxInputs; i++ { 65 | clone.Weights[i] = neuron.Weights[i] 66 | } 67 | clone.Bias = neuron.Bias 68 | 69 | return clone 70 | } 71 | 72 | // Mutate randomizing weights/bias based on probability 73 | func (neuron *Neuron) Mutate(probability float64) { 74 | for i := 0; i < neuron.MaxInputs; i++ { 75 | if probability >= cryptoRandomFloat() { 76 | neuron.Weights[i] += randomFloat(-1.0, 1.0) 77 | neuron.Momentums[i] = 0.0 78 | } 79 | } 80 | 81 | if probability >= cryptoRandomFloat() { 82 | neuron.Bias += randomFloat(-1.0, 1.0) 83 | neuron.Momentums[neuron.MaxInputs] = 0.0 84 | } 85 | } 86 | 87 | // Crossover two neurons merging weights and bias 88 | func (neuron *Neuron) Crossover(neuronB Neuron, dominant float64) *Neuron { 89 | new := NewNeuron(neuron.Layer, neuron.MaxInputs) 90 | 91 | for i := 0; i < new.MaxInputs; i++ { 92 | if cryptoRandomFloat() >= 0.5 { 93 | new.Weights[i] = neuron.Weights[i] 94 | } else { 95 | new.Weights[i] = neuronB.Weights[i] 96 | } 97 | } 98 | 99 | if cryptoRandomFloat() >= 0.5 { 100 | new.Bias = neuron.Bias 101 | } else { 102 | new.Bias = neuronB.Bias 103 | } 104 | 105 | return new 106 | } 107 | 108 | // Reset weights, bias and momentums 109 | func (neuron *Neuron) Reset() { 110 | for i := 0; i < neuron.MaxInputs; i++ { 111 | neuron.Weights[i] = randomFloat(-1.0, 1.0) 112 | neuron.Momentums[i] = 0.0 113 | } 114 | 115 | neuron.Bias = randomFloat(-1.0, 1.0) 116 | neuron.Momentums[neuron.MaxInputs] = 0.0 117 | } 118 | 119 | func randomFloat(min float64, max float64) float64 { 120 | return min + cryptoRandomFloat()*(max-min) 121 | } 122 | 123 | func cryptoRandomFloat() float64 { 124 | num, err := rand.Int(rand.Reader, big.NewInt(1e17)) 125 | if err != nil { 126 | panic(err) 127 | } 128 | return float64(num.Int64()) / float64(1e17) 129 | } 130 | 131 | func randomInt(max int64) int { 132 | num, err := rand.Int(rand.Reader, big.NewInt(max)) 133 | if err != nil { 134 | panic(err) 135 | } 136 | return int(num.Int64()) 137 | } 138 | --------------------------------------------------------------------------------