├── .gitignore ├── LICENSE.md ├── README.md ├── main.py └── neural_network_structure.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # PyCharm 60 | .idea\ 61 | .idea/ 62 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Milo Spencer-Harper 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Multi Layer Neural Network 2 | A multi layer neural network written in Python 3, which can be trained to solve the XOR problem. It demonstrates back 3 | propagation using Sigmoid as the activation function. 4 | 5 | It is built from scratch without using a machine learning library. There are no dependencies except the mathematics 6 | library numpy. 7 | 8 | # Network Architecture 9 | ![alt text](./neural_network_structure.png) 10 | 11 | # Training Set 12 | The Output is 1 if either Input 1 or Input 2 are 1, but not both. Input 3 does not have any bearing on the Output. 13 | The challenge is to train a neural network to detect the pattern using this training set. 14 | 15 | | Input 1 | Input 2 | Input 3 | Output | 16 | |---------|---------|---------|--------| 17 | | 0 | 0 | 1 | 0 | 18 | | 0 | 1 | 1 | 1 | 19 | | 1 | 0 | 1 | 1 | 20 | | 0 | 1 | 0 | 1 | 21 | | 1 | 0 | 0 | 1 | 22 | | 1 | 1 | 1 | 0 | 23 | | 0 | 0 | 0 | 0 | 24 | 25 | # Usage 26 | ``` 27 | pip install numpy 28 | ``` 29 | ``` 30 | python3 main.py 31 | ``` 32 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from numpy import exp, array, random, dot 2 | 3 | 4 | class NeuronLayer(): 5 | def __init__(self, number_of_neurons, number_of_inputs_per_neuron): 6 | self.synaptic_weights = 2 * random.random((number_of_inputs_per_neuron, number_of_neurons)) - 1 7 | 8 | 9 | class NeuralNetwork(): 10 | def __init__(self, layer1, layer2): 11 | self.layer1 = layer1 12 | self.layer2 = layer2 13 | 14 | # The Sigmoid function, which describes an S shaped curve. 15 | # We pass the weighted sum of the inputs through this function to 16 | # normalise them between 0 and 1. 17 | def __sigmoid(self, x): 18 | return 1 / (1 + exp(-x)) 19 | 20 | # The derivative of the Sigmoid function. 21 | # This is the gradient of the Sigmoid curve. 22 | # It indicates how confident we are about the existing weight. 23 | def __sigmoid_derivative(self, x): 24 | return x * (1 - x) 25 | 26 | # We train the neural network through a process of trial and error. 27 | # Adjusting the synaptic weights each time. 28 | def train(self, training_set_inputs, training_set_outputs, number_of_training_iterations): 29 | for iteration in range(number_of_training_iterations): 30 | # Pass the training set through our neural network 31 | output_from_layer_1, output_from_layer_2 = self.think(training_set_inputs) 32 | 33 | # Calculate the error for layer 2 (The difference between the desired output 34 | # and the predicted output). 35 | layer2_error = training_set_outputs - output_from_layer_2 36 | layer2_delta = layer2_error * self.__sigmoid_derivative(output_from_layer_2) 37 | 38 | # Calculate the error for layer 1 (By looking at the weights in layer 1, 39 | # we can determine by how much layer 1 contributed to the error in layer 2). 40 | layer1_error = layer2_delta.dot(self.layer2.synaptic_weights.T) 41 | layer1_delta = layer1_error * self.__sigmoid_derivative(output_from_layer_1) 42 | 43 | # Calculate how much to adjust the weights by 44 | layer1_adjustment = training_set_inputs.T.dot(layer1_delta) 45 | layer2_adjustment = output_from_layer_1.T.dot(layer2_delta) 46 | 47 | # Adjust the weights. 48 | self.layer1.synaptic_weights += layer1_adjustment 49 | self.layer2.synaptic_weights += layer2_adjustment 50 | 51 | # The neural network thinks. 52 | def think(self, inputs): 53 | output_from_layer1 = self.__sigmoid(dot(inputs, self.layer1.synaptic_weights)) 54 | output_from_layer2 = self.__sigmoid(dot(output_from_layer1, self.layer2.synaptic_weights)) 55 | return output_from_layer1, output_from_layer2 56 | 57 | # The neural network prints its weights 58 | def print_weights(self): 59 | print(" Layer 1 (4 neurons, each with 3 inputs):") 60 | print(self.layer1.synaptic_weights) 61 | print(" Layer 2 (1 neuron, with 4 inputs):") 62 | print(self.layer2.synaptic_weights) 63 | 64 | if __name__ == "__main__": 65 | 66 | #Seed the random number generator 67 | random.seed(1) 68 | 69 | # Create layer 1 (4 neurons, each with 3 inputs) 70 | layer1 = NeuronLayer(4, 3) 71 | 72 | # Create layer 2 (a single neuron with 4 inputs) 73 | layer2 = NeuronLayer(1, 4) 74 | 75 | # Combine the layers to create a neural network 76 | neural_network = NeuralNetwork(layer1, layer2) 77 | 78 | print("Stage 1) Random starting synaptic weights: ") 79 | neural_network.print_weights() 80 | 81 | # The training set. We have 7 examples, each consisting of 3 input values 82 | # and 1 output value. 83 | training_set_inputs = array([[0, 0, 1], [0, 1, 1], [1, 0, 1], [0, 1, 0], [1, 0, 0], [1, 1, 1], [0, 0, 0]]) 84 | training_set_outputs = array([[0, 1, 1, 1, 1, 0, 0]]).T 85 | 86 | # Train the neural network using the training set. 87 | # Do it 60,000 times and make small adjustments each time. 88 | neural_network.train(training_set_inputs, training_set_outputs, 60000) 89 | 90 | print("Stage 2) New synaptic weights after training: ") 91 | neural_network.print_weights() 92 | 93 | # Test the neural network with a new situation. 94 | print("Stage 3) Considering a new situation [1, 1, 0] -> ?: ") 95 | hidden_state, output = neural_network.think(array([1, 1, 0])) 96 | print(output) 97 | -------------------------------------------------------------------------------- /neural_network_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miloharper/multi-layer-neural-network/5c00122d13062d153cb3c2946f18271fb49bc244/neural_network_structure.png --------------------------------------------------------------------------------