├── NeuralNetwork
├── NeuralNetwork
│ ├── data.txt
│ ├── App.config
│ ├── Network
│ │ ├── Sigmoid.cs
│ │ ├── Dataset.cs
│ │ ├── Synapse.cs
│ │ ├── Neuron.cs
│ │ └── Network.cs
│ ├── Properties
│ │ └── AssemblyInfo.cs
│ ├── NeuralNetwork.csproj
│ └── Program.cs
└── NeuralNetwork.sln
├── ScreenShot.PNG
├── .gitignore
└── README.md
/NeuralNetwork/NeuralNetwork/data.txt:
--------------------------------------------------------------------------------
1 | 2 3 1
2 | 1 0 1
3 | 0 1 1
4 | 1 1 0
5 | 0 0 0
--------------------------------------------------------------------------------
/ScreenShot.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/XenocodeRCE/Neural-Network/master/ScreenShot.PNG
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/App.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/Network/Sigmoid.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace NeuralNetwork.Network
4 | {
5 | public static class Sigmoid
6 | {
7 | public static double Output(double x)
8 | {
9 | return x < -45.0 ? 0.0 : x > 45.0 ? 1.0 : 1.0 / (1.0 + Math.Exp(-x));
10 | }
11 |
12 | public static double Derivative(double x)
13 | {
14 | return x * (1 - x);
15 | }
16 | }
17 | }
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/Network/Dataset.cs:
--------------------------------------------------------------------------------
1 | namespace NeuralNetwork.Network
2 | {
3 | public class DataSet
4 | {
5 | #region -- Properties --
6 | public double[] Values { get; set; }
7 | public double[] Targets { get; set; }
8 | #endregion
9 |
10 | #region -- Constructor --
11 | public DataSet(double[] values, double[] targets)
12 | {
13 | Values = values;
14 | Targets = targets;
15 | }
16 | #endregion
17 | }
18 | }
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/Network/Synapse.cs:
--------------------------------------------------------------------------------
1 | namespace NeuralNetwork.Network
2 | {
3 | public class Synapse
4 | {
5 | #region -- Properties --
6 | public Neuron InputNeuron { get; set; }
7 | public Neuron OutputNeuron { get; set; }
8 | public double Weight { get; set; }
9 | public double WeightDelta { get; set; }
10 | #endregion
11 |
12 | #region -- Constructor --
13 | public Synapse(Neuron inputNeuron, Neuron outputNeuron)
14 | {
15 | InputNeuron = inputNeuron;
16 | OutputNeuron = outputNeuron;
17 | Weight = Network.GetRandom();
18 | }
19 | #endregion
20 | }
21 | }
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork.sln:
--------------------------------------------------------------------------------
1 |
2 | Microsoft Visual Studio Solution File, Format Version 12.00
3 | # Visual Studio 2012
4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NeuralNetwork", "NeuralNetwork\NeuralNetwork.csproj", "{1B513876-712C-4699-94A0-6F199CE38BD2}"
5 | EndProject
6 | Global
7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution
8 | Debug|Any CPU = Debug|Any CPU
9 | Release|Any CPU = Release|Any CPU
10 | EndGlobalSection
11 | GlobalSection(ProjectConfigurationPlatforms) = postSolution
12 | {1B513876-712C-4699-94A0-6F199CE38BD2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
13 | {1B513876-712C-4699-94A0-6F199CE38BD2}.Debug|Any CPU.Build.0 = Debug|Any CPU
14 | {1B513876-712C-4699-94A0-6F199CE38BD2}.Release|Any CPU.ActiveCfg = Release|Any CPU
15 | {1B513876-712C-4699-94A0-6F199CE38BD2}.Release|Any CPU.Build.0 = Release|Any CPU
16 | EndGlobalSection
17 | GlobalSection(SolutionProperties) = preSolution
18 | HideSolutionNode = FALSE
19 | EndGlobalSection
20 | EndGlobal
21 |
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
1 | using System.Reflection;
2 | using System.Runtime.CompilerServices;
3 | using System.Runtime.InteropServices;
4 |
5 | // General Information about an assembly is controlled through the following
6 | // set of attributes. Change these attribute values to modify the information
7 | // associated with an assembly.
8 | [assembly: AssemblyTitle("NeuralNetwork")]
9 | [assembly: AssemblyDescription("")]
10 | [assembly: AssemblyConfiguration("")]
11 | [assembly: AssemblyCompany("")]
12 | [assembly: AssemblyProduct("NeuralNetwork")]
13 | [assembly: AssemblyCopyright("Copyright © 2015")]
14 | [assembly: AssemblyTrademark("")]
15 | [assembly: AssemblyCulture("")]
16 |
17 | // Setting ComVisible to false makes the types in this assembly not visible
18 | // to COM components. If you need to access a type in this assembly from
19 | // COM, set the ComVisible attribute to true on that type.
20 | [assembly: ComVisible(false)]
21 |
22 | // The following GUID is for the ID of the typelib if this project is exposed to COM
23 | [assembly: Guid("b68eb954-9305-4573-8fa1-16a25c2b6e7f")]
24 |
25 | // Version information for an assembly consists of the following four values:
26 | //
27 | // Major Version
28 | // Minor Version
29 | // Build Number
30 | // Revision
31 | //
32 | // You can specify all the values or you can default the Build and Revision Numbers
33 | // by using the '*' as shown below:
34 | // [assembly: AssemblyVersion("1.0.*")]
35 | [assembly: AssemblyVersion("1.0.0.0")]
36 | [assembly: AssemblyFileVersion("1.0.0.0")]
37 |
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/Network/Neuron.cs:
--------------------------------------------------------------------------------
1 | using System.Collections.Generic;
2 | using System.Linq;
3 |
4 | namespace NeuralNetwork.Network
5 | {
6 | public class Neuron
7 | {
8 | #region -- Properties --
9 | public List InputSynapses { get; set; }
10 | public List OutputSynapses { get; set; }
11 | public double Bias { get; set; }
12 | public double BiasDelta { get; set; }
13 | public double Gradient { get; set; }
14 | public double Value { get; set; }
15 | #endregion
16 |
17 | #region -- Constructors --
18 | public Neuron()
19 | {
20 | InputSynapses = new List();
21 | OutputSynapses = new List();
22 | Bias = Network.GetRandom();
23 | }
24 |
25 | public Neuron(IEnumerable inputNeurons) : this()
26 | {
27 | foreach (var inputNeuron in inputNeurons)
28 | {
29 | var synapse = new Synapse(inputNeuron, this);
30 | inputNeuron.OutputSynapses.Add(synapse);
31 | InputSynapses.Add(synapse);
32 | }
33 | }
34 | #endregion
35 |
36 | #region -- Values & Weights --
37 | public virtual double CalculateValue()
38 | {
39 | return Value = Sigmoid.Output(InputSynapses.Sum(a => a.Weight * a.InputNeuron.Value) + Bias);
40 | }
41 |
42 | public double CalculateError(double target)
43 | {
44 | return target - Value;
45 | }
46 |
47 | public double CalculateGradient(double? target = null)
48 | {
49 | if(target == null)
50 | return Gradient = OutputSynapses.Sum(a => a.OutputNeuron.Gradient * a.Weight) * Sigmoid.Derivative(Value);
51 |
52 | return Gradient = CalculateError(target.Value) * Sigmoid.Derivative(Value);
53 | }
54 |
55 | public void UpdateWeights(double learnRate, double momentum)
56 | {
57 | var prevDelta = BiasDelta;
58 | BiasDelta = learnRate * Gradient;
59 | Bias += BiasDelta + momentum * prevDelta;
60 |
61 | foreach (var synapse in InputSynapses)
62 | {
63 | prevDelta = synapse.WeightDelta;
64 | synapse.WeightDelta = learnRate * Gradient * synapse.InputNeuron.Value;
65 | synapse.Weight += synapse.WeightDelta + momentum * prevDelta;
66 | }
67 | }
68 | #endregion
69 | }
70 | }
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/NeuralNetwork.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Debug
6 | AnyCPU
7 | {1B513876-712C-4699-94A0-6F199CE38BD2}
8 | Exe
9 | Properties
10 | NeuralNetwork
11 | NeuralNetwork
12 | v4.5
13 | 512
14 |
15 |
16 | AnyCPU
17 | true
18 | full
19 | false
20 | bin\Debug\
21 | DEBUG;TRACE
22 | prompt
23 | 4
24 |
25 |
26 | AnyCPU
27 | pdbonly
28 | true
29 | bin\Release\
30 | TRACE
31 | prompt
32 | 4
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 | Always
58 |
59 |
60 |
61 |
68 |
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/Network/Network.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 |
5 | namespace NeuralNetwork.Network
6 | {
7 | public class Network
8 | {
9 | #region -- Properties --
10 | public double LearnRate { get; set; }
11 | public double Momentum { get; set; }
12 | public List InputLayer { get; set; }
13 | public List HiddenLayer { get; set; }
14 | public List OutputLayer { get; set; }
15 | #endregion
16 |
17 | #region -- Globals --
18 | private static readonly Random Random = new Random();
19 | #endregion
20 |
21 | #region -- Constructor --
22 | public Network(int inputSize, int hiddenSize, int outputSize, double? learnRate = null, double? momentum = null)
23 | {
24 | LearnRate = learnRate ?? .4;
25 | Momentum = momentum ?? .9;
26 | InputLayer = new List();
27 | HiddenLayer = new List();
28 | OutputLayer = new List();
29 |
30 | for (var i = 0; i < inputSize; i++)
31 | InputLayer.Add(new Neuron());
32 |
33 | for (var i = 0; i < hiddenSize; i++)
34 | HiddenLayer.Add(new Neuron(InputLayer));
35 |
36 | for (var i = 0; i < outputSize; i++)
37 | OutputLayer.Add(new Neuron(HiddenLayer));
38 | }
39 | #endregion
40 |
41 | #region -- Training --
42 | public void Train(List dataSets, int numEpochs)
43 | {
44 | for (var i = 0; i < numEpochs; i++)
45 | {
46 | foreach (var dataSet in dataSets)
47 | {
48 | ForwardPropagate(dataSet.Values);
49 | BackPropagate(dataSet.Targets);
50 | }
51 | }
52 | }
53 |
54 | public void Train(List dataSets, double minimumError)
55 | {
56 | var error = 1.0;
57 | var numEpochs = 0;
58 |
59 | while (error > minimumError && numEpochs < int.MaxValue)
60 | {
61 | var errors = new List();
62 | foreach (var dataSet in dataSets)
63 | {
64 | ForwardPropagate(dataSet.Values);
65 | BackPropagate(dataSet.Targets);
66 | errors.Add(CalculateError(dataSet.Targets));
67 | }
68 | error = errors.Average();
69 | numEpochs++;
70 | }
71 | }
72 |
73 | private void ForwardPropagate(params double[] inputs)
74 | {
75 | var i = 0;
76 | InputLayer.ForEach(a => a.Value = inputs[i++]);
77 | HiddenLayer.ForEach(a => a.CalculateValue());
78 | OutputLayer.ForEach(a => a.CalculateValue());
79 | }
80 |
81 | private void BackPropagate(params double[] targets)
82 | {
83 | var i = 0;
84 | OutputLayer.ForEach(a => a.CalculateGradient(targets[i++]));
85 | HiddenLayer.ForEach(a => a.CalculateGradient());
86 | HiddenLayer.ForEach(a => a.UpdateWeights(LearnRate, Momentum));
87 | OutputLayer.ForEach(a => a.UpdateWeights(LearnRate, Momentum));
88 | }
89 |
90 | public double[] Compute(params double[] inputs)
91 | {
92 | ForwardPropagate(inputs);
93 | return OutputLayer.Select(a => a.Value).ToArray();
94 | }
95 |
96 | private double CalculateError(params double[] targets)
97 | {
98 | var i = 0;
99 | return OutputLayer.Sum(a => Math.Abs(a.CalculateError(targets[i++])));
100 | }
101 | #endregion
102 |
103 | #region -- Helpers --
104 | public static double GetRandom()
105 | {
106 | return 2 * Random.NextDouble() - 1;
107 | }
108 | #endregion
109 | }
110 |
111 | #region -- Enum --
112 | public enum TrainingType
113 | {
114 | Epoch,
115 | MinimumError
116 | }
117 | #endregion
118 | }
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ## Ignore Visual Studio temporary files, build results, and
2 | ## files generated by popular Visual Studio add-ons.
3 |
4 | # User-specific files
5 | *.suo
6 | *.user
7 | *.userosscache
8 | *.sln.docstates
9 |
10 | # User-specific files (MonoDevelop/Xamarin Studio)
11 | *.userprefs
12 |
13 | # Build results
14 | [Dd]ebug/
15 | [Dd]ebugPublic/
16 | [Rr]elease/
17 | [Rr]eleases/
18 | x64/
19 | x86/
20 | build/
21 | bld/
22 | [Bb]in/
23 | [Oo]bj/
24 |
25 | # Visual Studo 2015 cache/options directory
26 | .vs/
27 |
28 | # MSTest test Results
29 | [Tt]est[Rr]esult*/
30 | [Bb]uild[Ll]og.*
31 |
32 | # NUNIT
33 | *.VisualState.xml
34 | TestResult.xml
35 |
36 | # Build Results of an ATL Project
37 | [Dd]ebugPS/
38 | [Rr]eleasePS/
39 | dlldata.c
40 |
41 | *_i.c
42 | *_p.c
43 | *_i.h
44 | *.ilk
45 | *.meta
46 | *.obj
47 | *.pch
48 | *.pdb
49 | *.pgc
50 | *.pgd
51 | *.rsp
52 | *.sbr
53 | *.tlb
54 | *.tli
55 | *.tlh
56 | *.tmp
57 | *.tmp_proj
58 | *.log
59 | *.vspscc
60 | *.vssscc
61 | .builds
62 | *.pidb
63 | *.svclog
64 | *.scc
65 |
66 | # Chutzpah Test files
67 | _Chutzpah*
68 |
69 | # Visual C++ cache files
70 | ipch/
71 | *.aps
72 | *.ncb
73 | *.opensdf
74 | *.sdf
75 | *.cachefile
76 |
77 | # Visual Studio profiler
78 | *.psess
79 | *.vsp
80 | *.vspx
81 |
82 | # TFS 2012 Local Workspace
83 | $tf/
84 |
85 | # Guidance Automation Toolkit
86 | *.gpState
87 |
88 | # ReSharper is a .NET coding add-in
89 | _ReSharper*/
90 | *.[Rr]e[Ss]harper
91 | *.DotSettings.user
92 |
93 | # JustCode is a .NET coding addin-in
94 | .JustCode
95 |
96 | # TeamCity is a build add-in
97 | _TeamCity*
98 |
99 | # DotCover is a Code Coverage Tool
100 | *.dotCover
101 |
102 | # NCrunch
103 | _NCrunch_*
104 | .*crunch*.local.xml
105 |
106 | # MightyMoose
107 | *.mm.*
108 | AutoTest.Net/
109 |
110 | # Web workbench (sass)
111 | .sass-cache/
112 |
113 | # Installshield output folder
114 | [Ee]xpress/
115 |
116 | # DocProject is a documentation generator add-in
117 | DocProject/buildhelp/
118 | DocProject/Help/*.HxT
119 | DocProject/Help/*.HxC
120 | DocProject/Help/*.hhc
121 | DocProject/Help/*.hhk
122 | DocProject/Help/*.hhp
123 | DocProject/Help/Html2
124 | DocProject/Help/html
125 |
126 | # Click-Once directory
127 | publish/
128 |
129 | # Publish Web Output
130 | *.[Pp]ublish.xml
131 | *.azurePubxml
132 | # TODO: Comment the next line if you want to checkin your web deploy settings
133 | # but database connection strings (with potential passwords) will be unencrypted
134 | *.pubxml
135 | *.publishproj
136 |
137 | # NuGet Packages
138 | *.nupkg
139 | # The packages folder can be ignored because of Package Restore
140 | **/packages/*
141 | # except build/, which is used as an MSBuild target.
142 | !**/packages/build/
143 | # Uncomment if necessary however generally it will be regenerated when needed
144 | #!**/packages/repositories.config
145 |
146 | # Windows Azure Build Output
147 | csx/
148 | *.build.csdef
149 |
150 | # Windows Store app package directory
151 | AppPackages/
152 |
153 | # Others
154 | *.[Cc]ache
155 | ClientBin/
156 | [Ss]tyle[Cc]op.*
157 | ~$*
158 | *~
159 | *.dbmdl
160 | *.dbproj.schemaview
161 | *.pfx
162 | *.publishsettings
163 | node_modules/
164 | bower_components/
165 |
166 | # RIA/Silverlight projects
167 | Generated_Code/
168 |
169 | # Backup & report files from converting an old project file
170 | # to a newer Visual Studio version. Backup files are not needed,
171 | # because we have git ;-)
172 | _UpgradeReport_Files/
173 | Backup*/
174 | UpgradeLog*.XML
175 | UpgradeLog*.htm
176 |
177 | # SQL Server files
178 | *.mdf
179 | *.ldf
180 |
181 | # Business Intelligence projects
182 | *.rdl.data
183 | *.bim.layout
184 | *.bim_*.settings
185 |
186 | # Microsoft Fakes
187 | FakesAssemblies/
188 |
189 | # Node.js Tools for Visual Studio
190 | .ntvs_analysis.dat
191 |
192 | # Visual Studio 6 build log
193 | *.plg
194 |
195 | # Visual Studio 6 workspace options file
196 | *.opt
197 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Neural Networks
2 |
3 | Introduction
4 | ------------
5 | If this is your first foray into Neural Networks, welcome! I hope you enjoy yourself as much as I have.
6 |
7 | This project is my first attempt at creating an application that allows for quick interactions with a basic neural network. I find that I learn better when I have something quick and easy to tinker with that allows me to analyze pieces of a concept at my own pace. Hopefully, this will help some of you that learn in a similar manner.
8 |
9 | This project is written in C#.
10 |
11 | 
12 | What is a Neural Network?
13 | -----
14 | Great question!
15 |
16 | A Neural Network can be thought of as a series of nodes (or neurons) that are interconnected, much like they are in the brain. The network can have any number (N) of inputs and any number (M) of outputs. In between the inputs and outputs are a series of "hidden" neurons that make up the hidden layers of the network. These hidden layers provide the meat of the network and allow for some of the neat functionalities we can get out of a Neural Network.
17 |
18 | What are the Parts of a Neural Network?
19 | --------
20 | Before explaining the pieces of a neural network, it might be helpful to start with an example.
21 |
22 | Building off of [this excellent article from 2013](http://www.codingvision.net/miscellaneous/c-backpropagation-tutorial-xor), let's use the concept of Exclusive Or (XOR). XOR will output true when the inputs differ:
23 |
24 | | Input A | Input B | Output |
25 | |:-----:|:---------:|:-----:|
26 | | false | false | false |
27 | | false | true | true |
28 | | true | false | true |
29 | | true | true | false |
30 |
31 | Considering this, let's break down a Neural Network into its three basic parts:
32 |
33 | 1. The Inputs
34 | * These are the inputs into the Neural Network. From the XOR example above, the inputs would be Input A and Input B.
35 | * Each input can be considered a Neuron whose output is the initial input value.
36 | 2. The Hidden Layers
37 | * This is the meat of the Neural Network. This is where the magic happens. The Neurons in this layer are assigned weights for each of their inputs. These weights start off fairly random, but as the network is "trained" (discussed below), the weights are adjusted in order to make the neuron's output, and therefore the Neural Network's output closer to the expected result.
38 | 3. The Outputs
39 | * These are the outputs from the system. From the XOR example above, the output from the system would be either 'true' or 'false'. In the Neural Network, the Outputs are the last line of Neurons. These Neurons are also assigned a weight for each of their inputs and are "fed" by the Neurons in the hidden layer.
40 |
41 | Using the XOR example, if we were to give our Neural Network the inputs 'true' and 'false' we would expect the system to return 'true'.
42 |
43 | How Does it Work?
44 | -------
45 | Because I love examples, here's another:
46 |
47 | | Input A | Input B | Input C | Output |
48 | |:-----:|:---------:|:-----:|:------|
49 | | true | false | false | true |
50 | | false | true | true | false |
51 | | true | fase | true | false |
52 | | true | true | true | true |
53 |
54 | In the above table, we can infer the following patterns:
55 |
56 | 1. The output is true if the number of inputs set to true is odd OR the number of inputs set to false is even.
57 | 2. The output is false if the number of inputs set to true is even OR the number of inputs set to false is odd.
58 |
59 | The job of the Neural Network is to try and figure out that pattern. It does this via training.
60 |
61 | #### How Do We Train the Neural Network?
62 |
63 | Training the Neural Network is accomplished by giving it a set of input data and the expected results for those inputs. This data is then continuously run through the Neural Network until we can be reasonably sure that it has a grasp of the patterns present in that data.
64 |
65 | In this project, the Neural Network is trained via the following common Neural Network training methods:
66 |
67 | 1. Back-Propagation
68 | * After each set of inputs is run through the system and an output generated, that output is validated against the expected output.
69 | * The percentage of error that results is then propagated backwards (hence the name) through the Hidden Layers of the Neural Network. This adjusts the weights assigned to each of a neuron's inputs in the Hidden Layers.
70 | * Ideally, each Back-Propagation will bring the Neural Network's output closer to the expected output of the provided inputs.
71 | 2. Biases
72 | * Biases allow us to modify our activation function (discussed below) in order to generate a better output for each neuron.
73 | * [See Here](http://stackoverflow.com/questions/2480650/role-of-bias-in-neural-networks) for an excellent explanation as to what a bias does for a Neural Network.
74 | 3. Momentum
75 | * Used to prevent the system from converging to a local minimum.
76 | * [See Here](https://en.wikibooks.org/wiki/Artificial_Neural_Networks/Neural_Network_Basics#Momentum)
77 | 4. Learning Rate
78 | * This will change the overall learning speed of the system.
79 | * [See Here](https://en.wikibooks.org/wiki/Artificial_Neural_Networks/Neural_Network_Basics#Learning_Rate)
80 |
81 | #### What defines a Neuron's Output?
82 |
83 | A Neuron's output is defined by an [Activation Function](https://en.wikipedia.org/wiki/Activation_function).
84 |
85 | In our case, we are using a [Sigmoid Function](https://en.wikipedia.org/wiki/Sigmoid_function) to define each Neuron's output. The Sigmoid function will convert any value to a value between 0 and 1. In the Neural Network, the Sigmoid functions will be used to generate initial weights and to help update percent errors.
86 |
87 | How Do I Use this Program?
88 | ------
89 |
90 | This program is fairly simple to use.
91 |
92 | When it starts, you will be presented with the option to load the Neural Network configuration from a data file. The file is currently set up to use the common XOR example. This means that the network will be initialized with the following:
93 | * Number of Inputs: 2
94 | * Number of Hidden Neurons: 10
95 | * Number of Outputs: 1
96 |
97 | Test Data:
98 |
99 | | Input A | Input B | Output |
100 | |:-----:|:---------:|:-----:|
101 | | 0 | 0 | 0 |
102 | | 0 | 1 | 1 |
103 | | 1 | 0 | 1 |
104 | | 1 | 1 | 0 |
105 |
106 | To change the file (data.txt):
107 |
108 | 1. Make the first line as follows: {{number of inputs}} {{number of hidden neurons}} {{number of outputs}}
109 | 2. Add any number of lines representing your data. Simply put in your inputs (space delmited) followed by your expected outputs (space delimited).
110 |
111 | Alternatively, you can manually type in your own manual configuration and data sets. There is a fair amount of validation in place to guide you through the process.
112 |
113 | Once the data sets are...set... the program will begin training the Neural Network. The network can be trained by number of epochs or by setting a minimum error. All of this can be set via the constants in the program. If minimum error is used, the network will be trained until the error threshold is reached. If the error threshold can't be reached, the training will stop when the number of epochs run reaches the maximum integer value.
114 |
115 | After the Neural Network is trained, you will be asked to enter input data and verify the output. If the output is wrong, you can enter the correct output. The network will then be "retrained". If the input is correct, the network will be "encouraged".
116 |
117 | If you used the XOR example and gave inputs of 0 and 1, you might see output values like: 0.985902....
118 |
119 | This is the Neural Network attempting to get as close as possible to the expected output of 1. As the network is encouraged, this number will get higher.
120 |
121 | What's Next?
122 | -----
123 |
124 | I'm going to continue learning more about Neural Networks. My understanding of Neural Networks is FAR from complete and I still feel unknowledgeable in many areas. As I learn about new methods and research, I will consider adding them either to the network or as a sidenote in the readme.
125 |
126 | I have another repository set up to use this same network to learn image recognition. [That can be found here.](http://github.com/trentsartain/Neural-Network-Image-Recognition)
127 |
128 | Code Considerations
129 | ---
130 | Feel free to reuse any part of this program.
131 |
132 | #### Reusability
133 | The Network and its supporting classes are self-contained, meaning that the "UI" portion of the program only serves to gather the necessary information to instantiate the Network class and its supporting classes. You could theoretically take the Network and supporting classes and bring it into your own application with little to no modification. The network only requires the number of inputs, number of hidden neurons, the number of outputs and the max number of epochs to be instantiated.
134 |
135 | #### How do I Exit?
136 | I forgot that people sometimes want to close their applications via the command line. When the program starts asking you to verify outputs, you can type "exit" to quit. I'm a bit tired and will implement a better way to do this in the future.
137 |
138 | #### You Code Funny...
139 | Hopefully my code is readable and and reusable for you. I put a lot of effort into maintaining best practices and keeping true to the rules of "Low Coupling and High Cohesion". It's a learning process and I welcome critique.
140 |
141 | Resources
142 | -----
143 |
144 | I used a few resources while building this project. I'm super thankful for those who have done a lot of work previously.
145 |
146 | [I am Trask - A Neural Network in 11 Lines of Python](http://iamtrask.github.io/2015/07/12/basic-python-network/) - This piqued my intrest in Neural Networks when it popped up on Reddit recently.
147 |
148 | [The Nature of Code - Chapter 10: Neural Networks](http://natureofcode.com/book/chapter-10-neural-networks/) - This was often able to answer some of my questions and made for a great read.
149 |
150 | [C# Backpropagation Tutorial](http://www.codingvision.net/miscellaneous/c-backpropagation-tutorial-xor) - This was the initial C# project I looked at. I took and modified a few elements that I really liked such as the Sigmoid and Neuron classes.
151 |
152 | [A Step by Step Backpropagation Example](http://mattmazur.com/2015/03/17/a-step-by-step-backpropagation-example/) - This was an excellent explanation of Back-Propagation and helped me tremendously with some of the math involved.
153 |
154 | [Coding Neural Network Back-Propagation Using C#](https://visualstudiomagazine.com/Articles/2015/04/01/Back-Propagation-Using-C.aspx?Page=1) - This was another great C# example. Dr. James McCaffrey (the author) has a lot of great insights in this article and others that he has written on the subject.
155 |
156 | [Simple C# Artificial Neural Network](http://www.craigsprogramming.com/2014/01/simple-c-artificial-neural-network.html) - This article played a large role in the November 2015 refactoring. This convinced me to get rid of the layer class altogether and helped clean up the network training code.
157 |
--------------------------------------------------------------------------------
/NeuralNetwork/NeuralNetwork/Program.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.IO;
4 | using System.Linq;
5 | using NeuralNetwork.Network;
6 |
7 | namespace NeuralNetwork
8 | {
9 | internal class Program
10 | {
11 | #region -- Constants --
12 | private const int MaxEpochs = 5000;
13 | private const double MinimumError = 0.01;
14 | private const TrainingType TrainingType = Network.TrainingType.MinimumError;
15 | #endregion
16 |
17 | #region -- Variables --
18 | private static int _numInputParameters;
19 | private static int _numHiddenLayerNeurons;
20 | private static int _numOutputParameters;
21 | private static Network.Network _network;
22 | private static List _dataSets;
23 | #endregion
24 |
25 | #region -- Main --
26 | private static void Main()
27 | {
28 | Greet();
29 | SetupNetwork();
30 | TrainNetwork();
31 | VerifyTraining();
32 | }
33 | #endregion
34 |
35 | #region -- Network Training --
36 | private static void TrainNetwork()
37 | {
38 | PrintNewLine();
39 | PrintUnderline(50);
40 | Console.WriteLine("Training...");
41 |
42 | Train();
43 |
44 | PrintNewLine();
45 | Console.WriteLine("Training Complete!");
46 | PrintNewLine();
47 | }
48 |
49 | private static void VerifyTraining()
50 | {
51 | Console.WriteLine("Let's test it!");
52 | PrintNewLine();
53 |
54 | while (true)
55 | {
56 | PrintUnderline(50);
57 | var values = GetInputData(string.Format("Type {0} inputs: ", _numInputParameters));
58 | var results = _network.Compute(values);
59 | PrintNewLine();
60 |
61 | foreach (var result in results)
62 | {
63 | Console.WriteLine("Output: {0}", result);
64 | }
65 |
66 | PrintNewLine();
67 |
68 | var convertedResults = new double[results.Length];
69 | for (var i = 0; i < results.Length; i++) { convertedResults[i] = results[i] > 0.5 ? 1 : 0; }
70 |
71 | var message = string.Format("Was the result supposed to be {0}? (yes/no/exit)", String.Join(" ", convertedResults));
72 | if (!GetBool(message))
73 | {
74 | var offendingDataSet = _dataSets.FirstOrDefault(x => x.Values.SequenceEqual(values) && x.Targets.SequenceEqual(convertedResults));
75 | _dataSets.Remove(offendingDataSet);
76 |
77 | var expectedResults = GetExpectedResult("What were the expected results?");
78 | if(!_dataSets.Exists(x => x.Values.SequenceEqual(values) && x.Targets.SequenceEqual(expectedResults)))
79 | _dataSets.Add(new DataSet(values, expectedResults));
80 |
81 | PrintNewLine();
82 | Console.WriteLine("Retraining Network...");
83 | PrintNewLine();
84 |
85 | Train();
86 | }
87 | else
88 | {
89 | PrintNewLine();
90 | Console.WriteLine("Neat!");
91 | Console.WriteLine("Encouraging Network...");
92 | PrintNewLine();
93 |
94 | Train();
95 | }
96 | }
97 | }
98 |
99 | private static void Train()
100 | {
101 | _network.Train(_dataSets, TrainingType == TrainingType.Epoch ? MaxEpochs : MinimumError);
102 | }
103 | #endregion
104 |
105 | #region -- Network Setup --
106 | private static void Greet()
107 | {
108 | Console.WriteLine("We're going to create an artificial Neural Network!");
109 | Console.WriteLine("The network will use back propagation to train itself.");
110 | PrintUnderline(50);
111 | PrintNewLine();
112 | }
113 |
114 | private static void SetupNetwork()
115 | {
116 | if (GetBool("Do you want to read from the space delimited data.txt file? (yes/no/exit)"))
117 | {
118 | SetupFromFile();
119 | }
120 | else
121 | {
122 | SetNumInputParameters();
123 | SetNumNeuronsInHiddenLayer();
124 | SetNumOutputParameters();
125 | GetTrainingData();
126 | }
127 |
128 | Console.WriteLine("Creating Network...");
129 | _network = new Network.Network(_numInputParameters, _numHiddenLayerNeurons, _numOutputParameters);
130 | PrintNewLine();
131 | }
132 |
133 | private static void SetNumInputParameters()
134 | {
135 | PrintNewLine();
136 | Console.WriteLine("How many input parameters will there be? (2 or more)");
137 | _numInputParameters = GetInput("Input Parameters: ", 2);
138 | PrintNewLine(2);
139 | }
140 |
141 | private static void SetNumNeuronsInHiddenLayer()
142 | {
143 | Console.WriteLine("How many neurons in the hidden layer? (2 or more)");
144 | _numHiddenLayerNeurons = GetInput("Neurons: ", 2);
145 | PrintNewLine(2);
146 | }
147 |
148 | private static void SetNumOutputParameters()
149 | {
150 | Console.WriteLine("How many output parameters will there be? (1 or more)");
151 | _numOutputParameters = GetInput("Output Parameters: ", 1);
152 | PrintNewLine(2);
153 | }
154 |
155 | private static void GetTrainingData()
156 | {
157 | PrintUnderline(50);
158 | Console.WriteLine("Now, we need some input data.");
159 | PrintNewLine();
160 |
161 | _dataSets = new List();
162 | for (var i = 0; i < 4; i++)
163 | {
164 | var values = GetInputData(String.Format("Data Set {0}", i + 1));
165 | var expectedResult = GetExpectedResult(String.Format("Expected Result for Data Set {0}:", i + 1));
166 | _dataSets.Add(new DataSet(values, expectedResult));
167 | }
168 | }
169 |
170 | private static double[] GetInputData(string message)
171 | {
172 | Console.WriteLine(message);
173 | var line = GetLine();
174 |
175 | while (line == null || line.Split(' ').Count() != _numInputParameters)
176 | {
177 | Console.WriteLine("{0} inputs are required.", _numInputParameters);
178 | PrintNewLine();
179 | Console.WriteLine(message);
180 | line = GetLine();
181 | }
182 |
183 | var values = new double[_numInputParameters];
184 | var lineNums = line.Split(' ');
185 | for(var i = 0; i < lineNums.Length; i++)
186 | {
187 | double num;
188 | if (Double.TryParse(lineNums[i], out num))
189 | {
190 | values[i] = num;
191 | }
192 | else
193 | {
194 | Console.WriteLine("You entered an invalid number. Try again");
195 | PrintNewLine(2);
196 | return GetInputData(message);
197 | }
198 | }
199 |
200 | return values;
201 | }
202 |
203 | private static double[] GetExpectedResult(string message)
204 | {
205 | Console.WriteLine(message);
206 | var line = GetLine();
207 |
208 | while (line == null || line.Split(' ').Count() != _numOutputParameters)
209 | {
210 | Console.WriteLine("{0} outputs are required.", _numOutputParameters);
211 | PrintNewLine();
212 | Console.WriteLine(message);
213 | line = GetLine();
214 | }
215 |
216 | var values = new double[_numOutputParameters];
217 | var lineNums = line.Split(' ');
218 | for (var i = 0; i < lineNums.Length; i++)
219 | {
220 | int num;
221 | if (int.TryParse(lineNums[i], out num) && (num == 0 || num == 1))
222 | {
223 | values[i] = num;
224 | }
225 | else
226 | {
227 | Console.WriteLine("You must enter 1s and 0s!");
228 | PrintNewLine(2);
229 | return GetExpectedResult(message);
230 | }
231 | }
232 |
233 | return values;
234 | }
235 | #endregion
236 |
237 | #region -- I/O Help --
238 | private static void SetupFromFile()
239 | {
240 | _dataSets = new List();
241 | var fileContent = File.ReadAllText("data.txt");
242 | var lines = fileContent.Split(new []{Environment.NewLine}, StringSplitOptions.RemoveEmptyEntries);
243 |
244 | if (lines.Length < 2)
245 | {
246 | WriteError("There aren't enough lines in the file. The first line should have 3 integers representing the number of inputs, the number of hidden neurons and the number of outputs." +
247 | "\r\nThere should also be at least one line of data.");
248 | }
249 | else
250 | {
251 | var setupParameters = lines[0].Split(' ');
252 | if (setupParameters.Length != 3)
253 | WriteError("There aren't enough setup parameters.");
254 |
255 | if (!int.TryParse(setupParameters[0], out _numInputParameters) || !int.TryParse(setupParameters[1], out _numHiddenLayerNeurons) || !int.TryParse(setupParameters[2], out _numOutputParameters))
256 | WriteError("The setup parameters are malformed. There must be 3 integers.");
257 |
258 | if (_numInputParameters < 2)
259 | WriteError("The number of input parameters must be greater than or equal to 2.");
260 |
261 | if (_numHiddenLayerNeurons < 2)
262 | WriteError("The number of hidden neurons must be greater than or equal to 2.");
263 |
264 | if (_numOutputParameters < 1)
265 | WriteError("The number of hidden neurons must be greater than or equal to 1.");
266 | }
267 |
268 | for (var lineIndex = 1; lineIndex < lines.Length; lineIndex++)
269 | {
270 | var items = lines[lineIndex].Split(' ');
271 | if (items.Length != _numInputParameters + _numOutputParameters)
272 | WriteError(String.Format("The data file is malformed. There were {0} elements on line {1} instead of {2}", items.Length, lineIndex + 1, _numInputParameters + _numOutputParameters));
273 |
274 | var values = new double[_numInputParameters];
275 | for (var i = 0; i < _numInputParameters; i++)
276 | {
277 | double num;
278 | if (!double.TryParse(items[i], out num))
279 | WriteError(String.Format("The data file is malformed. On line {0}, input parameter {1} is not a valid number.", lineIndex + 1, items[i]));
280 | else
281 | values[i] = num;
282 | }
283 |
284 | var expectedResults = new double[_numOutputParameters];
285 | for (var i = 0; i < _numOutputParameters; i++)
286 | {
287 | int num;
288 | if (!int.TryParse(items[_numInputParameters + i], out num))
289 | Console.WriteLine("The data file is malformed. On line {0}, output paramater {1} is not a valid number.", lineIndex, items[i]);
290 | else
291 | expectedResults[i] = num;
292 | }
293 | _dataSets.Add(new DataSet(values, expectedResults));
294 | }
295 | }
296 | #endregion
297 |
298 | #region -- Console Helpers --
299 |
300 | private static string GetLine()
301 | {
302 | var line = Console.ReadLine();
303 | return line == null ? string.Empty : line.Trim();
304 | }
305 |
306 | private static int GetInput(string message, int min)
307 | {
308 | Console.Write(message);
309 | var num = GetNumber();
310 |
311 | while (num < min)
312 | {
313 | Console.Write(message);
314 | num = GetNumber();
315 | }
316 |
317 | return num;
318 | }
319 |
320 | private static int GetNumber()
321 | {
322 | int num;
323 | var line = GetLine();
324 | return line != null && int.TryParse(line, out num) ? num : 0;
325 | }
326 |
327 | private static bool GetBool(string message)
328 | {
329 | Console.WriteLine(message);
330 | Console.Write("Answer: ");
331 | var line = GetLine();
332 |
333 | bool answer;
334 | while (line == null || !TryGetBoolResponse(line.ToLower(), out answer))
335 | {
336 | if (line == "exit")
337 | Environment.Exit(0);
338 |
339 | Console.WriteLine(message);
340 | Console.Write("Answer: ");
341 | line = GetLine();
342 | }
343 |
344 | PrintNewLine();
345 | return answer;
346 | }
347 |
348 | private static bool TryGetBoolResponse(string line, out bool answer)
349 | {
350 | answer = false;
351 | if (string.IsNullOrEmpty(line)) return false;
352 |
353 | if (bool.TryParse(line, out answer)) return true;
354 |
355 | switch (line[0])
356 | {
357 | case 'y':
358 | answer = true;
359 | return true;
360 | case 'n':
361 | return true;
362 | }
363 |
364 | return false;
365 | }
366 |
367 | private static void PrintNewLine(int numNewLines = 1)
368 | {
369 | for (var i = 0; i < numNewLines; i++)
370 | Console.WriteLine();
371 | }
372 |
373 | private static void PrintUnderline(int numUnderlines)
374 | {
375 | for (var i = 0; i < numUnderlines; i++)
376 | Console.Write('-');
377 | PrintNewLine(2);
378 | }
379 |
380 | private static void WriteError(string error)
381 | {
382 | Console.WriteLine(error);
383 | Console.ReadLine();
384 | Environment.Exit(0);
385 | }
386 | #endregion
387 | }
388 | }
--------------------------------------------------------------------------------