├── src ├── NeuralLink.cpp ├── NetworkFunction.cpp ├── NeuronFactory.cpp ├── .directory ├── CMakeLists.txt ├── NeuralNetwork.cpp ├── Neuron.cpp └── trainAlgorithm.cpp ├── README ├── test ├── CMakeLists.txt └── LogicalElement.cpp ├── CMakeLists.txt └── include └── NeuralNetworks ├── NetworkFunction.h ├── NeuronFactory.h ├── trainAlgorithm.h ├── NeuralLink.h ├── Neuron.h └── NeuralNetwork.h /src/NeuralLink.cpp: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/NetworkFunction.cpp: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/NeuronFactory.cpp: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/.directory: -------------------------------------------------------------------------------- 1 | [Dolphin] 2 | Timestamp=2013,10,15,15,11,22 3 | ViewMode=1 4 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | This is an educational project dedicated to neural networks. 2 | In order to build the project run cmake with the path to the root of the project. 3 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories ("${PROJECT_SOURCE_DIR}/include") 2 | add_executable( LogicalElement LogicalElement.cpp) 3 | target_link_libraries(LogicalElement NeuralNetwork) 4 | 5 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories ("${PROJECT_SOURCE_DIR}/include") 2 | add_library(NeuralNetwork STATIC NeuralNetwork.cpp Neuron.cpp NetworkFunction.cpp NeuralLink.cpp NeuronFactory.cpp trainAlgorithm.cpp) 3 | add_library(NeuralNetworkShared SHARED NeuralNetwork.cpp Neuron.cpp NetworkFunction.cpp NeuralLink.cpp NeuronFactory.cpp trainAlgorithm.cpp) 4 | 5 | target_link_libraries (NeuralNetwork) 6 | 7 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | 3 | 4 | 5 | project(NeuralNetwork) 6 | 7 | set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) 8 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) 9 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) 10 | 11 | add_subdirectory (src) 12 | add_subdirectory (test) 13 | 14 | -------------------------------------------------------------------------------- /include/NeuralNetworks/NetworkFunction.h: -------------------------------------------------------------------------------- 1 | /* 2 | * NetworkFunction.h 3 | * 4 | * Created on: Sep 25, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | #ifndef NETWORKFUNCTION_H_ 9 | #define NETWORKFUNCTION_H_ 10 | 11 | #include 12 | 13 | class NetworkFunction { 14 | public: 15 | NetworkFunction(){}; 16 | virtual ~NetworkFunction(){}; 17 | virtual double Process( double inParam ) = 0; 18 | virtual double Derivative( double inParam ) = 0; 19 | }; 20 | 21 | class Linear : public NetworkFunction { 22 | public: 23 | Linear(){}; 24 | virtual ~Linear(){}; 25 | virtual double Process( double inParam ){ return inParam; }; 26 | virtual double Derivative( double inParam ){ return 0; }; 27 | }; 28 | 29 | 30 | class Sigmoid : public NetworkFunction { 31 | public: 32 | Sigmoid(){}; 33 | virtual ~Sigmoid(){}; 34 | virtual double Process( double inParam ){ return ( 1 / ( 1 + exp( -inParam ) ) ); }; 35 | virtual double Derivative( double inParam ){ return ( this->Process(inParam)*(1 - this->Process(inParam)) );}; 36 | }; 37 | 38 | class BipolarSigmoid : public NetworkFunction { 39 | public: 40 | BipolarSigmoid(){}; 41 | virtual ~BipolarSigmoid(){}; 42 | virtual double Process( double inParam ){ return ( 2 / ( 1 + exp( -inParam ) ) - 1 ) ;}; 43 | virtual double Derivative( double inParam ){ return ( 0.5 * ( 1 + this->Process( inParam ) ) * ( 1 - this->Process( inParam ) ) ); }; 44 | }; 45 | 46 | #endif /* NETWORKFUNCTION_H_ */ 47 | 48 | 49 | -------------------------------------------------------------------------------- /include/NeuralNetworks/NeuronFactory.h: -------------------------------------------------------------------------------- 1 | /* 2 | * NeuronFactory.h 3 | * 4 | * Created on: Sep 25, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | #ifndef NEURONFACTORY_H_ 9 | #define NEURONFACTORY_H_ 10 | 11 | #include "Neuron.h" 12 | 13 | template 14 | class NeuronFactory 15 | { 16 | public: 17 | NeuronFactory(){}; 18 | virtual ~NeuronFactory(){}; 19 | virtual Neuron * CreateInputNeuron( std::vector *>& inNeuronsLinkTo, NetworkFunction * inNetFunc ) = 0; 20 | virtual Neuron * CreateOutputNeuron( NetworkFunction * inNetFunc ) = 0; 21 | virtual Neuron * CreateHiddenNeuron( std::vector *>& inNeuronsLinkTo, NetworkFunction * inNetFunc ) = 0; 22 | 23 | }; 24 | 25 | template 26 | class PerceptronNeuronFactory : public NeuronFactory 27 | { 28 | public: 29 | PerceptronNeuronFactory(){}; 30 | virtual ~PerceptronNeuronFactory(){}; 31 | virtual Neuron * CreateInputNeuron( std::vector *>& inNeuronsLinkTo, NetworkFunction * inNetFunc ){ return new Neuron( inNeuronsLinkTo, inNetFunc ); }; 32 | virtual Neuron * CreateOutputNeuron( NetworkFunction * inNetFunc ){ return new OutputLayerNeuronDecorator( new Neuron( inNetFunc ) ); }; 33 | virtual Neuron * CreateHiddenNeuron( std::vector *>& inNeuronsLinkTo, NetworkFunction * inNetFunc ){ return new HiddenLayerNeuronDecorator( new Neuron( inNeuronsLinkTo, inNetFunc ) ); }; 34 | }; 35 | 36 | 37 | #endif /* NEURONFACTORY_H_ */ 38 | -------------------------------------------------------------------------------- /include/NeuralNetworks/trainAlgorithm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * trainAlgorithm.h 3 | * 4 | * Created on: Sep 24, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | #ifndef TRAINALGORITHM_H_ 9 | #define TRAINALGORITHM_H_ 10 | 11 | #include 12 | 13 | template 14 | class NeuralNetwork; 15 | 16 | template 17 | class TrainAlgorithm 18 | { 19 | public: 20 | virtual ~TrainAlgorithm(){}; 21 | virtual double Train(const std::vector& inData, const std::vector& inTarget) = 0; 22 | virtual void WeightsInitialization() = 0; 23 | protected: 24 | }; 25 | 26 | template 27 | class Hebb : public TrainAlgorithm 28 | { 29 | public: 30 | Hebb(NeuralNetwork * inNeuralNetwork) : mNeuralNetwork(inNeuralNetwork){}; 31 | virtual ~Hebb(){}; 32 | virtual double Train(const std::vector& inData, const std::vector& inTarget); 33 | virtual void WeightsInitialization(); 34 | protected: 35 | NeuralNetwork * mNeuralNetwork; 36 | }; 37 | 38 | template 39 | class Backpropagation : public TrainAlgorithm 40 | { 41 | public: 42 | Backpropagation(NeuralNetwork * inNeuralNetwork); 43 | virtual ~Backpropagation(){}; 44 | virtual double Train(const std::vector& inData, const std::vector& inTarget); 45 | virtual void WeightsInitialization(); 46 | protected: 47 | void NguyenWidrowWeightsInitialization(); 48 | void CommonInitialization(); 49 | NeuralNetwork * mNeuralNetwork; 50 | }; 51 | 52 | 53 | 54 | 55 | #endif /* TRAINALGORITHM_H_ */ 56 | -------------------------------------------------------------------------------- /include/NeuralNetworks/NeuralLink.h: -------------------------------------------------------------------------------- 1 | /* 2 | * NeuralLink.h 3 | * 4 | * Created on: Sep 19, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | #ifndef NEURALLINK_H_ 9 | #define NEURALLINK_H_ 10 | 11 | template 12 | class Neuron; 13 | 14 | 15 | 16 | template 17 | class NeuralLink 18 | { 19 | public: 20 | NeuralLink( ) : mWeightToNeuron( 0.0 ), mNeuronLinkedTo( 0 ), mWeightCorrectionTerm( 0 ), mErrorInformationTerm( 0 ), mLastTranslatedSignal( 0 ){ }; 21 | NeuralLink( Neuron * inNeuronLinkedTo, double inWeightToNeuron = 0.0 ) : mWeightToNeuron( inWeightToNeuron ), mNeuronLinkedTo( inNeuronLinkedTo ), mWeightCorrectionTerm( 0 ), mErrorInformationTerm( 0 ), mLastTranslatedSignal( 0 ){ }; 22 | 23 | void SetWeight( const double& inWeight ){ mWeightToNeuron = inWeight; }; 24 | const double& GetWeight( ){ return mWeightToNeuron; }; 25 | 26 | void SetNeuronLinkedTo( Neuron * inNeuronLinkedTo ){ mNeuronLinkedTo = inNeuronLinkedTo; }; 27 | Neuron * GetNeuronLinkedTo( ){ return mNeuronLinkedTo; }; 28 | 29 | void SetWeightCorrectionTerm( double inWeightCorrectionTerm ){ mWeightCorrectionTerm = inWeightCorrectionTerm; }; 30 | double GetWeightCorrectionTerm( ){ return mWeightCorrectionTerm; }; 31 | 32 | void UpdateWeight( ){ mWeightToNeuron = mWeightToNeuron + mWeightCorrectionTerm; }; 33 | 34 | double GetErrorInFormationTerm( ){ return mErrorInformationTerm; }; 35 | void SetErrorInFormationTerm( double inEITerm ){ mErrorInformationTerm = inEITerm; }; 36 | 37 | void SetLastTranslatedSignal( double inLastTranslatedSignal ){ mLastTranslatedSignal = inLastTranslatedSignal; }; 38 | double GetLastTranslatedSignal( ){ return mLastTranslatedSignal; }; 39 | protected: 40 | double mWeightToNeuron; 41 | Neuron * mNeuronLinkedTo; 42 | double mWeightCorrectionTerm; 43 | double mErrorInformationTerm; 44 | double mLastTranslatedSignal; 45 | }; 46 | 47 | 48 | #endif /* NEURALLINK_H_ */ 49 | -------------------------------------------------------------------------------- /test/LogicalElement.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cpp 3 | * 4 | * Created on: Sep 27, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | 9 | 10 | #include "NeuralNetworks/NeuralNetwork.h" 11 | #include "NeuralNetworks/trainAlgorithm.h" 12 | #include "stdlib.h" 13 | #include 14 | 15 | 16 | 17 | 18 | 19 | int main() 20 | { 21 | 22 | std::vector > DataToFeedNN; 23 | std::vector Data1; 24 | Data1.push_back(1.0); 25 | Data1.push_back(1.0); 26 | 27 | DataToFeedNN.push_back(Data1); 28 | 29 | std::vector Data2; 30 | Data2.push_back(1.0); 31 | Data2.push_back(-1.0); 32 | 33 | DataToFeedNN.push_back(Data2); 34 | 35 | std::vector Data3; 36 | Data3.push_back(-1.0); 37 | Data3.push_back(1.0); 38 | 39 | DataToFeedNN.push_back(Data3); 40 | 41 | std::vector Data4; 42 | Data4.push_back(-1.0); 43 | Data4.push_back(-1.0); 44 | 45 | DataToFeedNN.push_back(Data4); 46 | 47 | std::vector Data5; 48 | Data5.push_back(0.5); 49 | Data5.push_back(-1.0); 50 | //DataToFeedNN.push_back(Data5); 51 | 52 | std::vector > trainingSample; 53 | std::vector ts1; 54 | ts1.push_back(1); 55 | 56 | std::vector ts2; 57 | ts2.push_back(-1); 58 | 59 | std::vector ts3; 60 | ts3.push_back(-1); 61 | 62 | std::vector ts4; 63 | ts4.push_back(-1); 64 | 65 | std::vector ts5; 66 | ts5.push_back(-1); 67 | 68 | trainingSample.push_back(ts1); 69 | trainingSample.push_back(ts2); 70 | trainingSample.push_back(ts3); 71 | trainingSample.push_back(ts4); 72 | //trainingSample.push_back(ts5); 73 | 74 | NeuralNetwork * NN = new NeuralNetwork(2,1,1,4); 75 | NN->SetMinMSE(0.01); 76 | NN->Train(DataToFeedNN,trainingSample); 77 | 78 | 79 | std::cout << std::endl; 80 | std::cout << "Input data: { 1, 1 }\n"; 81 | NN->GetNetResponse(DataToFeedNN[0]); 82 | 83 | std::cout << std::endl; 84 | std::cout << "Input data: { 1, 0 }\n"; 85 | NN->GetNetResponse(DataToFeedNN[1]); 86 | 87 | std::cout << std::endl; 88 | std::cout << "Input data: { 0, 1 }\n"; 89 | NN->GetNetResponse(DataToFeedNN[2]); 90 | 91 | std::cout << std::endl; 92 | std::cout << "Input data: { 0, 0 }\n"; 93 | NN->GetNetResponse(DataToFeedNN[3]); 94 | 95 | 96 | std::cout << std::endl; 97 | std::cout << "Input data: { test }\n"; 98 | NN->GetNetResponse(Data5); 99 | 100 | delete NN; 101 | 102 | return 0; 103 | } 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /src/NeuralNetwork.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * NeuralNetwork.cpp 3 | * 4 | * Created on: Sep 20, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | 9 | #include "NeuralNetworks/NeuralNetwork.h" 10 | 11 | template 12 | NeuralNetwork::NeuralNetwork( const int& inInputs, const int& inOutputs, const int& inNumOfHiddenLayers, const int& inNumOfNeuronsInHiddenLayers, const char * inTypeOfNeuralNetwork ) 13 | { 14 | /* 15 | * Protection against the fools. 16 | * The neural net needs at least 1 input neuron and 1 output. 17 | */ 18 | 19 | if(inInputs > 0 && inOutputs > 0){ 20 | 21 | mMinMSE = 0.01; 22 | mMeanSquaredError = 0; 23 | mInputs = inInputs; 24 | mOutputs = inOutputs; 25 | mHidden = inNumOfNeuronsInHiddenLayers; 26 | 27 | /* 28 | * Network function's declarations for input and output neurons. 29 | */ 30 | 31 | NetworkFunction * OutputNeuronsFunc; 32 | NetworkFunction * InputNeuronsFunc; 33 | 34 | /* 35 | * At least two layers require - input and output; 36 | */ 37 | 38 | std::vector *> outputLayer; 39 | std::vector *> inputLayer; 40 | 41 | /* 42 | * This block of strcmps decides what training algorithm and neuron factory we should use as well as what 43 | * network function every node will have. 44 | */ 45 | 46 | if( strcmp( inTypeOfNeuralNetwork, "MultiLayerPerceptron" ) == 0){ 47 | mNeuronFactory = new PerceptronNeuronFactory; 48 | mTrainingAlgoritm = new Backpropagation(this); 49 | 50 | OutputNeuronsFunc = new BipolarSigmoid; 51 | InputNeuronsFunc = new Linear; 52 | 53 | } 54 | 55 | /* 56 | * Output layers creation 57 | */ 58 | 59 | for(int iNumOfOutputs = 0; iNumOfOutputs < inOutputs; iNumOfOutputs++){ 60 | outputLayer.push_back( mNeuronFactory->CreateOutputNeuron(OutputNeuronsFunc) ); 61 | } 62 | mLayers.push_back(outputLayer); 63 | 64 | /* 65 | * Hidden layers creation 66 | */ 67 | 68 | for(int i = 0; i < inNumOfHiddenLayers; i++){ 69 | std::vector *> HiddenLayer; 70 | for(int j = 0; j < inNumOfNeuronsInHiddenLayers; j++ ){ 71 | Neuron * hidden = mNeuronFactory->CreateHiddenNeuron(mLayers[0], OutputNeuronsFunc); 72 | HiddenLayer.push_back(hidden); 73 | } 74 | mBiasLayer.insert(mBiasLayer.begin(),mNeuronFactory->CreateInputNeuron(mLayers[0], InputNeuronsFunc)); 75 | mLayers.insert(mLayers.begin(),HiddenLayer); 76 | 77 | } 78 | 79 | /* 80 | * Input layers creation 81 | */ 82 | 83 | for(int iNumOfInputs = 0; iNumOfInputs < inInputs; iNumOfInputs++){ 84 | inputLayer.push_back(mNeuronFactory->CreateInputNeuron(mLayers[0], InputNeuronsFunc)); 85 | } 86 | mBiasLayer.insert(mBiasLayer.begin(),mNeuronFactory->CreateInputNeuron(mLayers[0],InputNeuronsFunc)); 87 | mLayers.insert(mLayers.begin(),inputLayer); 88 | 89 | mTrainingAlgoritm->WeightsInitialization(); 90 | } 91 | else{ 92 | std::cout << "Error in Neural Network constructor: The number of input and output neurons has to be more than 0!\n"; 93 | } 94 | } 95 | 96 | template 97 | NeuralNetwork::~NeuralNetwork() 98 | { 99 | delete mNeuronFactory; 100 | delete mTrainingAlgoritm; 101 | 102 | for( unsigned int uNumOfBiases = 0; uNumOfBiases < mBiasLayer.size(); uNumOfBiases++ ){ 103 | delete mBiasLayer[ uNumOfBiases ]; 104 | } 105 | 106 | for( unsigned int uNumOfLayers = 0; uNumOfLayers < mLayers.size(); uNumOfLayers++){ 107 | for( unsigned int uNumOfNeurons = 0; uNumOfNeurons < mLayers[uNumOfLayers].size(); uNumOfNeurons++ ){ 108 | delete mLayers[ uNumOfLayers ].at( uNumOfNeurons ); 109 | } 110 | } 111 | 112 | } 113 | 114 | template 115 | bool NeuralNetwork::Train( const std::vector >& inData, const std::vector >& inTarget ) 116 | { 117 | bool trues = true; 118 | int iIteration = 0; 119 | while(trues){ 120 | iIteration++; 121 | for(int i = 0; i < inData.size(); i++){ 122 | mTrainingAlgoritm->Train( inData[i], inTarget[i] ); 123 | } 124 | double MSE = this->GetMSE(); 125 | if( MSE < mMinMSE){ 126 | std::cout << "At " << iIteration << " iteration MSE: " << MSE << " was achieved\n"; 127 | trues = false; 128 | } 129 | this->ResetMSE(); 130 | } 131 | //return mTrainingAlgoritm->Train( inData,inTarget); 132 | return trues; 133 | } 134 | 135 | template 136 | std::vector NeuralNetwork::GetNetResponse( const std::vector& inData ) 137 | { 138 | std::vector netResponse; 139 | if(inData.size() != mInputs){ 140 | std::cout << "Input data dimensions are wrong, expected: " << mInputs << " elements\n"; 141 | return netResponse; 142 | } 143 | else{ 144 | for(unsigned int indexOfData = 0; indexOfData < this->GetInputLayer().size(); indexOfData++){ 145 | this->GetInputLayer().at(indexOfData)->Input(inData[indexOfData]); 146 | } 147 | 148 | for(unsigned int numOfLayers = 0; numOfLayers < mLayers.size() - 1; numOfLayers++){ 149 | mBiasLayer[numOfLayers]->Input(1.0); 150 | 151 | for(unsigned int indexOfData = 0; indexOfData < mLayers.at(numOfLayers).size(); indexOfData++){ 152 | mLayers.at(numOfLayers).at(indexOfData)->Fire(); 153 | } 154 | 155 | mBiasLayer[numOfLayers]->Fire(); 156 | } 157 | 158 | std::cout << "Net response is: { "; 159 | for(unsigned int indexOfOutputElements = 0; indexOfOutputElements < mOutputs; indexOfOutputElements++){ 160 | 161 | /* 162 | * For every neuron in output layer, make it fire its sum of charges; 163 | */ 164 | 165 | double res = this->GetOutputLayer().at(indexOfOutputElements)->Fire(); 166 | 167 | std::cout << "res: " << res << std::endl; 168 | 169 | 170 | } 171 | std::cout << " } \n"; 172 | this->ResetCharges(); 173 | return netResponse; 174 | 175 | } 176 | 177 | 178 | 179 | } 180 | 181 | template 182 | void NeuralNetwork::ResetCharges() 183 | { 184 | for(unsigned int i = 0; i < mLayers.size(); i++ ){ 185 | for(unsigned int indexOfOutputElements = 0; indexOfOutputElements < mLayers.at(i).size(); indexOfOutputElements++){ 186 | mLayers.at(i).at(indexOfOutputElements)->ResetSumOfCharges(); 187 | 188 | 189 | } 190 | //mBiasLayer[i]->ResetSumOfCharges(); 191 | } 192 | for(unsigned int i = 0; i < mLayers.size()-1; i++ ){ 193 | mBiasLayer[i]->ResetSumOfCharges(); 194 | } 195 | 196 | } 197 | 198 | template 199 | void NeuralNetwork::UpdateWeights() 200 | { 201 | for(unsigned int indOfLayer = 0; indOfLayer < mLayers.size(); indOfLayer++){ 202 | for(unsigned int indOfNeuron = 0; indOfNeuron < mLayers[indOfLayer].size(); indOfNeuron++){ 203 | mLayers[indOfLayer].at(indOfNeuron)->PerformWeightsUpdating(); 204 | } 205 | } 206 | } 207 | 208 | template 209 | void NeuralNetwork::ShowNetworkState() 210 | { 211 | std::cout << std::endl; 212 | for(unsigned int indOfLayer = 0; indOfLayer < mLayers.size(); indOfLayer++){ 213 | std::cout << "Layer index: " << indOfLayer << std::endl; 214 | for(unsigned int indOfNeuron = 0; indOfNeuron < mLayers[indOfLayer].size(); indOfNeuron++){ 215 | std::cout << " Neuron index: " << indOfNeuron << std::endl; 216 | mLayers[indOfLayer].at(indOfNeuron)->ShowNeuronState(); 217 | } 218 | if(indOfLayer < mBiasLayer.size()){ 219 | std::cout << " Bias: " << std::endl; 220 | mBiasLayer[indOfLayer]->ShowNeuronState(); 221 | } 222 | } 223 | } 224 | 225 | template class NeuralNetwork; 226 | template class NeuralNetwork; 227 | template class NeuralNetwork; 228 | -------------------------------------------------------------------------------- /include/NeuralNetworks/Neuron.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Neuron.h 3 | * 4 | * Created on: Sep 19, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | #ifndef NEURON_H_ 9 | #define NEURON_H_ 10 | 11 | #include "NeuralLink.h" 12 | #include "NetworkFunction.h" 13 | #include 14 | #include 15 | 16 | const double LearningRate = 0.01; 17 | 18 | 19 | /** 20 | * Neuron base class. 21 | * Represents a basic element of neural network, node in the net's graph. 22 | * There are several possibilities for creation an object of type Neuron, different constructors suites for 23 | * different situations. 24 | */ 25 | 26 | template 27 | class Neuron 28 | { 29 | public: 30 | 31 | /** 32 | * A default Neuron constructor. 33 | * - Description: Creates a Neuron; general purposes. 34 | * - Purpose: Creates a Neuron, linked to nothing, with a Linear network function. 35 | * - Prerequisites: None. 36 | */ 37 | 38 | Neuron( ) : mNetFunc( new Linear ), mSumOfCharges( 0.0 ) { }; 39 | 40 | /** 41 | * A Neuron constructor based on NetworkFunction. 42 | * - Description: Creates a Neuron; mostly designed to create an output kind of neurons. 43 | * @param inNetFunc - a network function which is producing neuron's output signal; 44 | * - Purpose: Creates a Neuron, linked to nothing, with a specific network function. 45 | * - Prerequisites: The existence of NetworkFunction object. 46 | */ 47 | 48 | Neuron( NetworkFunction * inNetFunc ) : mNetFunc( inNetFunc ), mSumOfCharges( 0.0 ) { }; 49 | 50 | Neuron( std::vector *>& inLinksToNeurons, NetworkFunction * inNetFunc ) : 51 | mNetFunc( inNetFunc ), 52 | mLinksToNeurons(inLinksToNeurons), 53 | mSumOfCharges(0.0) { }; 54 | 55 | /** 56 | * A Neuron constructor based on layer of Neurons. 57 | * - Description: Creates a Neuron; mostly designed to create an input and hidden kinds of neurons. 58 | * @param inNeuronsLinkTo - a vector of pointers to Neurons which is representing a layer; 59 | * @param inNetFunc - a network function which is producing neuron's output signal; 60 | * - Purpose: Creates a Neuron, linked to every Neuron in provided layer. 61 | * - Prerequisites: The existence of std::vector and NetworkFunction. 62 | */ 63 | 64 | Neuron( std::vector& inNeuronsLinkTo, NetworkFunction * inNetFunc ); 65 | 66 | virtual ~Neuron( ); 67 | 68 | virtual std::vector *>& GetLinksToNeurons( ) { return mLinksToNeurons; }; 69 | virtual NeuralLink * at( const int& inIndexOfNeuralLink ) { return mLinksToNeurons[ inIndexOfNeuralLink ]; }; 70 | 71 | virtual void SetLinkToNeuron( NeuralLink * inNeuralLink ) { mLinksToNeurons.push_back( inNeuralLink ); }; 72 | 73 | virtual void Input( double inInputData ) { mSumOfCharges += inInputData; }; 74 | virtual double Fire( ); 75 | virtual int GetNumOfLinks( ) { return mLinksToNeurons.size( ); }; 76 | virtual double GetSumOfCharges( ); 77 | virtual void ResetSumOfCharges( ) { mSumOfCharges = 0.0; }; 78 | virtual double Process( ) { return mNetFunc->Process( mSumOfCharges ); }; 79 | virtual double Process( double inArg ) { return mNetFunc->Process( inArg ); }; 80 | virtual double Derivative( ) { return mNetFunc->Derivative( mSumOfCharges ); }; 81 | 82 | virtual void SetInputLink( NeuralLink * inLink ) { mInputLinks.push_back( inLink ); }; 83 | virtual std::vector *>& GetInputLink( ) { return mInputLinks; }; 84 | 85 | 86 | 87 | virtual double PerformTrainingProcess( double inTarget ); 88 | virtual void PerformWeightsUpdating( ); 89 | 90 | virtual void ShowNeuronState( ); 91 | protected: 92 | NetworkFunction * mNetFunc; 93 | std::vector *> mInputLinks; 94 | std::vector *> mLinksToNeurons; 95 | 96 | double mSumOfCharges; 97 | }; 98 | 99 | template 100 | class OutputLayerNeuronDecorator : public Neuron 101 | { 102 | public: 103 | OutputLayerNeuronDecorator( Neuron * inNeuron ) { mOutputCharge = 0; mNeuron = inNeuron; }; 104 | virtual ~OutputLayerNeuronDecorator( ); 105 | 106 | virtual std::vector *>& GetLinksToNeurons( ) { return mNeuron->GetLinksToNeurons( ) ;}; 107 | virtual NeuralLink * at( const int& inIndexOfNeuralLink ) { return ( mNeuron->at( inIndexOfNeuralLink ) ) ;}; 108 | virtual void SetLinkToNeuron( NeuralLink * inNeuralLink ) { mNeuron->SetLinkToNeuron( inNeuralLink ); }; 109 | virtual double GetSumOfCharges( ) { return mNeuron->GetSumOfCharges( ); }; 110 | 111 | virtual void ResetSumOfCharges( ) { mNeuron->ResetSumOfCharges( ); }; 112 | virtual void Input( double inInputData ) { mNeuron->Input( inInputData ); }; 113 | virtual double Fire( ); 114 | virtual int GetNumOfLinks( ) { return mNeuron->GetNumOfLinks( ); }; 115 | 116 | 117 | virtual double Process( ) { return mNeuron->Process( ); }; 118 | virtual double Process( double inArg ) { return mNeuron->Process( inArg ); }; 119 | 120 | virtual double Derivative( ) { return mNeuron->Derivative( ); }; 121 | 122 | virtual void SetInputLink( NeuralLink * inLink ) { mNeuron->SetInputLink( inLink ); }; 123 | virtual std::vector *>& GetInputLink( ) { return mNeuron->GetInputLink( ); }; 124 | 125 | virtual double PerformTrainingProcess( double inTarget ); 126 | virtual void PerformWeightsUpdating( ); 127 | virtual void ShowNeuronState( ) { mNeuron->ShowNeuronState( ); }; 128 | protected: 129 | double mOutputCharge; 130 | Neuron * mNeuron; 131 | 132 | }; 133 | 134 | template 135 | class HiddenLayerNeuronDecorator : public Neuron 136 | { 137 | public: 138 | HiddenLayerNeuronDecorator( Neuron * inNeuron ) { mNeuron = inNeuron; }; 139 | virtual ~HiddenLayerNeuronDecorator( ); 140 | 141 | virtual std::vector *>& GetLinksToNeurons( ) { return mNeuron->GetLinksToNeurons( ); }; 142 | virtual void SetLinkToNeuron( NeuralLink * inNeuralLink ) { mNeuron->SetLinkToNeuron( inNeuralLink ); }; 143 | virtual double GetSumOfCharges( ) { return mNeuron->GetSumOfCharges( ) ;}; 144 | 145 | virtual void ResetSumOfCharges( ) {mNeuron->ResetSumOfCharges( ); }; 146 | virtual void Input( double inInputData ) { mNeuron->Input( inInputData ); }; 147 | virtual double Fire( ); 148 | virtual int GetNumOfLinks( ) { return mNeuron->GetNumOfLinks( ); }; 149 | virtual NeuralLink * at( const int& inIndexOfNeuralLink ) { return ( mNeuron->at( inIndexOfNeuralLink) ); }; 150 | 151 | virtual double Process( ) { return mNeuron->Process( ); }; 152 | virtual double Process( double inArg ) { return mNeuron->Process( inArg ); }; 153 | 154 | virtual double Derivative( ) { return mNeuron->Derivative( ); }; 155 | 156 | virtual void SetInputLink( NeuralLink * inLink ) { mNeuron->SetInputLink( inLink ); }; 157 | virtual std::vector *>& GetInputLink( ) { return mNeuron->GetInputLink( ); }; 158 | 159 | virtual double PerformTrainingProcess( double inTarget ); 160 | virtual void PerformWeightsUpdating( ); 161 | 162 | virtual void ShowNeuronState( ) { mNeuron->ShowNeuronState( ); }; 163 | protected: 164 | 165 | 166 | Neuron * mNeuron; 167 | 168 | }; 169 | 170 | 171 | #endif /* NEURON_H_ */ 172 | -------------------------------------------------------------------------------- /src/Neuron.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Neuron.cpp 3 | * 4 | * Created on: Sep 20, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | #include "NeuralNetworks/Neuron.h" 9 | 10 | /** 11 | * A neuron constructor. 12 | * Description: Constructor for neuron, based on a std::vector of already exist neurons and net function. 13 | * Purpose: Creating a new neuron with neural links to every neuron in incoming vector and predefined function. 14 | * Prerequisites: Existence of a vector with neurons and net function. 15 | */ 16 | 17 | template 18 | Neuron::Neuron(std::vector * >& inNeuronsLinkTo, NetworkFunction * inNetFunc) 19 | { 20 | /* 21 | * Net Function is an activation function for neuron 22 | */ 23 | 24 | 25 | mNetFunc = inNetFunc; 26 | 27 | /* 28 | * Initially there is no input data, so sum of charges equal 0 29 | */ 30 | 31 | 32 | mSumOfCharges = 0.0; 33 | 34 | 35 | for(unsigned int i = 0; i < inNeuronsLinkTo.size(); i++){ 36 | 37 | /* 38 | * Creating a link, based on Neuron from vector for every neuron in vector 39 | */ 40 | 41 | NeuralLink * pLink = new NeuralLink( inNeuronsLinkTo[i], 0.0 ); 42 | 43 | /* 44 | * Newly created neuron will know who is linked to it, by maintaining a vector of links called mLinksToNeurons. 45 | */ 46 | 47 | mLinksToNeurons.push_back( pLink ); 48 | 49 | /* 50 | * A neuron, which is linked to newly created neuron, will know as well what its linked to, by maintaining a vector of input links. 51 | */ 52 | 53 | inNeuronsLinkTo[i]->SetInputLink( pLink ); 54 | /* std::cin.get(); 55 | NeuralLink * pInLink = inNeuronsLinkTo[i]->GetInputLink().back(); 56 | pInLink->SetWeightCorrectionTerm(10); 57 | std::cout << pLink->GetWeightCorrectionTerm() << std::endl; 58 | std::cout << pLink << std::endl; 59 | std::cout << pInLink<< std::endl; 60 | std::cin.get();*/ 61 | } 62 | 63 | } 64 | 65 | template 66 | Neuron::~Neuron() 67 | { 68 | //delete mNetFunc; 69 | 70 | /* for( unsigned int uNumOfLinks = 0; uNumOfLinks < mLinksToNeurons.size(); uNumOfLinks++){ 71 | delete mLinksToNeurons[uNumOfLinks]; 72 | }*/ 73 | 74 | } 75 | 76 | template 77 | OutputLayerNeuronDecorator::~OutputLayerNeuronDecorator() 78 | { 79 | delete mNeuron; 80 | } 81 | 82 | template 83 | HiddenLayerNeuronDecorator::~HiddenLayerNeuronDecorator() 84 | { 85 | delete mNeuron; 86 | } 87 | 88 | 89 | template 90 | double HiddenLayerNeuronDecorator::Fire() 91 | { 92 | /* 93 | * Hidden unit applies its activation function to compute its output signal 94 | * and sends this signal to all units in the layer above (output units). 95 | */ 96 | 97 | for(int iLink = 0; iLink < this->GetNumOfLinks(); iLink++){ 98 | 99 | NeuralLink * pCurrentLink = mNeuron->at(iLink); 100 | Neuron * pCurrentNeuronLinkedTo = pCurrentLink->GetNeuronLinkedTo(); 101 | 102 | const double dWeight = mNeuron->at(iLink)->GetWeight(); 103 | double dCharge = mNeuron->GetSumOfCharges(); 104 | double dZj = (mNeuron->Process(dCharge)); 105 | double dOutput = dZj*dWeight; 106 | 107 | pCurrentLink->SetLastTranslatedSignal(dZj); 108 | 109 | pCurrentNeuronLinkedTo->Input( dOutput ); 110 | 111 | //std::cout << "Link: " << iLink << ", " << "Hidden Neuron fired: " << dOutput << " as func of: " << dCharge << " * " << dWeight << std::endl; 112 | } 113 | 114 | return mNeuron->GetSumOfCharges(); 115 | } 116 | 117 | template 118 | double OutputLayerNeuronDecorator::Fire() 119 | { 120 | 121 | //double temp = mNeuron->GetSumOfCharges(); 122 | double output = this->Process(); 123 | mOutputCharge = output; 124 | //std::cout << "Output Neuron fired: " << output << " as func of: " << temp << std::endl; 125 | return output; 126 | 127 | }; 128 | 129 | template 130 | double Neuron::Fire() 131 | { 132 | for(int iLink = 0; iLink < this->GetNumOfLinks(); iLink++){ 133 | NeuralLink * pCurrentLink = mLinksToNeurons[iLink]; 134 | Neuron * pCurrentNeuronLinkedTo = pCurrentLink->GetNeuronLinkedTo(); 135 | 136 | const double dWeight = pCurrentLink->GetWeight(); 137 | double dCharge = mSumOfCharges; 138 | double dXi = (mNetFunc->Process(dCharge)); 139 | double dOutput = dXi*dWeight; 140 | 141 | pCurrentLink->SetLastTranslatedSignal(dXi); 142 | pCurrentNeuronLinkedTo->Input( dOutput ); 143 | //std::cout << "Link: " << iLink << ", Neuron fired: " << dOutput << " as func of: " << dCharge << " * " << dWeight << std::endl; 144 | //mLinksToNeurons[iLink]->GetNeuronLinkedTo()->Input(mNetFunc->Process(mSumOfCharges*mLinksToNeurons[iLink]->GetWeight())); 145 | } 146 | //mSumOfCharges = 0; 147 | return mSumOfCharges; 148 | } 149 | 150 | template 151 | double Neuron::GetSumOfCharges() 152 | { 153 | 154 | return mSumOfCharges; 155 | } 156 | 157 | 158 | 159 | 160 | template 161 | double Neuron::PerformTrainingProcess(double inTarget) 162 | { 163 | return 0; 164 | } 165 | 166 | template 167 | double HiddenLayerNeuronDecorator::PerformTrainingProcess(double inTarget) 168 | { 169 | 170 | /* 171 | * Hidden unit sums its delta inputs from units in the layer above 172 | */ 173 | double dDeltaInputs = 0; 174 | for(int iOutputLink = 0; iOutputLink < (this->GetNumOfLinks()); iOutputLink++){ 175 | NeuralLink * pOutputLink = (this->GetLinksToNeurons()).at(iOutputLink); 176 | double dErrorInformationTerm = pOutputLink->GetErrorInFormationTerm(); 177 | double dWeight = pOutputLink->GetWeight(); 178 | dDeltaInputs = dDeltaInputs + (dWeight*dErrorInformationTerm); 179 | } 180 | 181 | /* for(int iOutputLink = 0; iOutputLink < (this->GetNumOfLinks()); iOutputLink++){ 182 | NeuralLink * pOutputLink = (this->GetLinksToNeurons()).at(iOutputLink); 183 | pOutputLink->UpdateWeight(); 184 | }*/ 185 | 186 | double dErrorInformationTermj = dDeltaInputs * (this->Derivative()); 187 | //std::cout << "dErrorInformationTermjHidden: " << dErrorInformationTermj << " as: " << dDeltaInputs << " * " << this->Derivative() << " .Derivative of: " << mNeuron->GetSumOfCharges()<< std::endl; 188 | //std::cin.get(); 189 | /* 190 | * For every link to that hidden neuron, (inputLinks) calculate its weight correction term 191 | * and update the link with it. 192 | */ 193 | for(unsigned int iInputLink = 0; iInputLink < (this->GetInputLink()).size(); iInputLink++){ 194 | NeuralLink * pInputLink = (this->GetInputLink()).at(iInputLink); 195 | double Xi = pInputLink->GetLastTranslatedSignal(); 196 | double dWeightCorrectionTerm = Xi*dErrorInformationTermj; 197 | //std::cout << "dWeightCorrectionTerm: " << dWeightCorrectionTerm << std::endl; 198 | pInputLink->SetWeightCorrectionTerm(LearningRate*dWeightCorrectionTerm); 199 | 200 | 201 | /* 202 | * Then hidden unit has to tell the input neurons the value of it ErrorInformationTerm, so we are setting its value 203 | * in the link object. 204 | */ 205 | 206 | pInputLink->SetErrorInFormationTerm(dErrorInformationTermj); 207 | } 208 | return 0; 209 | } 210 | 211 | template 212 | double OutputLayerNeuronDecorator::PerformTrainingProcess(double inTarget) 213 | { 214 | double res; 215 | double dErrorInformationTerm = (inTarget - mOutputCharge) * mNeuron->Derivative(); 216 | res = pow(inTarget - mOutputCharge,2); 217 | //std::cout << "dErrorInformationTermOutput: " << dErrorInformationTerm << " as: " << "(" << inTarget << " - " << mOutputCharge << ")" << " * " << mNeuron->Derivative() << " .Derivative of: " << mNeuron->GetSumOfCharges()<< std::endl; 218 | //std::cin.get(); 219 | 220 | /* 221 | * For every link to that output, (inputLinks) calculate its weight correction term 222 | * and update the link with it. 223 | */ 224 | for(unsigned int iInputLink = 0; iInputLink < (this->GetInputLink()).size(); iInputLink++){ 225 | NeuralLink * pInputLink = (this->GetInputLink()).at(iInputLink); 226 | double Zj = pInputLink->GetLastTranslatedSignal(); 227 | double dWeightCorrectionTerm = Zj*dErrorInformationTerm; 228 | //std::cout << "dWeightCorrectionTerm: " << dWeightCorrectionTerm << std::endl; 229 | pInputLink->SetWeightCorrectionTerm(LearningRate*dWeightCorrectionTerm); 230 | 231 | 232 | /* 233 | * Then output unit has to tell the hidden neurons the value of it ErrorInformationTerm, so we are setting its value 234 | * in the link object. 235 | */ 236 | 237 | pInputLink->SetErrorInFormationTerm(dErrorInformationTerm); 238 | } 239 | 240 | 241 | return res; 242 | } 243 | 244 | template 245 | void Neuron::PerformWeightsUpdating() 246 | { 247 | 248 | } 249 | 250 | template 251 | void HiddenLayerNeuronDecorator::PerformWeightsUpdating() 252 | { 253 | for(unsigned int iInputLink = 0; iInputLink < (this->GetInputLink()).size(); iInputLink++){ 254 | NeuralLink * pInputLink = (this->GetInputLink()).at(iInputLink); 255 | 256 | pInputLink->UpdateWeight(); 257 | //std::cout<<""; 258 | } 259 | } 260 | 261 | template 262 | void OutputLayerNeuronDecorator::PerformWeightsUpdating() 263 | { 264 | for(unsigned int iInputLink = 0; iInputLink < (this->GetInputLink()).size(); iInputLink++){ 265 | NeuralLink * pInputLink = (this->GetInputLink()).at(iInputLink); 266 | 267 | pInputLink->UpdateWeight(); 268 | //std::cout<<""; 269 | } 270 | } 271 | 272 | template 273 | void Neuron::ShowNeuronState() 274 | { 275 | /* 276 | * Printing out Neuron's link's weights 277 | */ 278 | 279 | for(unsigned int iNumOfOutLinks = 0; iNumOfOutLinks < mLinksToNeurons.size(); iNumOfOutLinks++ ){ 280 | NeuralLink * pNeuralLink = mLinksToNeurons.at(iNumOfOutLinks); 281 | std::cout << " Link index: " << iNumOfOutLinks << std::endl; 282 | std::cout << " Weight: " << pNeuralLink->GetWeight() << "; Weight correction term: " << pNeuralLink->GetWeightCorrectionTerm(); 283 | std::cout << std::endl; 284 | 285 | } 286 | } 287 | 288 | template class Neuron; 289 | template class Neuron; 290 | template class Neuron; 291 | 292 | template class OutputLayerNeuronDecorator; 293 | template class OutputLayerNeuronDecorator; 294 | template class OutputLayerNeuronDecorator; 295 | 296 | template class HiddenLayerNeuronDecorator; 297 | template class HiddenLayerNeuronDecorator; 298 | template class HiddenLayerNeuronDecorator; 299 | -------------------------------------------------------------------------------- /include/NeuralNetworks/NeuralNetwork.h: -------------------------------------------------------------------------------- 1 | /* 2 | * NeuralNetwork.h 3 | * 4 | * Created on: Sep 20, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | #ifndef NEURALNETWORK_H_ 9 | #define NEURALNETWORK_H_ 10 | 11 | 12 | 13 | #include "NeuronFactory.h" 14 | #include "trainAlgorithm.h" 15 | #include 16 | #include 17 | 18 | template 19 | class TrainAlgorithm; 20 | 21 | /** 22 | * Neural network class. 23 | * An object of that type represents a neural network of several types: 24 | * - Single layer perceptron; 25 | * - Multiple layers perceptron. 26 | * 27 | * There are several training algorithms available as well: 28 | * - Perceptron; 29 | * - Backpropagation. 30 | * 31 | * How to use this class: 32 | * To be able to use neural network , you have to create an instance of that class, specifying 33 | * a number of input neurons, output neurons, number of hidden layers and amount of neurons in hidden layers. 34 | * You can also specify a type of neural network, by passing a string with a name of neural network, otherwise 35 | * MultiLayerPerceptron will be used. ( A training algorithm can be changed via public calls); 36 | * 37 | * Once the neural network was created, all u have to do is to set the biggest MSE required to achieve during 38 | * the training phase ( or u can skip this step, then mMinMSE will be set to 0.01 ), 39 | * train the network by providing a training data with target results. 40 | * Afterwards u can obtain the net response by feeding the net with data; 41 | * 42 | */ 43 | 44 | template 45 | class NeuralNetwork 46 | { 47 | 48 | public: 49 | 50 | /** 51 | * A Neural Network constructor. 52 | * - Description: A template constructor. T is a data type, all the nodes will operate with. Create a neural network by providing it with: 53 | * @param inInputs - an integer argument - number of input neurons of newly created neural network; 54 | * @param inOutputs- an integer argument - number of output neurons of newly created neural network; 55 | * @param inNumOfHiddenLayers - an integer argument - number of hidden layers of newly created neural network, default is 0; 56 | * @param inNumOfNeuronsInHiddenLayers - an integer argument - number of neurons in hidden layers of newly created neural network ( note that every hidden layer has the same amount of neurons), default is 0; 57 | * @param inTypeOfNeuralNetwork - a const char * argument - a type of neural network, we are going to create. The values may be: 58 | *
    59 | *
  • MultiLayerPerceptron;
    • 60 | *
  • Default is MultiLayerPerceptron.
    • 61 | *
62 | * - Purpose: Creates a neural network for solving some interesting problems. 63 | * - Prerequisites: The template parameter has to be picked based on your input data. 64 | * 65 | */ 66 | NeuralNetwork( const int& inInputs, 67 | const int& inOutputs, 68 | const int& inNumOfHiddenLayers = 0, 69 | const int& inNumOfNeuronsInHiddenLayers = 0, 70 | const char * inTypeOfNeuralNetwork = "MultiLayerPerceptron" 71 | ); 72 | 73 | ~NeuralNetwork( ); 74 | 75 | /** 76 | * Public method Train. 77 | * - Description: Method for training the network. 78 | * - Purpose: Trains a network, so the weights on the links adjusted in the way to be able to solve problem. 79 | * - Prerequisites: 80 | * @param inData - a vector of vectors with data to train with; 81 | * @param inTarget - a vector of vectors with target data; 82 | * - the number of data samples and target samples has to be equal; 83 | * - the data and targets has to be in the appropriate order u want the network to learn. 84 | */ 85 | 86 | bool Train( const std::vector >& inData, 87 | const std::vector >& inTarget ); 88 | 89 | /** 90 | * Public method GetNetResponse. 91 | * - Description: Method for actually get response from net by feeding it with data. 92 | * - Purpose: By calling this method u make the network evaluate the response for u. 93 | * - Prerequisites: 94 | * @param inData - a vector data to feed with. 95 | */ 96 | 97 | std::vector GetNetResponse( const std::vector& inData ); 98 | 99 | /** 100 | * Public method SetAlgorithm. 101 | * - Description: Setter for algorithm of training the net. 102 | * - Purpose: Can be used for dynamic change of training algorithm. 103 | * - Prerequisites: 104 | * @param inTrainingAlgorithm - an existence of already created object of type TrainAlgorithm. 105 | */ 106 | 107 | void SetAlgorithm( TrainAlgorithm * inTrainingAlgorithm ) { mTrainingAlgoritm = inTrainingAlgorithm; }; 108 | 109 | /** 110 | * Public method SetNeuronFactory. 111 | * - Description: Setter for the factory, which is making neurons for the net. 112 | * - Purpose: Can be used for dynamic change of neuron factory. 113 | * - Prerequisites: 114 | * @param inNeuronFactory - an existence of already created object of type NeuronFactory. 115 | */ 116 | 117 | void SetNeuronFactory( NeuronFactory * inNeuronFactory ) { mNeuronFactory = inNeuronFactory; }; 118 | 119 | /** 120 | * Public method ShowNetworkState. 121 | * - Description: Prints current state to the standard output: weight of every link. 122 | * - Purpose: Can be used for monitoring the weights change during training of the net. 123 | * - Prerequisites: None. 124 | */ 125 | 126 | void ShowNetworkState( ); 127 | 128 | /** 129 | * Public method GetMinMSE. 130 | * - Description: Returns the biggest MSE required to achieve during the training phase. 131 | * - Purpose: Can be used for getting the biggest MSE required to achieve during the training phase. 132 | * - Prerequisites: None. 133 | */ 134 | 135 | const double& GetMinMSE( ) { return mMinMSE; }; 136 | 137 | /** 138 | * Public method SetMinMSE. 139 | * - Description: Setter for the biggest MSE required to achieve during the training phase. 140 | * - Purpose: Can be used for setting the biggest MSE required to achieve during the training phase. 141 | * - Prerequisites: 142 | * @param inMinMse - double value, the biggest MSE required to achieve during the training phase. 143 | */ 144 | 145 | void SetMinMSE( const double& inMinMse ) { mMinMSE = inMinMse; }; 146 | 147 | /** 148 | * Friend class. 149 | */ 150 | 151 | friend class Hebb; 152 | 153 | /** 154 | * Friend class. 155 | */ 156 | 157 | friend class Backpropagation; 158 | 159 | protected: 160 | 161 | /** 162 | * Protected method GetLayer. 163 | * - Description: Getter for the layer by index of that layer. 164 | * - Purpose: Can be used by inner implementation for getting access to neural network's layers. 165 | * - Prerequisites: 166 | * @param inInd - an integer index of layer. 167 | */ 168 | 169 | std::vector *>& GetLayer( const int& inInd ) { return mLayers[inInd]; }; 170 | 171 | /** 172 | * Protected method size. 173 | * - Description: Returns the number of layers in the network. 174 | * - Purpose: Can be used by inner implementation for getting number of layers in the network. 175 | * - Prerequisites: None. 176 | */ 177 | 178 | unsigned int size( ) { return mLayers.size( ); }; 179 | 180 | /** 181 | * Protected method GetNumOfOutputs. 182 | * - Description: Returns the number of units in the output layer. 183 | * - Purpose: Can be used by inner implementation for getting number of units in the output layer. 184 | * - Prerequisites: None. 185 | */ 186 | 187 | std::vector *>& GetOutputLayer( ) { return mLayers[mLayers.size( )-1]; }; 188 | 189 | /** 190 | * Protected method GetInputLayer. 191 | * - Description: Returns the input layer. 192 | * - Purpose: Can be used by inner implementation for getting the input layer. 193 | * - Prerequisites: None. 194 | */ 195 | 196 | std::vector *>& GetInputLayer( ) { return mLayers[0]; }; 197 | 198 | /** 199 | * Protected method GetBiasLayer. 200 | * - Description: Returns the vector of Biases. 201 | * - Purpose: Can be used by inner implementation for getting vector of Biases. 202 | * - Prerequisites: None. 203 | */ 204 | 205 | std::vector *>& GetBiasLayer( ) { return mBiasLayer; }; 206 | 207 | /** 208 | * Protected method UpdateWeights. 209 | * - Description: Updates the weights of every link between the neurons. 210 | * - Purpose: Can be used by inner implementation for updating the weights of links between the neurons. 211 | * - Prerequisites: None, but only makes sense, when its called during the training phase. 212 | */ 213 | 214 | void UpdateWeights( ); 215 | 216 | /** 217 | * Protected method ResetCharges. 218 | * - Description: Resets the neuron's data received during iteration of net training. 219 | * - Purpose: Can be used by inner implementation for reset the neuron's data between iterations. 220 | * - Prerequisites: None, but only makes sense, when its called during the training phase. 221 | */ 222 | 223 | void ResetCharges( ); 224 | 225 | /** 226 | * Protected method AddMSE. 227 | * - Description: Changes MSE during the training phase. 228 | * - Purpose: Can be used by inner implementation for changing MSE during the training phase. 229 | * - Prerequisites: 230 | * @param inInd - a double amount of MSE to be add. 231 | */ 232 | 233 | void AddMSE( double inPortion ) { mMeanSquaredError += inPortion; }; 234 | 235 | /** 236 | * Protected method GetMSE. 237 | * - Description: Getter for MSE value. 238 | * - Purpose: Can be used by inner implementation for getting access to the MSE value. 239 | * - Prerequisites: None. 240 | */ 241 | 242 | double GetMSE( ) { return mMeanSquaredError; }; 243 | 244 | /** 245 | * Protected method ResetMSE. 246 | * - Description: Resets MSE value. 247 | * - Purpose: Can be used by inner implementation for resetting MSE value. 248 | * - Prerequisites: None. 249 | */ 250 | 251 | void ResetMSE( ) { mMeanSquaredError = 0; }; 252 | 253 | 254 | NeuronFactory * mNeuronFactory; /*!< Member, which is responsible for creating neurons @see SetNeuronFactory */ 255 | TrainAlgorithm * mTrainingAlgoritm; /*!< Member, which is responsible for the way the network will trained @see SetAlgorithm */ 256 | std::vector *> > mLayers; /*!< Inner representation of neural networks */ 257 | std::vector *> mBiasLayer; /*!< Container for biases */ 258 | unsigned int mInputs, mOutputs, mHidden; /*!< Number of inputs, outputs and hidden units */ 259 | double mMeanSquaredError; /*!< Mean Squared Error which is changing every iteration of the training*/ 260 | double mMinMSE; /*!< The biggest Mean Squared Error required for training to stop*/ 261 | }; 262 | 263 | 264 | 265 | 266 | #endif /* NEURALNETWORK_H_ */ 267 | -------------------------------------------------------------------------------- /src/trainAlgorithm.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * trainAlgorithm.cpp 3 | * 4 | * Created on: Sep 25, 2013 5 | * Author: cheryuri 6 | */ 7 | 8 | 9 | #include "NeuralNetworks/trainAlgorithm.h" 10 | #include "NeuralNetworks/NeuralNetwork.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | template 17 | double Hebb::Train(const std::vector& inData, const std::vector& inTarget) 18 | { 19 | if(inData.size() != mNeuralNetwork->GetInputs() || inTarget.size() != mNeuralNetwork->GetOutputs()){ 20 | std::cout << "Input data dimensions are wrong, expected: " << mNeuralNetwork->GetInputs() << " elements\n"; 21 | return false; 22 | } 23 | else{ 24 | for(unsigned int indexOfData = 0; indexOfData < mNeuralNetwork->GetInputs(); indexOfData++){ 25 | mNeuralNetwork->GetInputLayer().at(indexOfData)->Input(inData[indexOfData]); 26 | } 27 | 28 | for(unsigned int numOfLayers = 0; numOfLayers < mNeuralNetwork->size() - 1; numOfLayers++){ 29 | mNeuralNetwork->GetBiasLayer().at(numOfLayers)->Input(1.0); 30 | 31 | for(unsigned int indexOfData = 0; indexOfData < mNeuralNetwork->GetLayer(numOfLayers).size(); indexOfData++){ 32 | mNeuralNetwork->GetLayer(numOfLayers).at(indexOfData)->Fire(); 33 | } 34 | 35 | mNeuralNetwork->GetBiasLayer().at(numOfLayers)->Fire(); 36 | } 37 | 38 | 39 | 40 | //output inTarget vector 41 | std::cout << "InTarget vector: { "; 42 | for(unsigned int i = 0; i < inTarget.size(); i++){ 43 | std::cout << inTarget[i] << " "; 44 | } 45 | std::cout << " }\n"; 46 | 47 | 48 | std::vector netResponse; 49 | for(unsigned int indexOfOutputElements = 0; indexOfOutputElements < mNeuralNetwork->GetNumOfOutputs(); indexOfOutputElements++){ 50 | 51 | /* 52 | * For every neuron in output layer, make it fire its sum of charges; 53 | */ 54 | 55 | double res = mNeuralNetwork->GetOutputLayer().at(indexOfOutputElements)->Fire(); 56 | if(res > 0.0){ 57 | netResponse.push_back(1); 58 | } 59 | else if( res == 0.0 ){ 60 | netResponse.push_back(0); 61 | } 62 | else{ 63 | netResponse.push_back(-1); 64 | } 65 | 66 | } 67 | 68 | //output netResponse vector 69 | std::cout << "netResponse vector: { "; 70 | for(unsigned int i = 0; i < netResponse.size(); i++){ 71 | std::cout << netResponse[i] << " "; 72 | } 73 | std::cout << " }\n"; 74 | 75 | for(unsigned int indexOfOutputElements = 0; indexOfOutputElements < mNeuralNetwork->GetNumOfOutputs(); indexOfOutputElements++){ 76 | if ( netResponse[indexOfOutputElements] != inTarget[indexOfOutputElements] ){ 77 | 78 | 79 | 80 | 81 | for(unsigned int numOfLayers = 0; numOfLayers < mNeuralNetwork->mLayers.size() - 1; numOfLayers++){ 82 | 83 | for(unsigned int NumOfNeurons = 0; NumOfNeurons < mNeuralNetwork->GetLayers().at(numOfLayers).size(); NumOfNeurons++){ 84 | Neuron * currentInputNeuron = (mNeuralNetwork->GetLayers().at(numOfLayers))[NumOfNeurons]; 85 | 86 | for(int NumOfWeights = 0; NumOfWeights < currentInputNeuron->GetNumOfLinks(); NumOfWeights++){ 87 | 88 | //std::cout << "x" << NumOfNeurons << ": " << inData[NumOfNeurons] << ", x" << NumOfNeurons << " * " << inTarget[NumOfWeights] << " + " << currentInputNeuron->at(NumOfWeights)->GetWeight() << " = " << currentInputNeuron->at(NumOfWeights)->GetWeight() + inTarget[NumOfWeights]*inData[NumOfNeurons] << std::endl; 89 | 90 | currentInputNeuron->at(NumOfWeights)->SetWeight(currentInputNeuron->at(NumOfWeights)->GetWeight() + inTarget[NumOfWeights]*inData[NumOfNeurons]); 91 | } 92 | 93 | } 94 | 95 | for(int NumOfWeights = 0; NumOfWeights < mNeuralNetwork->GetBiasLayer().at(numOfLayers)->GetNumOfLinks(); NumOfWeights++){ 96 | 97 | //std::cout << "Bias" << ": " << 1 << ", Bias " << " * " << 1 << " + " << mNeuralNetwork->GetBiasLayer().at(numOfLayers)->at(NumOfWeights)->GetWeight() << " = " <GetBiasLayer().at(numOfLayers)->at(NumOfWeights)->GetWeight() + inTarget[NumOfWeights]*1 << std::endl; 98 | 99 | mNeuralNetwork->GetBiasLayer().at(numOfLayers)->at(NumOfWeights)->SetWeight(mNeuralNetwork->GetBiasLayer().at(numOfLayers)->at(NumOfWeights)->GetWeight() + inTarget[NumOfWeights]*1); 100 | 101 | } 102 | 103 | } 104 | mNeuralNetwork->ResetCharges(); 105 | return false; 106 | } 107 | } 108 | 109 | mNeuralNetwork->ResetCharges(); 110 | return true; 111 | } 112 | 113 | } 114 | 115 | 116 | template 117 | void Backpropagation::NguyenWidrowWeightsInitialization() 118 | { 119 | /* 120 | * Step 0. Initialize weights ( Set to small values ) 121 | */ 122 | srand((unsigned)time(0)); 123 | 124 | 125 | /* 126 | * For every layer, for every neuron and bias in that layer, for every link in that neuron, set the weight 127 | * to random number from 0 to 1; 128 | * 129 | */ 130 | 131 | double dNumOfInputs = mNeuralNetwork->mInputs; 132 | double dNumOfHiddens = mNeuralNetwork->mHidden; 133 | double degree = 1.0 / dNumOfInputs ; 134 | double dScaleFactor = 0.7*( pow( dNumOfHiddens , degree ) ); 135 | 136 | for(unsigned int layerInd = 0; layerInd < mNeuralNetwork->size(); layerInd++){ 137 | for(unsigned int neuronInd = 0; neuronInd < mNeuralNetwork->GetLayer(layerInd).size(); neuronInd++){ 138 | Neuron * currentNeuron = mNeuralNetwork->GetLayer(layerInd).at(neuronInd); 139 | for(int linkInd = 0; linkInd < currentNeuron->GetNumOfLinks(); linkInd++){ 140 | NeuralLink * currentNeuralLink = currentNeuron->at(linkInd); 141 | float pseudoRandWeight = -0.5 + (float)rand()/((float)RAND_MAX/(0.5 + 0.5)); 142 | //float pseudoRandWeight = 0; 143 | currentNeuralLink->SetWeight(pseudoRandWeight); 144 | 145 | //std::cout << "layerInd: " << layerInd << ", neuronInd: " << neuronInd << ", linkInd: " << linkInd << ", Weight: " << currentNeuralLink->GetWeight() << std::endl; 146 | 147 | } 148 | } 149 | } 150 | 151 | 152 | for(unsigned int neuronHiddenInd = 0; neuronHiddenInd < mNeuralNetwork->GetLayer(1).size(); neuronHiddenInd++){ 153 | //Neuron * currentHiddenNeuron = mNeuralNetwork->GetLayer(1).at(neuronHiddenInd); 154 | 155 | double dSquaredNorm = 0; 156 | 157 | for(unsigned int neuronInputInd = 0; neuronInputInd < mNeuralNetwork->GetLayer(0).size(); neuronInputInd++){ 158 | Neuron * currentInputNeuron = mNeuralNetwork->GetLayer(0).at(neuronInputInd); 159 | 160 | NeuralLink * currentNeuralLink = currentInputNeuron->at(neuronHiddenInd); 161 | 162 | dSquaredNorm += pow(currentNeuralLink->GetWeight(),2.0); 163 | } 164 | 165 | double dNorm = sqrt(dSquaredNorm); 166 | 167 | for(unsigned int neuronInputInd = 0; neuronInputInd < mNeuralNetwork->GetLayer(0).size(); neuronInputInd++){ 168 | Neuron * currentInputNeuron = mNeuralNetwork->GetLayer(0).at(neuronInputInd); 169 | 170 | NeuralLink * currentNeuralLink = currentInputNeuron->at(neuronHiddenInd); 171 | 172 | double dNewWeight = ( dScaleFactor * ( currentNeuralLink->GetWeight() ) ) / dNorm; 173 | currentNeuralLink->SetWeight(dNewWeight); 174 | } 175 | 176 | } 177 | 178 | for(unsigned int layerInd = 0; layerInd < mNeuralNetwork->size() - 1; layerInd++){ 179 | 180 | Neuron * Bias = mNeuralNetwork->GetBiasLayer().at(layerInd); 181 | for(int linkInd = 0; linkInd < Bias->GetNumOfLinks(); linkInd++){ 182 | NeuralLink * currentNeuralLink = Bias->at(linkInd); 183 | float pseudoRandWeight = -dScaleFactor + (float)rand()/((float)RAND_MAX/(dScaleFactor + dScaleFactor)); 184 | //float pseudoRandWeight = 0; 185 | currentNeuralLink->SetWeight(pseudoRandWeight); 186 | //std::cout << "layerInd Bias: " << layerInd << ", linkInd: " << linkInd << ", Weight: " << currentNeuralLink->GetWeight() << std::endl; 187 | } 188 | } 189 | } 190 | 191 | template 192 | void Backpropagation::CommonInitialization() 193 | { 194 | /* 195 | * Step 0. Initialize weights ( Set to small values ) 196 | */ 197 | 198 | srand((unsigned)time(0)); 199 | 200 | 201 | /* 202 | * For every layer, for every neuron and bias in that layer, for every link in that neuron, set the weight 203 | * to random number from 0 to 1; 204 | * 205 | */ 206 | 207 | for(unsigned int layerInd = 0; layerInd < mNeuralNetwork->size(); layerInd++){ 208 | for(unsigned int neuronInd = 0; neuronInd < mNeuralNetwork->GetLayer(layerInd).size(); neuronInd++){ 209 | Neuron * currentNeuron = mNeuralNetwork->GetLayer(layerInd).at(neuronInd); 210 | for(int linkInd = 0; linkInd < currentNeuron->GetNumOfLinks(); linkInd++){ 211 | NeuralLink * currentNeuralLink = currentNeuron->at(linkInd); 212 | float pseudoRandWeight = -0.5 + (float)rand()/((float)RAND_MAX/(0.5 + 0.5)); 213 | //float pseudoRandWeight = 0; 214 | currentNeuralLink->SetWeight(pseudoRandWeight); 215 | 216 | //std::cout << "layerInd: " << layerInd << ", neuronInd: " << neuronInd << ", linkInd: " << linkInd << ", Weight: " << currentNeuralLink->GetWeight() << std::endl; 217 | 218 | } 219 | } 220 | } 221 | for(unsigned int layerInd = 0; layerInd < mNeuralNetwork->size() - 1; layerInd++){ 222 | 223 | Neuron * Bias = mNeuralNetwork->GetBiasLayer().at(layerInd); 224 | for(int linkInd = 0; linkInd < Bias->GetNumOfLinks(); linkInd++){ 225 | NeuralLink * currentNeuralLink = Bias->at(linkInd); 226 | float pseudoRandWeight = -0.5 + (float)rand()/((float)RAND_MAX/(0.5 + 0.5)); 227 | //float pseudoRandWeight = 0; 228 | currentNeuralLink->SetWeight(pseudoRandWeight); 229 | 230 | //std::cout << "layerInd Bias: " << layerInd << ", linkInd: " << linkInd << ", Weight: " << currentNeuralLink->GetWeight() << std::endl; 231 | 232 | } 233 | } 234 | } 235 | 236 | template 237 | void Backpropagation::WeightsInitialization() 238 | { 239 | this->NguyenWidrowWeightsInitialization(); 240 | 241 | } 242 | 243 | template 244 | Backpropagation::Backpropagation(NeuralNetwork * inNeuralNetwork) 245 | { 246 | mNeuralNetwork = inNeuralNetwork; 247 | //this->WeightsInitialization(); 248 | 249 | 250 | } 251 | 252 | 253 | /*template 254 | Backpropagation::Backpropagation(NeuralNetwork * inNeuralNetwork) 255 | { 256 | mNeuralNetwork = inNeuralNetwork; 257 | 258 | 259 | * Step 0. Initialize weights ( Set to small values ) 260 | 261 | srand((unsigned)time(0)); 262 | 263 | 264 | 265 | * For every layer, for every neuron and bias in that layer, for every link in that neuron, set the weight 266 | * to random number from 0 to 1; 267 | * 268 | 269 | 270 | for(unsigned int layerInd = 0; layerInd < mNeuralNetwork->size(); layerInd++){ 271 | for(unsigned int neuronInd = 0; neuronInd < mNeuralNetwork->GetLayer(layerInd).size(); neuronInd++){ 272 | Neuron * currentNeuron = mNeuralNetwork->GetLayer(layerInd).at(neuronInd); 273 | for(int linkInd = 0; linkInd < currentNeuron->GetNumOfLinks(); linkInd++){ 274 | NeuralLink * currentNeuralLink = currentNeuron->at(linkInd); 275 | float pseudoRandWeight = -0.5 + (float)rand()/((float)RAND_MAX/(0.5 + 0.5)); 276 | //float pseudoRandWeight = 0; 277 | currentNeuralLink->SetWeight(pseudoRandWeight); 278 | 279 | //std::cout << "layerInd: " << layerInd << ", neuronInd: " << neuronInd << ", linkInd: " << linkInd << ", Weight: " << currentNeuralLink->GetWeight() << std::endl; 280 | 281 | } 282 | } 283 | } 284 | for(unsigned int layerInd = 0; layerInd < mNeuralNetwork->size() - 1; layerInd++){ 285 | 286 | Neuron * Bias = mNeuralNetwork->GetBiasLayer().at(layerInd); 287 | for(int linkInd = 0; linkInd < Bias->GetNumOfLinks(); linkInd++){ 288 | NeuralLink * currentNeuralLink = Bias->at(linkInd); 289 | float pseudoRandWeight = -0.5 + (float)rand()/((float)RAND_MAX/(0.5 + 0.5)); 290 | //float pseudoRandWeight = 0; 291 | currentNeuralLink->SetWeight(pseudoRandWeight); 292 | 293 | //std::cout << "layerInd Bias: " << layerInd << ", linkInd: " << linkInd << ", Weight: " << currentNeuralLink->GetWeight() << std::endl; 294 | 295 | } 296 | } 297 | }*/ 298 | 299 | 300 | template 301 | double Backpropagation::Train(const std::vector& inData, const std::vector& inTarget) 302 | { 303 | /* 304 | * Check incoming data 305 | */ 306 | 307 | double result = 0; 308 | if( inData.size() != mNeuralNetwork->mInputs || inTarget.size() != mNeuralNetwork->mOutputs ){ 309 | std::cout << "Input data dimensions are wrong, expected: " << mNeuralNetwork->mInputs << " elements\n"; 310 | 311 | return -1; 312 | } 313 | else{ 314 | 315 | /* 316 | * Step 3. Feedforward: Each input unit receives input signal and 317 | * broadcast this signal to all units in the layer above (the hidden units) 318 | */ 319 | 320 | for(unsigned int indexOfData = 0; indexOfData < mNeuralNetwork->mInputs; indexOfData++){ 321 | //std::cout << "input" << indexOfData << ": " << inData[indexOfData] << std::endl; 322 | mNeuralNetwork->GetInputLayer().at(indexOfData)->Input(inData[indexOfData]); 323 | } 324 | 325 | 326 | for(unsigned int numOfLayer = 0; numOfLayer < mNeuralNetwork->size() - 1; numOfLayer++){ 327 | mNeuralNetwork->GetBiasLayer().at(numOfLayer)->Input(1.0); 328 | //std::cout << "BiasInput" << std::endl; 329 | //std::cout << "Layer: " << numOfLayer << std::endl; 330 | for(unsigned int indexOfNeuronInLayer = 0; indexOfNeuronInLayer < mNeuralNetwork->GetLayer(numOfLayer).size(); indexOfNeuronInLayer++){ 331 | //std::cout << "IndexOfNeuron: " << indexOfNeuronInLayer << std::endl; 332 | mNeuralNetwork->GetLayer(numOfLayer).at(indexOfNeuronInLayer)->Fire(); 333 | } 334 | //std::cout << "Bias: " << numOfLayer << std::endl; 335 | mNeuralNetwork->GetBiasLayer().at(numOfLayer)->Fire(); 336 | for(int i = 0; i < mNeuralNetwork->GetBiasLayer().at(numOfLayer)->GetNumOfLinks(); i++){ 337 | mNeuralNetwork->GetBiasLayer().at(numOfLayer)->GetLinksToNeurons().at(i)->SetLastTranslatedSignal(1); 338 | } 339 | } 340 | 341 | /* 342 | * Step 5. Each output unit applies its activation function to compute its output 343 | * signal. 344 | */ 345 | 346 | 347 | std::vector netResponseYk; 348 | for(unsigned int indexOfOutputElements = 0; indexOfOutputElements < mNeuralNetwork->mOutputs; indexOfOutputElements++){ 349 | 350 | double Yk = mNeuralNetwork->GetOutputLayer().at(indexOfOutputElements)->Fire(); 351 | netResponseYk.push_back(Yk); 352 | 353 | } 354 | 355 | /* 356 | * Step 6. Backpropagation of error 357 | * Computing error information for each output unit. 358 | */ 359 | 360 | for(unsigned int indexOfData = 0; indexOfData < mNeuralNetwork->mOutputs; indexOfData++){ 361 | result = mNeuralNetwork->GetOutputLayer().at(indexOfData)->PerformTrainingProcess(inTarget[indexOfData]); 362 | mNeuralNetwork->AddMSE(result); 363 | } 364 | 365 | 366 | /* 367 | * FIXME: Net should perform training process not only for last layer and layer before last, but also for any 368 | * layers except input one, so fix it DUDE! 369 | */ 370 | 371 | for(unsigned int iIndOfLayer = mNeuralNetwork->size() - 2; iIndOfLayer > 0 ; iIndOfLayer--){ 372 | for(unsigned int indexOfNeuron = 0; indexOfNeuron < mNeuralNetwork->GetLayer(iIndOfLayer).size(); indexOfNeuron++){ 373 | mNeuralNetwork->GetLayer(iIndOfLayer).at(indexOfNeuron)->PerformTrainingProcess(0); 374 | } 375 | } 376 | mNeuralNetwork->UpdateWeights(); 377 | 378 | 379 | mNeuralNetwork->ResetCharges(); 380 | return result; 381 | } 382 | } 383 | 384 | 385 | template class Backpropagation; 386 | template class Backpropagation; 387 | template class Backpropagation; 388 | --------------------------------------------------------------------------------