├── LICENSE
├── load_mnist.h
├── main.cpp
├── README.md
├── NeuralNetwork.h
├── loadcsv.h
└── NeuralNetwork.cpp


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2025 muchlakshay
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/load_mnist.h:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <fstream>
 3 | #include <vector>
 4 | #include <Eigen/Dense>
 5 | 
 6 | using namespace Eigen;
 7 | 
 8 | int read_int(std::ifstream& stream) {
 9 |     unsigned char bytes[4];
10 |     stream.read(reinterpret_cast<char*>(bytes), 4);
11 |     return (int(bytes[0]) << 24) | (int(bytes[1]) << 16) | (int(bytes[2]) << 8) | int(bytes[3]);
12 | }
13 | 
14 | MatrixXd load_mnist_images(const std::string& filename, int& num_images, int& rows, int& cols) {
15 |     std::ifstream file(filename, std::ios::binary);
16 |     if (!file.is_open()) throw std::runtime_error("Cannot open image file!");
17 | 
18 |     int magic = read_int(file);
19 |     num_images = read_int(file);
20 |     rows = read_int(file);
21 |     cols = read_int(file);
22 | 
23 |     MatrixXd images(num_images, rows * cols);
24 | 
25 |     for (int i = 0; i < num_images; ++i) {
26 |         for (int j = 0; j < rows * cols; ++j) {
27 |             unsigned char pixel = 0;
28 |             file.read(reinterpret_cast<char*>(&pixel), 1);
29 |             images(i, j) = static_cast<double>(pixel) / 255.0; // normalize
30 |         }
31 |     }
32 | 
33 |     return images;
34 | }
35 | 
36 | VectorXd load_mnist_labels(const std::string& filename, int& num_labels) {
37 |     std::ifstream file(filename, std::ios::binary);
38 |     if (!file.is_open()) throw std::runtime_error("Cannot open label file!");
39 | 
40 |     int magic = read_int(file);
41 |     num_labels = read_int(file);
42 | 
43 |     VectorXd labels(num_labels);
44 | 
45 |     for (int i = 0; i < num_labels; ++i) {
46 |         unsigned char label = 0;
47 |         file.read(reinterpret_cast<char*>(&label), 1);
48 |         labels(i) = static_cast<int>(label);
49 |     }
50 | 
51 |     return labels;
52 | }
53 | 
54 | //one hot encode labels
55 | MatrixXd toOneHot(VectorXd& labels, int num_labels){
56 |     MatrixXd one_hot {MatrixXd::Zero(labels.rows(), num_labels)};
57 |     for (std::size_t i {}; i<labels.rows(); ++i){
58 |         one_hot(i, static_cast<int>(labels(i))) = 1.0;
59 |     }
60 |     return one_hot;
61 | }
62 | 


--------------------------------------------------------------------------------
/main.cpp:
--------------------------------------------------------------------------------
 1 | #include "NeuralNetwork.h"
 2 | #include "load_mnist.h"
 3 | #include "loadcsv.h"
 4 | 
 5 | int main(){
 6 | 
 7 | //--------------LOAD CSV FILES------------------------
 8 | 
 9 |     // Data data {load_csv("filename.csv", "label_column", 0.8)}; //--> change training data ratio from 
10 |                                                                    // 0.8 to something else if you want
11 | 
12 |     // MatrixXd X_train {toMatrixXd(data.X_train)};
13 |     // MatrixXd X_test {toMatrixXd(data.X_test)};
14 |     // VectorXd Y_train {toVectorXd(data.Y_train)};
15 |     // VectorXd Y_test {toVectorXd(data.Y_test)};
16 | 
17 | //you can use toOneHot() to one hot encode the labels
18 | 
19 | //--------------LOAD MNIST----------------------------
20 |     
21 |     int num_images_train, rows_train, cols_train, num_labels_train;
22 |     int num_images_test, rows_test, cols_test, num_labels_test;
23 | 
24 |     //change path to where your MNIST data is 
25 |     MatrixXd train_images = load_mnist_images("MNIST/train-images.idx3-ubyte", num_images_train, rows_train, cols_train);
26 |     MatrixXd test_images = load_mnist_images("MNIST/t10k-images.idx3-ubyte", num_images_test, rows_test, cols_test);
27 |     VectorXd train_labels = load_mnist_labels("MNIST/train-labels.idx1-ubyte", num_labels_train);
28 |     VectorXd test_labels = load_mnist_labels("MNIST/t10k-labels.idx1-ubyte", num_labels_test);
29 | 
30 |     MatrixXd train_labels_oh {toOneHot(train_labels, 10)};
31 |     MatrixXd test_labels_oh {toOneHot(test_labels, 10)};
32 | 
33 |     NeuralNetwork nn {{784, 16, 16, 10}, {"relu", "relu", "softmax"}};
34 |     nn.learn(train_images, train_labels_oh, 30, 0.01, 32, "cross_entropy_loss", true);
35 | 
36 |     MatrixXd pred {nn.predict(test_images)};
37 | 
38 |     for (int i {}; i<test_labels_oh.rows(); ++i){
39 |         std::cout<<"Actual: "<<test_labels_oh.row(i)<<" Predicted: ";
40 |         for (int j {}; j<10; ++j){
41 |             std::cout<<std::round(pred(i,j))<<" ";
42 |         }
43 |         std::cout<<std::endl;
44 |     }
45 | 
46 |     std::cout<<"Accuracy: "<<accuracy(test_labels_oh, pred);
47 | 
48 |     return 0;
49 | }
50 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # MLP-From-Scratch
 2 | A C++ implementation of a Multilayer Perceptron (MLP) neural network using Eigen, supporting multiple activation and loss functions, mini-batch gradient descent, and backpropagation for training.
 3 | 
 4 | ## Features
 5 | 
 6 | - **Customizable Architecture**: You can define the number of layers and the number of neurons per layer.
 7 | - **Multiple Activation Functions**: Supports ReLU, Sigmoid, Tanh, Leaky Relu and Softmax for non-linearity.
 8 | - **Loss Functions**: Supports Mean Squared Error (MSE), Binary Cross Entropy and Cross-Entropy loss functions for regression and classification tasks.
 9 | - **Backpropagation**: Implements the backpropagation algorithm for training the network.
10 | - **Mini-batch Gradient Descent**: Optimizes the network using mini-batch gradient descent, allowing for more efficient training on large datasets.
11 | - **Modular Design**: The code is organized in a way that allows for easy extension and modification (e.g., adding new activation functions, loss functions, etc.).
12 | 
13 | ## Prerequisites
14 | 
15 | - C++11 or later
16 | - [Eigen](https://eigen.tuxfamily.org/) (header-only C++ library for linear algebra)
17 | 
18 | ## Getting Started
19 | 
20 | ### Clone the repository
21 | 
22 | ```bash
23 | git clone https://github.com/muchlakshay/MLP-From-Scratch.git
24 | cd MLP-From-Scratch
25 | ```
26 | ### Compile and run
27 | To compile and run the project, use a C++ compiler that supports C++11 or later (e.g., g++, clang++):
28 | 
29 | ```bash
30 | g++  main.cpp NeuralNetwork.cpp -o NN -I/path/to/eigen
31 | ./NN
32 | ```
33 | Replace /path/to/eigen with the actual path where Eigen is installed or located.
34 | 
35 | ## Training
36 | - The network can be trained on your own dataset. You can define the input data (features) and target data (labels) in the form of Eigen MatrixXd by loading csv data file using "loadcsv.h" header file, specify the number of epochs, mini-batch size, and learning rate.
37 | ```cpp
38 | NeuralNetwork mlp({784, 16, 16, 10}, {"relu", "relu", "softmax"});  // 784 input neurons, 2 hidden layer with 16 neurons and 10 output neuron
39 | mlp.learn(input_data, labels, 1000, 32, 0.01, "cross_entropy_loss", true); // input data, labels, batch size, learning rate, loss function, verbose (true by default)
40 | ```
41 | ## Test Run On MNIST 
42 | ![Screenshot 2025-04-22 114559](https://github.com/user-attachments/assets/a934e0be-2e67-4015-aada-24e67a29b6f4)
43 | 
44 | - 95% accuracy on MNIST handwritten digit classification with just 20 epochs
45 | 


--------------------------------------------------------------------------------
/NeuralNetwork.h:
--------------------------------------------------------------------------------
 1 | #include <Eigen/Dense>
 2 | #include <iostream>
 3 | #include <random>
 4 | #include <functional>
 5 | #include <cmath>
 6 | 
 7 | using namespace Eigen;
 8 | 
 9 | class NeuralNetwork{
10 |     struct ActivationFunction{
11 |         std::function<double(double)> fn;
12 |         std::function<double(double)> derivative;
13 |     };
14 |     
15 |     struct LossFunction{
16 |         std::function<VectorXd(MatrixXd, MatrixXd)> loss;
17 |         std::function<MatrixXd(MatrixXd, MatrixXd)> derivative;
18 |     };
19 | 
20 |     std::vector<MatrixXd> weights;
21 |     std::vector<VectorXd> bias;
22 |     std::vector<MatrixXd> weights_gradients;
23 |     std::vector<VectorXd> bias_gradients;
24 |     std::vector<MatrixXd> layers_output;
25 |     std::vector<MatrixXd> layers_activations;
26 |     std::vector<MatrixXd> layers_errors;
27 | 
28 |     double learning_rate {};
29 | 
30 |     const std::vector<int> layers;
31 |     std::vector<std::string> activation_function;
32 | 
33 |     void initialize(const std::vector<int>& layers);
34 |     ActivationFunction getFunction(int layer);
35 |     LossFunction getLossFunction(const std::string& loss);
36 |     double cross_entropy_loss(const MatrixXd& y, const MatrixXd& y_hat);
37 |     MatrixXd softmax(const MatrixXd& z);
38 | 
39 |     void shuffle(const MatrixXd& non_shuffled_features, MatrixXd& store_shuffled_features, 
40 |                  const MatrixXd& non_shuffled_labels, MatrixXd& store_shuffled_labels);
41 | 
42 |     void feedForward(const MatrixXd& batch);
43 |     void backpropagate(const MatrixXd& batch, const MatrixXd& labels, int batch_size, const std::string& loss_function);
44 |     void updateWeights();
45 | 
46 |     double computeLoss(const MatrixXd& outputs, const MatrixXd& labels, int batch_size, const std::string& loss_function);
47 | 
48 | public:
49 | 
50 |     NeuralNetwork(const std::vector<int>& layers, const std::vector<std::string>& activation_function)
51 |     :layers{layers}, activation_function {activation_function}{
52 |         initialize(layers);
53 |     }
54 | 
55 |     void learn(const MatrixXd& training_data, const MatrixXd& training_labels, int epochs,
56 |         double eta, int batch_size, const std::string& loss_function, bool verbose=true);
57 | 
58 |     MatrixXd predict(const MatrixXd& to_predict);
59 |     const std::vector<MatrixXd>& getWeights() const {return weights;}
60 | };
61 | 
62 | double accuracy(const MatrixXd& labels, const MatrixXd& predicted);
63 | 
64 | 
65 | 
66 | 


--------------------------------------------------------------------------------
/loadcsv.h:
--------------------------------------------------------------------------------
  1 | //this module for loading csv files is inefficient, it first extract data into 2d std::vectors
  2 | //and then convert it to Eigen MatrixXd. While i can write it to directly write into Eigen MatrixXs
  3 | //but i am lazy!!
  4 | 
  5 | #pragma once
  6 | 
  7 | #include "loadcsv.h"
  8 | #include <iostream>
  9 | #include <fstream>
 10 | #include <vector>
 11 | #include <sstream>
 12 | 
 13 | struct Data {
 14 |     std::vector<std::string> header;
 15 |     std::vector<std::vector<double>> X_train;
 16 |     std::vector<double> Y_train;
 17 |     std::vector<std::vector<double>> X_test;
 18 |     std::vector<double> Y_test;
 19 |     
 20 | };
 21 | 
 22 | //load data line by line from a csv file
 23 | Data load_csv(const std::string& filename, const std::string& target_column, double training_ratio){
 24 |     std::ifstream file (filename);
 25 |     std::string line;
 26 |     std::vector<std::string> header;
 27 | 
 28 |     Data data;
 29 | 
 30 |     int count {1};
 31 | 
 32 |     //runs for every line
 33 |     while(std::getline(file, line)){
 34 |         std::stringstream str_stream (line);
 35 |         std::string temp_str;
 36 | 
 37 |         //extract column names 
 38 |         if (count==1) {
 39 | 
 40 |             while (std::getline(str_stream,temp_str,',')){
 41 |                 data.header.push_back(temp_str);
 42 |             }
 43 |             ++count; 
 44 |             continue;
 45 |         }
 46 | 
 47 |         std::vector<double> temp_vec;
 48 | 
 49 |         //extract data rows and store it in temp_vec
 50 |         while (std::getline(str_stream, temp_str, ',')){
 51 |             if (temp_str.empty()||temp_str==" "){
 52 |                 temp_vec.push_back(0.0);
 53 |             }
 54 |             else {
 55 |                 temp_vec.push_back(std::stod(temp_str));
 56 |             }
 57 |         }
 58 | 
 59 |         //data row is added in data vector
 60 |         (data.X_train).push_back(temp_vec);
 61 |     }
 62 |     file.close();
 63 | 
 64 |     //extract the index of target column i.e column to predict
 65 |     int target_index {};
 66 |     for (const auto& column : data.header){
 67 |         if (column==target_column){
 68 |             break;
 69 |         }
 70 |         ++target_index;
 71 |     }
 72 | 
 73 |     //copies target column in Y_train and deletes it from X_train
 74 |     for (auto& row : data.X_train){
 75 |         data.Y_train.push_back(row[target_index]);
 76 |         row.erase(row.begin()+target_index);
 77 |     }
 78 | 
 79 |     //Split data according to the ratio given as argument into X_tain, X_test, Y_train and Y_test
 80 |     //ik its inefficient
 81 |     int training_size {static_cast<int>(data.X_train.size()*training_ratio)};
 82 |     for (int i = data.X_train.size(); i>training_size; --i){
 83 |         data.X_test.push_back(data.X_train[i-1]);
 84 |         data.Y_test.push_back(data.Y_train[i-1]);
 85 |         data.X_train.erase(data.X_train.begin()+i);
 86 |         data.Y_train.erase(data.Y_train.begin()+i);
 87 |     }
 88 | 
 89 |     return data;
 90 | }
 91 | 
 92 | //convert 2d std::vector to Eigen MatrixXd;
 93 | Eigen::MatrixXd toMatrixXd(std::vector<std::vector<double>>& data){
 94 |     Eigen::MatrixXd mat (data.size(), data[0].size());
 95 |     for (std::size_t i {}; i<data.size(); ++i){
 96 |         mat.row(i) = Eigen::Map<Eigen::RowVectorXd>(data[i].data(), data[i].size());
 97 |     }
 98 |     return mat;
 99 | }
100 | 
101 | //convert std::vector to Eigen VectorXd;
102 | Eigen::VectorXd toVectorXd(std::vector<double>& data){   
103 |     return Eigen::Map<Eigen::VectorXd>(data.data(), data.size());
104 | }
105 | 
106 | //one hot encode labels (for multiclass classification)
107 | Eigen::MatrixXd toOneHot(Eigen::VectorXd& labels, int num_labels){
108 |     Eigen::MatrixXd one_hot {Eigen::MatrixXd::Zero(labels.rows(), num_labels)};
109 |     for (std::size_t i {}; i<labels.rows(); ++i){
110 |         one_hot(i, static_cast<int>(labels(i))) = 1.0;
111 |     }
112 |     return one_hot;
113 | }
114 | 
115 | //noramlize the data
116 | void normalizeMatrixXd(Eigen::MatrixXd& matrix){
117 |     matrix = (matrix.rowwise() - matrix.colwise().mean());
118 |     matrix = matrix.array().rowwise() / matrix.colwise().norm().array();
119 | }
120 | 
121 | 


--------------------------------------------------------------------------------
/NeuralNetwork.cpp:
--------------------------------------------------------------------------------
  1 | #include "NeuralNetwork.h"
  2 | 
  3 | //----------------------MODEL INITIALIZATION----------------------
  4 | 
  5 | void NeuralNetwork::initialize(const std::vector<int>& layers){
  6 | 
  7 |     std::random_device rd;
  8 |     std::mt19937 mt(rd());
  9 | 
 10 |     for (std::size_t l {1}; l < layers.size(); ++l){
 11 |         int in = layers[l-1]; //layer input size
 12 |         int out = layers[l]; //layer output size
 13 | 
 14 |         //xavier xavier initialization
 15 |         double limit = std::sqrt(6.0 / (in + out));
 16 |         std::uniform_real_distribution<double> dist(-limit, limit); //xavier limit
 17 | 
 18 |         //Create a matrix of weights according to the layer input and output size
 19 |         MatrixXd w(out, in);
 20 |         for (int row {}; row < w.rows(); ++row){
 21 |             for (int col {}; col < w.cols(); ++col){
 22 |                 w(row, col) = dist(mt);
 23 |             }
 24 |         }
 25 | 
 26 |         //Push weight matrix of layer into a vector that contains weights of all layers
 27 |         weights.push_back(w);
 28 | 
 29 |         //Zero initialize the bias vector according to layer output size
 30 |         //and push back into vector containing bias of all layers 
 31 |         bias.push_back(VectorXd::Zero(out));
 32 | 
 33 |         //dummy initialize gradients for easy access according to layer index
 34 |         weights_gradients.push_back(MatrixXd::Zero(out, in));
 35 |         bias_gradients.push_back(VectorXd::Zero(out));
 36 | 
 37 |         //dummy initialize layer error for easy access according to layer index
 38 |         layers_errors.push_back(MatrixXd::Zero(1,out));
 39 |     }
 40 | }
 41 | 
 42 | //----------------------GET ACTIVATION FUNCTIONS----------------------
 43 | 
 44 | NeuralNetwork::ActivationFunction NeuralNetwork::getFunction(int layer){
 45 | 
 46 |     //activation function lambda with respective derivatives
 47 | 
 48 |     auto sigmoid {[](double z){return 1.0/(1.0+std::exp(-z));}};
 49 |     auto sigmoid_prime {[](double z){return (1.0/(1.0+std::exp(-z)))*(1.0-(1.0/(1.0+std::exp(-z))));}};
 50 | 
 51 |     auto tanh {[](double z){return std::tanh(z);}};
 52 |     auto tanh_prime{[](double z){return 1-std::tanh(z)*std::tanh(z);}};
 53 | 
 54 |     auto relu {[](double z){return std::max(0.0, z);}};
 55 |     auto relu_prime {[](double z){return z>0?1.0:0.0;}};
 56 | 
 57 |     auto leaky_relu {[](double z){return z>0?z:z*0.01;}};
 58 |     auto leaky_relu_prime {[](double z){return z>0?1.0:0.01;}};
 59 | 
 60 |     auto linear {[](double z){return z;}};
 61 |     auto linear_prime{[](double z){return 1.0;}};
 62 | 
 63 |     //activation function of the layer 
 64 |     std::string fn {activation_function[layer]};
 65 | 
 66 |     //return appropriate activation function and its derivative
 67 |     if (fn=="sigmoid") return {sigmoid, sigmoid_prime};
 68 |     else if (fn=="tanh") return {tanh, tanh_prime};
 69 |     else if (fn=="relu") return {relu, relu_prime};
 70 |     else if (fn=="leaky_relu") return {leaky_relu, leaky_relu_prime};
 71 |     else if (fn=="linear") return {linear, linear_prime};
 72 |     else return {};
 73 | }
 74 | 
 75 | //----------------------GET LOSS FUNCTION----------------------
 76 | 
 77 | NeuralNetwork::LossFunction NeuralNetwork::getLossFunction(const std::string& loss){
 78 | 
 79 |     //loss functions lambdas and their derivatives
 80 | 
 81 |     auto mse = [](const MatrixXd y, const MatrixXd y_hat) -> VectorXd {
 82 |         VectorXd losses(y.rows());
 83 |         for (int i = 0; i < y.rows(); ++i) {
 84 |             losses(i) = 0.5 * (y.row(i) - y_hat.row(i)).squaredNorm(); // squaredNorm = sum of squares
 85 |         }
 86 |         return losses;
 87 |     };
 88 |     auto mse_prime {
 89 |         [] (MatrixXd y, MatrixXd y_hat) -> MatrixXd {
 90 |             return y_hat-y;
 91 |         }
 92 |     };
 93 | 
 94 |     auto binary_cross_entropy{
 95 |         [](MatrixXd y, MatrixXd y_hat) -> VectorXd {
 96 |             for (int row {}; row<y_hat.rows(); ++row){
 97 |                 for (int col {}; col<y_hat.cols(); ++col){
 98 |                     y_hat(row, col) = std::clamp(y_hat(row, col), 1e-7, 1 - 1e-7);
 99 |                 }
100 |             }
101 |             VectorXd losses(y.rows());
102 |             for (int i = 0; i < y.rows(); ++i) {
103 |                 losses(i) = - (y.row(i).array() * y_hat.row(i).array().log() + 
104 |                                 (1 - y.row(i).array()) * (1 - y_hat.row(i).array()).log()).sum();
105 |             }
106 |             return losses;
107 |         }
108 |     };
109 |     auto binary_cross_entropy_prime{
110 |         [](MatrixXd y, MatrixXd y_hat) -> MatrixXd {
111 |             for (int row {}; row<y_hat.rows(); ++row){
112 |                 for (int col {}; col<y_hat.cols(); ++col){
113 |                     y_hat(row, col) = std::clamp(y_hat(row, col), 1e-7, 1 - 1e-7);
114 |                 }
115 |             }
116 |             return (y_hat - y).array() / (y_hat.array() * (1 - y_hat.array()));
117 |         }
118 |     };
119 | 
120 |     //return appropriate loss function and its derivative
121 |     if (loss=="mse") return {mse, mse_prime};
122 |     else if (loss=="binary_cross_entropy") return {binary_cross_entropy, binary_cross_entropy_prime};
123 |     else return {};
124 | }
125 | 
126 | //----------------------SEPARATE FUNCTION FOR CROSS ENTROPY----------------------
127 | 
128 | double NeuralNetwork::cross_entropy_loss(const MatrixXd& y, const MatrixXd& y_hat) {
129 |     const double epsilon = 1e-15;
130 |     MatrixXd y_hat_clamped = y_hat.array().min(1.0 - epsilon).max(epsilon);
131 |     double loss = 0.0;
132 |     for (int i = 0; i < y.rows(); ++i) {
133 |         loss -= (y.row(i).array() * y_hat_clamped.row(i).array().log()).sum();
134 |     }
135 |     return loss;
136 | }
137 | 
138 | //----------------------SEPARATE FUNCTION FOR SOFTMAX----------------------
139 | MatrixXd NeuralNetwork::softmax(const MatrixXd& z){
140 |     MatrixXd result = z;
141 |     for (int i = 0; i < z.rows(); ++i) {
142 |         double max = z.row(i).maxCoeff();
143 |         VectorXd exp_row = (z.row(i).array() - max).exp();
144 |         double sum = exp_row.sum();
145 |         result.row(i) = exp_row / sum;
146 |     }
147 |     return result;
148 | }
149 | 
150 | //----------------------SHUFFLE DATASET----------------------
151 | 
152 | void NeuralNetwork::shuffle(const MatrixXd& non_shuffled_features, MatrixXd& store_shuffled_features, 
153 |     const MatrixXd& non_shuffled_labels, MatrixXd& store_shuffled_labels){
154 | 
155 |     std::random_device rd;
156 |     std::mt19937 mt(rd());
157 | 
158 |     //Indicies of Data  
159 |     std::vector<int> indicies(non_shuffled_features.rows());
160 |     for (std::size_t i {}; i<non_shuffled_features.rows(); ++i){
161 |         indicies[i] = i;
162 |     }
163 | 
164 |     //Shuffle the indicies
165 |     std::shuffle(indicies.begin(), indicies.end(), mt);
166 | 
167 |     //using shuffled indicies store data it another matrix (both data features and labels)  
168 |     for (std::size_t i {}; i<indicies.size(); ++i){
169 |         store_shuffled_features.row(i)=non_shuffled_features.row(indicies[i]);
170 |         store_shuffled_labels.row(i)=non_shuffled_labels.row(indicies[i]);
171 |     }
172 | }
173 | 
174 | //---------------------FEED FORWARD----------------------
175 | 
176 | void NeuralNetwork::feedForward(const MatrixXd& batch){
177 | 
178 |     //Check if batch is of suitable dimensions
179 |     if (batch.cols() != layers[0]){
180 |         std::cout<<"Input data size doesn't match the input layer size.\n";
181 |         return;
182 |     }
183 | 
184 |     //clear the previous batch outputs (z) and activations (a)
185 |     layers_output.clear();
186 |     layers_activations.clear();
187 | 
188 |     //input to layer (initially the batch for first layer)
189 |     MatrixXd input_to_layer {batch};
190 | 
191 |     //Runs for every layer weight matrix 
192 |     for (std::size_t l {}; l<weights.size(); ++l){
193 | 
194 |         //calculate the weighted sum
195 |         MatrixXd weighted_sum {input_to_layer*weights[l].transpose()};
196 |         weighted_sum.rowwise() += bias[l].transpose();
197 | 
198 |         //get suitable activation function according to layer index
199 |         auto func {getFunction(l)};
200 | 
201 |         //layer activations
202 |         MatrixXd activations;
203 | 
204 |         //for output layer if it has softmax activation 
205 |         if (activation_function[l]=="softmax"){
206 |             activations = softmax(weighted_sum);
207 |         }
208 |         //for hidden layers
209 |         else{
210 |             auto func {getFunction(l)};
211 |             activations = weighted_sum.unaryExpr(func.fn);
212 |         }
213 | 
214 |         //Push back the weighted sum (z) into the std::vector contating outputs of all layers
215 |         layers_output.push_back(weighted_sum);
216 | 
217 |         //Push back the activations (a) into the std::vector contating activations of all layers
218 |         layers_activations.push_back(activations);
219 | 
220 |         //Set input for next layer to output of current layer
221 |         input_to_layer = layers_activations[l];
222 |     }
223 | 
224 | }
225 | 
226 | //----------------------BACKPROPAGATION----------------------
227 | 
228 | void NeuralNetwork::backpropagate(const MatrixXd& batch, const MatrixXd& labels, int batch_size, const std::string& loss_function){
229 | 
230 |     //Starts from output layer (-2 because layer data member contains input layer too)
231 |     for (int l {static_cast<int>(layers.size())-2}; l>=0; --l){
232 | 
233 |         //Get appropriate loss and activation function for their derivatives
234 |         auto func {getFunction(l)};
235 |         auto loss {getLossFunction(loss_function)};
236 | 
237 |         //if layer is output layer, calculate error of output layer
238 |         if (l==layers.size()-2){
239 | 
240 |             //number of rows in output layer
241 |             int rows {static_cast<int>(layers_activations.back().rows())};
242 | 
243 |             // ∂C/∂a
244 |             MatrixXd loss_wrt_activations (batch_size, labels.cols());
245 |             // ∂a/∂z
246 |             MatrixXd activations_wrt_output;
247 | 
248 |             //if activation is softmax and loss is cross entropy
249 |             if (activation_function.back()=="softmax" && loss_function=="cross_entropy_loss"){
250 |                 loss_wrt_activations = layers_activations.back()-labels;
251 | 
252 |                 //store error in vector containg errors of all layers
253 |                 layers_errors[l] = loss_wrt_activations;
254 |             }
255 |             //if activation is sigmoid and loss is binary cross entropy
256 |             else if (activation_function.back()=="sigmoid" && loss_function=="binary_cross_entropy"){
257 |                 loss_wrt_activations = layers_activations.back()-labels;
258 |                 layers_errors[l] = loss_wrt_activations;
259 |             }
260 |             //for any other loss and activation
261 |             else {
262 |                 loss_wrt_activations = loss.derivative(labels, layers_activations.back());
263 |                 activations_wrt_output = layers_output.back().unaryExpr(func.derivative);
264 |                 //store error in vector containg errors of all layers
265 |                 layers_errors[l] = loss_wrt_activations.cwiseProduct(activations_wrt_output);
266 |             }
267 |             continue;
268 |         }
269 | 
270 |         //calculate errors of hidden layers and store it in vector containg errors of all layers
271 |         layers_errors[l] = (layers_errors[l+1]*weights[l+1]).cwiseProduct(layers_output[l].unaryExpr(func.derivative));
272 |     }
273 | 
274 |     //runs for each layers's weight matrix
275 |     for (int l {}; l<weights.size(); ++l){
276 | 
277 |         //if output layer
278 |         if (l==0){
279 |             //compute gradients
280 |             weights_gradients[l] = layers_errors[l].transpose()*batch;
281 |             //avegrage them 
282 |             weights_gradients[l]/=batch_size;
283 |         }
284 |         //if hidden layer
285 |         else {
286 |             //compute gradients
287 |             weights_gradients[l] = layers_errors[l].transpose()*layers_activations[l-1];
288 |             //average them
289 |             weights_gradients[l]/=batch_size;
290 |         }
291 |         //computer bias gradients;
292 |         bias_gradients[l] = layers_errors[l].colwise().mean();
293 |     }
294 | } 
295 | 
296 | //----------------------UPDATE WEIGHTS----------------------
297 | 
298 | void NeuralNetwork::updateWeights(){
299 |     for (std::size_t i {}; i<weights.size(); ++i){
300 |         weights[i] -= learning_rate*weights_gradients[i];
301 |         bias[i] -= learning_rate*bias_gradients[i];
302 |     }
303 | }
304 | 
305 | //----------------------LOSS COMPUTATION----------------------
306 | 
307 | double NeuralNetwork::computeLoss(const MatrixXd& outputs, const MatrixXd& labels, int batch_size, const std::string& loss_function){
308 |     double loss {};
309 |     if (loss_function=="cross_entropy_loss"){
310 |         loss = cross_entropy_loss(labels, outputs);
311 |         loss/=batch_size;
312 |         return loss;
313 |     }
314 |     auto loss_func {getLossFunction(loss_function)};
315 |     for (std::size_t i {}; i<batch_size; ++i){
316 |         loss += loss_func.loss(labels, outputs).sum();
317 |     }
318 |     loss/=batch_size;
319 |     return loss;
320 | }
321 | 
322 | //----------------------TRAINING FUNCTION----------------------
323 | 
324 | void NeuralNetwork::learn(const MatrixXd& training_data, const MatrixXd& training_labels, int epochs,
325 |                          double eta, int batch_size, const std::string& loss_function, bool verbose){
326 |     
327 |     //Separate Matricies to store shuffled features and labels 
328 |     MatrixXd training_data_s (training_data.rows(), training_data.cols());
329 |     MatrixXd training_labels_s (training_labels.rows(), training_labels.cols());
330 | 
331 |     learning_rate=eta;
332 | 
333 |     std::cout<<"Learning..\n";
334 | 
335 |     int rows {static_cast<int>(training_data_s.rows())};
336 | 
337 |     //Runs for each epoch
338 |     for (int e {1}; e<=epochs; ++e){
339 | 
340 |         //shuffle features and labels
341 |         shuffle(training_data, training_data_s, training_labels, training_labels_s);
342 |         double loss {};
343 |         
344 |         //Divide whole dataset into batches
345 |         for (int start {}; start<rows; start+=batch_size){
346 | 
347 |             //current_batch_size sets to number remaining rows if rows left are less then batch_size 
348 |             int current_batch_size = std::min(batch_size, rows-start);
349 | 
350 |             //Create Batch of current_batch_size
351 |             MatrixXd batch_features {training_data_s.block(start, 0, current_batch_size, training_data_s.cols())};
352 |             MatrixXd batch_labels {training_labels_s.block(start, 0, current_batch_size, training_labels_s.cols())};
353 | 
354 |             //Feed forward the batch
355 |             feedForward(batch_features);
356 | 
357 |             //Backpropagate the Errors and Calculate Gradients
358 |             backpropagate(batch_features, batch_labels, current_batch_size, loss_function);
359 | 
360 |             //update Weights using gradients;
361 |             updateWeights();
362 | 
363 |             //compute the loss
364 |             loss+=computeLoss(layers_activations.back(), batch_labels, current_batch_size, loss_function);
365 |         }
366 |         if (verbose && e%5==0){
367 |             std::cout<<"Epoch: "<<e<<" Loss: "<<loss<<"\n";
368 |         }
369 |     
370 |     }
371 |     std::cout << "Learning complete!\n";
372 | }
373 | 
374 | //----------------------PREDICTION FUNCTION----------------------
375 | 
376 | MatrixXd NeuralNetwork::predict(const MatrixXd& to_predict){
377 |         feedForward(to_predict);
378 |         return layers_activations.back();
379 | }
380 | 
381 | //----------------------ACCURACY CALCULATION----------------------
382 | double accuracy(const MatrixXd& labels, const MatrixXd& predicted){
383 |     int correct = 0;
384 |     for (int i = 0; i < labels.rows(); ++i){
385 |         Eigen::Index label_index, pred_index;
386 |         labels.row(i).maxCoeff(&label_index);
387 |         predicted.row(i).maxCoeff(&pred_index);
388 |         if (label_index == pred_index) ++correct;
389 |     }
390 |     return correct / static_cast<double>(labels.rows());
391 | }
392 | 


--------------------------------------------------------------------------------