├── .gitignore ├── CMakeLists.txt ├── FindEigen3.cmake ├── LICENSE ├── README.md ├── mnist.cpp ├── nn.cpp ├── nn.h ├── nntest.cpp └── tutorial.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | build 3 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8) 2 | IF(COMMAND CMAKE_POLICY) 3 | CMAKE_POLICY(SET CMP0003 OLD) 4 | ENDIF(COMMAND CMAKE_POLICY) 5 | 6 | PROJECT( nn ) 7 | 8 | SET(CMAKE_BUILD_TYPE "Release") 9 | SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}) 10 | SET(CMAKE_INSTALL_PREFIX ${PROJECT_SOURCE_DIR} CACHE INTERNAL "Prefix prepended to install directories" FORCE) 11 | 12 | # color definitions 13 | IF(NOT WIN32) 14 | STRING(ASCII 27 Esc) 15 | SET(ColourReset "${Esc}[m") 16 | SET(BoldRed "${Esc}[1;31m") 17 | SET(BoldGreen "${Esc}[1;32m") 18 | SET(BoldMagenta "${Esc}[1;35m") 19 | ENDIF() 20 | 21 | FIND_PACKAGE(OpenMP QUIET) 22 | 23 | FIND_PACKAGE(Eigen3 REQUIRED) 24 | 25 | INCLUDE_DIRECTORIES(${EIGEN3_INCLUDE_DIR}) 26 | 27 | ADD_LIBRARY(nn nn.cpp) 28 | 29 | ADD_EXECUTABLE(tutorial tutorial.cpp) 30 | TARGET_LINK_LIBRARIES(tutorial nn) 31 | 32 | ADD_EXECUTABLE(mnist mnist.cpp) 33 | TARGET_LINK_LIBRARIES(mnist nn) 34 | 35 | MESSAGE("") 36 | 37 | # ----- OpenMP ----- 38 | IF (OPENMP_FOUND) 39 | 40 | SET (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 41 | SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 42 | MESSAGE("${BoldGreen}nn will be compiled with OpenMP support. ${ColourReset}" ) 43 | 44 | ELSE() 45 | 46 | MESSAGE("${BoldMagenta}nn will run in single core mode since OpenMP was not found.${ColourReset}" ) 47 | 48 | ENDIF() 49 | 50 | # ----- Testing ----- 51 | OPTION(WITH_GTEST "Download and compile unit tests using googletest. " OFF) 52 | 53 | IF (${WITH_GTEST}) 54 | 55 | ENABLE_TESTING() 56 | 57 | INCLUDE(ExternalProject) 58 | 59 | # ----- Download and build gtest ----- 60 | ExternalProject_Add(googletest 61 | SVN_REPOSITORY "http://googletest.googlecode.com/svn/tags/release-1.7.0" 62 | UPDATE_COMMAND "" 63 | INSTALL_COMMAND "" 64 | ) 65 | 66 | INCLUDE_DIRECTORIES (${CMAKE_CURRENT_BINARY_DIR}/googletest-prefix/src/googletest/include) 67 | LINK_DIRECTORIES (${CMAKE_CURRENT_BINARY_DIR}/googletest-prefix/src/googletest-build) 68 | 69 | ADD_EXECUTABLE(nntest nntest.cpp) 70 | TARGET_LINK_LIBRARIES(nntest nn gtest gtest_main pthread) 71 | 72 | MESSAGE("${BoldGreen}Compiling tests using googletest.${ColourReset}" ) 73 | ADD_TEST( nntest nntest ) 74 | 75 | ELSE() 76 | 77 | MESSAGE("${BoldMagenta}Tests will not be compiled. Run cmake with -DWITH_GTEST=ON to compile unit tests.${ColourReset}" ) 78 | 79 | ENDIF() 80 | 81 | MESSAGE("") 82 | -------------------------------------------------------------------------------- /FindEigen3.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find Eigen3 lib 2 | # 3 | # This module supports requiring a minimum version, e.g. you can do 4 | # find_package(Eigen3 3.1.2) 5 | # to require version 3.1.2 or newer of Eigen3. 6 | # 7 | # Once done this will define 8 | # 9 | # EIGEN3_FOUND - system has eigen lib with correct version 10 | # EIGEN3_INCLUDE_DIR - the eigen include directory 11 | # EIGEN3_VERSION - eigen version 12 | 13 | # Copyright (c) 2006, 2007 Montel Laurent, 14 | # Copyright (c) 2008, 2009 Gael Guennebaud, 15 | # Copyright (c) 2009 Benoit Jacob 16 | # Redistribution and use is allowed according to the terms of the 2-clause BSD license. 17 | 18 | if(NOT Eigen3_FIND_VERSION) 19 | if(NOT Eigen3_FIND_VERSION_MAJOR) 20 | set(Eigen3_FIND_VERSION_MAJOR 2) 21 | endif(NOT Eigen3_FIND_VERSION_MAJOR) 22 | if(NOT Eigen3_FIND_VERSION_MINOR) 23 | set(Eigen3_FIND_VERSION_MINOR 91) 24 | endif(NOT Eigen3_FIND_VERSION_MINOR) 25 | if(NOT Eigen3_FIND_VERSION_PATCH) 26 | set(Eigen3_FIND_VERSION_PATCH 0) 27 | endif(NOT Eigen3_FIND_VERSION_PATCH) 28 | 29 | set(Eigen3_FIND_VERSION "${Eigen3_FIND_VERSION_MAJOR}.${Eigen3_FIND_VERSION_MINOR}.${Eigen3_FIND_VERSION_PATCH}") 30 | endif(NOT Eigen3_FIND_VERSION) 31 | 32 | macro(_eigen3_check_version) 33 | file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/src/Core/util/Macros.h" _eigen3_version_header) 34 | 35 | string(REGEX MATCH "define[ \t]+EIGEN_WORLD_VERSION[ \t]+([0-9]+)" _eigen3_world_version_match "${_eigen3_version_header}") 36 | set(EIGEN3_WORLD_VERSION "${CMAKE_MATCH_1}") 37 | string(REGEX MATCH "define[ \t]+EIGEN_MAJOR_VERSION[ \t]+([0-9]+)" _eigen3_major_version_match "${_eigen3_version_header}") 38 | set(EIGEN3_MAJOR_VERSION "${CMAKE_MATCH_1}") 39 | string(REGEX MATCH "define[ \t]+EIGEN_MINOR_VERSION[ \t]+([0-9]+)" _eigen3_minor_version_match "${_eigen3_version_header}") 40 | set(EIGEN3_MINOR_VERSION "${CMAKE_MATCH_1}") 41 | 42 | set(EIGEN3_VERSION ${EIGEN3_WORLD_VERSION}.${EIGEN3_MAJOR_VERSION}.${EIGEN3_MINOR_VERSION}) 43 | if(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 44 | set(EIGEN3_VERSION_OK FALSE) 45 | else(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 46 | set(EIGEN3_VERSION_OK TRUE) 47 | endif(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 48 | 49 | if(NOT EIGEN3_VERSION_OK) 50 | 51 | message(STATUS "Eigen3 version ${EIGEN3_VERSION} found in ${EIGEN3_INCLUDE_DIR}, " 52 | "but at least version ${Eigen3_FIND_VERSION} is required") 53 | endif(NOT EIGEN3_VERSION_OK) 54 | endmacro(_eigen3_check_version) 55 | 56 | if (EIGEN3_INCLUDE_DIR) 57 | 58 | # in cache already 59 | _eigen3_check_version() 60 | set(EIGEN3_FOUND ${EIGEN3_VERSION_OK}) 61 | 62 | else (EIGEN3_INCLUDE_DIR) 63 | 64 | find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library 65 | PATHS 66 | ${CMAKE_INSTALL_PREFIX}/include 67 | ${KDE4_INCLUDE_DIR} 68 | PATH_SUFFIXES eigen3 eigen 69 | ) 70 | 71 | if(EIGEN3_INCLUDE_DIR) 72 | _eigen3_check_version() 73 | endif(EIGEN3_INCLUDE_DIR) 74 | 75 | include(FindPackageHandleStandardArgs) 76 | find_package_handle_standard_args(Eigen3 DEFAULT_MSG EIGEN3_INCLUDE_DIR EIGEN3_VERSION_OK) 77 | 78 | mark_as_advanced(EIGEN3_INCLUDE_DIR) 79 | 80 | endif(EIGEN3_INCLUDE_DIR) 81 | 82 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Manuel Blum 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Multi-layer perceptrons using RPROP 2 | =================================== 3 | 4 | **nn** is a lightweight neural network library using resilient propagation for adapting the weights 5 | 6 | Installation 7 | ------------ 8 | 9 | **nn** was tested on Ubuntu, Arch Linux and MacOS 10 | 11 | * install [CMake](http://http://www.cmake.org/), [Eigen3](http://eigen.tuxfamily.org/) and [subversion](http://subversion.tigris.org/). On Ubuntu this is done as follows: 12 | 13 | sudo apt-get install cmake subversion libeigen3-dev 14 | 15 | * clone the **nn** repository or download it [here](https://bitbucket.org/mblum/nn/get/master.tar.gz) 16 | * change to the **nn** directory and create a build folder 17 | 18 | cd path/to/nn 19 | mkdir build 20 | 21 | * run cmake from within the build folder and compile the library using make 22 | 23 | cd build 24 | cmake .. 25 | make 26 | 27 | * run the example code 28 | 29 | ./tutorial 30 | 31 | * to compile unit tests for **nn**, run cmake with the option `-DWITH_GTEST=ON` 32 | 33 | cmake .. -DWITH_GTEST=ON 34 | make 35 | make test 36 | 37 | 38 | License 39 | ------- 40 | 41 | **nn** is free software, licensed under the BSD license. A copy of this license is distributed with the software. 42 | 43 | Usage of the library 44 | -------------------- 45 | 46 | The source code for this tutorial can be found in `tutorial.cpp`. 47 | 48 | ### Preparing your data 49 | 50 | Organize your training data into a *(m x n_input)* matrix containing the training inputs. Each row of this matrix corresponds to a training sample and each column to a feature. Prepare a matrix of size *(m x n_output)* containing the target values, where *n_output* is the number of dimensions of the output. 51 | 52 | matrix_t X(m, n_input); 53 | matrix_t Y(m, n_output); 54 | 55 | // fill with data 56 | 57 | ### Initializing the neural network 58 | 59 | This neural network implementation only supports fully connected feed forward multi-layer perceptrons (MLPs) with sigmoidal activation functions. The neurons are organized into *k* layers. There is at least one input layer, one output layer and an arbitrary number of hidden layers. Each neuron has outgoing connections to all neurons in the subsequent layer. The number of neurons in the input and the output layer is given by the dimensionality of the training data. After specifying the network topology you can create the `NeuralNet` object. The weights will be initialized randomly. 60 | 61 | Eigen::VectorXi topo(k); 62 | topo << n_input, n1, n2, ..., n_output; 63 | 64 | // initialize a neural network with given topology 65 | NeuralNet nn(topo); 66 | 67 | ### Scaling the data 68 | 69 | When working with MLPs you should always scale your data, such that all the features are in the same range and the output values are between 0 and 1. You can do this by passing your training data to the `autoscale` function, which computes the optimal mapping. After calling `autoscale` this mapping will be performed automatically, so you only have to do this once. To reset the scaling parameters to standard values call `autoscale_reset`. 70 | 71 | nn.autoscale(X,Y); 72 | 73 | ### Training the network 74 | 75 | Alternate between computing the quadratic loss of the MLP and adapting the parameters until the loss converges. You can also specify a regularization parameter *lambda*, which punishes large weights and thereby avoids overfitting. 76 | 77 | for (int i = 0; i < max_steps; ++i) { 78 | err = nn.loss(X, Y, lambda); 79 | nn.rprop(); 80 | } 81 | 82 | ### Making predictions 83 | 84 | If you trained a model, you can make predictions on new data by passing it through the network and observe the activation on the output layer. 85 | 86 | nn.forward_pass(X_test); 87 | matrix_t Y_test = nn.get_activation(); 88 | 89 | ### Reading and writing models to disk 90 | 91 | You can read and write MLPs to binary files. 92 | 93 | // write model to disk 94 | nn.write(filename); 95 | 96 | // read model from disk 97 | NeuralNet nn(filename); 98 | 99 | ### Changing the floating number precision 100 | 101 | **nn** uses double precision floats by default. You can change this behaviour in the file `nn.h`. 102 | 103 | #define F_TYPE double 104 | 105 | ### MNIST dataset 106 | 107 | In order to test **nn** on the MNIST dataset, download the dataset from [here](http://yann.lecun.com/exdb/mnist/) and run the `mnist` tool. 108 | 109 | ./mnist path/to/data 110 | 111 | The tool will train a MLP with two hidden layers, containing 300 and 100 neurons respectively connected by 266.610 weights. Using this setup error rates below 5% are accomplished on the test dataset. 112 | 113 | ### Make nn run in parallel ### 114 | 115 | Some algorithms of the Eigen library can exploit the multiple cores present in your hardware. This will happen automatically, if your compiler supports it. You can control the number of threads that will be used using by setting the OpenMP OMP_NUM_THREADS environment variable. 116 | 117 | OMP_NUM_THREADS=n ./my_program 118 | 119 | ### Using **nn** in your own project 120 | 121 | Just copy `nn.h` and `nn.cpp` into your workspace, make sure that the `Eigen` headers are found and start coding! 122 | -------------------------------------------------------------------------------- /mnist.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Manuel Blum 2 | // All rights reserved. 3 | 4 | // Define this symbol to enable runtime tests for allocations 5 | //#define EIGEN_RUNTIME_NO_MALLOC 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "nn.h" 14 | 15 | inline void swap(int &val) 16 | { 17 | val = (val<<24) | ((val<<8) & 0x00ff0000) | ((val>>8) & 0x0000ff00) | (val>>24); 18 | } 19 | 20 | matrix_t read_mnist_images(std::string filename) 21 | { 22 | matrix_t X; 23 | std::ifstream fs(filename.c_str(), std::ios::binary); 24 | if(fs) { 25 | int magic_number, num_images, num_rows, num_columns; 26 | fs.read((char*)&magic_number, sizeof(magic_number)); 27 | fs.read((char*)&num_images, sizeof(num_images)); 28 | fs.read((char*)&num_rows, sizeof(num_rows)); 29 | fs.read((char*)&num_columns, sizeof(num_columns)); 30 | if (magic_number != 2051) { 31 | swap(magic_number); 32 | swap(num_images); 33 | swap(num_rows); 34 | swap(num_columns); 35 | } 36 | 37 | X = matrix_t::Zero(num_images, num_rows*num_columns); 38 | 39 | for (size_t i=0; i 5 | #include 6 | #include 7 | 8 | #include "nn.h" 9 | 10 | 11 | // default parameters for Rprop 12 | const RpropParams NeuralNet::p = {0.1, 50, 1e-6, 0.5, 1.2}; 13 | 14 | NeuralNet::NeuralNet(Eigen::VectorXi &topology) { 15 | assert(topology.size()>1); 16 | init_layer(topology); 17 | init_weights(0.5); 18 | autoscale_reset(); 19 | } 20 | 21 | NeuralNet::NeuralNet(const char *filename) { 22 | std::ifstream fs(filename, std::ios::in | std::ios::binary); 23 | if (fs) { 24 | // number of layers 25 | int num_layers; 26 | fs.read((char *)&num_layers, sizeof(int)); 27 | Eigen::VectorXi topology(num_layers); 28 | // topology 29 | fs.read((char *) topology.data(), topology.rows() * sizeof(int) ); 30 | init_layer(topology); 31 | autoscale_reset(); 32 | // scaling parameters 33 | fs.read((char *) Xscale.data(), Xscale.size() * sizeof(F_TYPE)); 34 | fs.read((char *) Xshift.data(), Xshift.size() * sizeof(F_TYPE)); 35 | fs.read((char *) Yscale.data(), Yscale.size() * sizeof(F_TYPE)); 36 | fs.read((char *) Yshift.data(), Yshift.size() * sizeof(F_TYPE)); 37 | // weights 38 | for (int i=1; i