├── BPNN.hpp
├── CNN.hpp
├── CPPCON_CN_Workspace.zip
├── LSTM.hpp
├── README.md
├── RNN.hpp
├── RNN_N.hpp
├── Samples.cpp
├── include
    ├── BPNN.inl
    ├── CNN.inl
    ├── LSTM.inl
    ├── Parameter.hpp
    ├── RNN.inl
    └── RNN_N.inl
├── math
    ├── Matrix.hpp
    └── sigfunc.h
└── util
    ├── TupleTool.hpp
    └── UnpackArgs.hpp


/BPNN.hpp:
--------------------------------------------------------------------------------
 1 | //-------------------------------------------------------------------------------
 2 | // @brief
 3 | //     Deep BP neural network
 4 | //
 5 | // @author
 6 | //     Millhaus.Chen @time 2017/07/28 15:16
 7 | //-------------------------------------------------------------------------------
 8 | #pragma once
 9 | 
10 | #include "math/sigfunc.h"
11 | #include "math/Matrix.hpp"
12 | #include "util/UnpackArgs.hpp"
13 | #include "util/TupleTool.hpp"
14 | #include "include/Parameter.hpp"
15 | 
16 | #include <tuple>
17 | #include <utility>
18 | 
19 | namespace mtl {
20 | 
21 | /// Type helper
22 | template<typename I, int... Layers> struct BPNNType;
23 | template<std::size_t... I, int... Layers>
24 | struct BPNNType<std::index_sequence<I...>, Layers...>
25 | {
26 |     typedef /// Weights type
27 |     std::tuple<
28 |             Matrix<
29 |                     double,
30 |                     UnpackInts<I, Layers...>::value,
31 |                     UnpackInts<I + 1, Layers...>::value
32 |             >...
33 |     > Weights;
34 | 
35 |     typedef /// Thresholds type
36 |     std::tuple<
37 |             Matrix<
38 |                     double,
39 |                     1,
40 |                     UnpackInts<I + 1, Layers...>::value
41 |             >...
42 |     > Thresholds;
43 | };
44 | 
45 | /// The neural network class
46 | template<int... Layers>
47 | class BPNN : public NNParam
48 | {
49 |     static const int N = sizeof...(Layers);
50 |     using expander = int[];
51 | 
52 | public:
53 |     using InMatrix = Matrix<double, 1, UnpackInts<0, Layers...>::value>;
54 |     using OutMatrix = Matrix<double, 1, UnpackInts<N - 1, Layers...>::value>;
55 | 
56 | public:
57 |     BPNN<Layers...>& init();
58 | 
59 |     template<class LX, class LY, class W, class T>
60 |     void forward(LX& layerX, LY& layerY, W& weight, T& threshold);
61 |     template<class LX, class W, class T, class DX, class DY>
62 |     void backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY);
63 | 
64 |     template<std::size_t... I>
65 |     bool train(const InMatrix& input, const OutMatrix& output, int times, double nor, std::index_sequence<I...>);
66 |     bool train(const InMatrix& input, const OutMatrix& output, int times = 1, double nor = 1)
67 |     {   return train(input, output, times, nor, std::make_index_sequence<N - 1>());
68 |     }
69 | 
70 |     template<std::size_t... I>
71 |     double simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor, std::index_sequence<I...>);
72 |     double simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor = 1)
73 |     {   return simulate(input, output, expect, nor, std::make_index_sequence<N - 1>());
74 |     }
75 | 
76 | public:
77 |     std::tuple<Matrix<double, 1, Layers>...> m_layers;
78 |     typename BPNNType<std::make_index_sequence<N - 1>, Layers...>::Weights m_weights;
79 |     typename BPNNType<std::make_index_sequence<N - 1>, Layers...>::Thresholds m_thresholds;
80 |     std::tuple<Matrix<double, 1, Layers>...> m_deltas;
81 |     OutMatrix m_aberrmx;
82 | };
83 | 
84 | }
85 | 
86 | #include "include/BPNN.inl"


--------------------------------------------------------------------------------
/CNN.hpp:
--------------------------------------------------------------------------------
 1 | //-------------------------------------------------------------------------------
 2 | // @brief
 3 | //     Deep CNN neural network
 4 | //
 5 | // @author
 6 | //     Millhaus.Chen @time 2017/10/07 10:58
 7 | //-------------------------------------------------------------------------------
 8 | #pragma once
 9 | 
10 | #include "math/sigfunc.h"
11 | #include "math/Matrix.hpp"
12 | #include "util/UnpackArgs.hpp"
13 | #include "util/TupleTool.hpp"
14 | #include "include/Parameter.hpp"
15 | 
16 | #include <tuple>
17 | #include <utility>
18 | 
19 | namespace mtl {
20 | 
21 | namespace cnn
22 | {
23 |     /// Type helper
24 |     template<typename I, int... Layers>
25 |     struct Type;
26 |     template<std::size_t... I, int... Layers>
27 |     struct Type<std::index_sequence<I...>, Layers...>
28 |     {
29 |         typedef /// Weights type
30 |         std::tuple<
31 |                 Matrix<
32 |                         double,
33 |                         UnpackInts<I, Layers...>::value,
34 |                         UnpackInts<I + 1, Layers...>::value
35 |                 >...
36 |         > Weights;
37 | 
38 |         typedef /// Thresholds type
39 |         std::tuple<
40 |                 Matrix<
41 |                         double,
42 |                         1,
43 |                         UnpackInts<I + 1, Layers...>::value
44 |                 >...
45 |         > Thresholds;
46 |     };
47 | }
48 | 
49 | /// The neural network class
50 | template<int... Layers>
51 | class CNN : public NNParam
52 | {
53 |     static const int N = sizeof...(Layers);
54 |     using expander = int[];
55 | 
56 | public:
57 |     using InMatrix = Matrix<double, 1, UnpackInts<0, Layers...>::value>;
58 |     using OutMatrix = Matrix<double, 1, UnpackInts<N - 1, Layers...>::value>;
59 | 
60 | public:
61 |     CNN<Layers...>& init();
62 | 
63 |     template<class LX, class LY, class W, class T>
64 |     void forward(LX& layerX, LY& layerY, W& weight, T& threshold);
65 |     template<class LX, class W, class T, class DX, class DY>
66 |     void backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY);
67 | 
68 |     template<std::size_t... I>
69 |     bool train(const InMatrix& input, const OutMatrix& output, int times, double nor, std::index_sequence<I...>);
70 |     bool train(const InMatrix& input, const OutMatrix& output, int times = 1, double nor = 1)
71 |     {   return train(input, output, times, nor, std::make_index_sequence<N - 1>());
72 |     }
73 | 
74 |     template<std::size_t... I>
75 |     double simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor, std::index_sequence<I...>);
76 |     double simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor = 1)
77 |     {   return simulate(input, output, expect, nor, std::make_index_sequence<N - 1>());
78 |     }
79 | 
80 | public:
81 |     std::tuple<Matrix<double, 1, Layers>...> m_layers;
82 |     typename cnn::Type<std::make_index_sequence<N - 1>, Layers...>::Weights m_weights;
83 |     typename cnn::Type<std::make_index_sequence<N - 1>, Layers...>::Thresholds m_thresholds;
84 |     std::tuple<Matrix<double, 1, Layers>...> m_deltas;
85 |     OutMatrix m_aberrmx;
86 | };
87 | 
88 | }
89 | 
90 | #include "include/CNN.inl"


--------------------------------------------------------------------------------
/CPPCON_CN_Workspace.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bowdar/DeepLearning/687dd2215605c686919531802be7f3470740bd49/CPPCON_CN_Workspace.zip


--------------------------------------------------------------------------------
/LSTM.hpp:
--------------------------------------------------------------------------------
  1 | //-------------------------------------------------------------------------------
  2 | // @brief
  3 | //     Long Short-Term Memory deep neural network
  4 | //
  5 | // @author
  6 | //     Millhaus.Chen @time 2017/09/10 11:04
  7 | //-------------------------------------------------------------------------------
  8 | #pragma once
  9 | 
 10 | #include "math/sigfunc.h"
 11 | #include "math/Matrix.hpp"
 12 | #include "util/UnpackArgs.hpp"
 13 | #include "util/TupleTool.hpp"
 14 | #include "include/Parameter.hpp"
 15 | 
 16 | #include <tuple>
 17 | #include <utility>
 18 | 
 19 | namespace mtl {
 20 | 
 21 | namespace lstm
 22 | {
 23 |     enum Gate : unsigned char
 24 |     {   f = 0,
 25 |         i,
 26 |         C,
 27 |         o,
 28 |     };
 29 | 
 30 |     /// Type helper
 31 |     template<typename I, int... Layers>
 32 |     struct Type;
 33 |     template<std::size_t... I, int... Layers>
 34 |     struct Type<std::index_sequence<I...>, Layers...>
 35 |     {
 36 |         typedef /// Weights type
 37 |         std::tuple<
 38 |                 Matrix<
 39 |                         double,
 40 |                         UnpackInts<I, Layers...>::value,
 41 |                         UnpackInts<I + 1, Layers...>::value
 42 |                 >[o]...
 43 |         > Weights;
 44 | 
 45 |         typedef /// Thresholds and deltas type
 46 |         std::tuple<
 47 |                 Matrix<
 48 |                         double,
 49 |                         1,
 50 |                         UnpackInts<I + 1, Layers...>::value
 51 |                 >[o]...
 52 |         > Thresholds;
 53 | 
 54 |         typedef /// RWeights type
 55 |         std::tuple<
 56 |                 Matrix<
 57 |                         double,
 58 |                         UnpackInts<I, Layers...>::value,
 59 |                         UnpackInts<I, Layers...>::value
 60 |                 >[o]...
 61 |         > RWeights;
 62 | 
 63 |         /// RCells type
 64 |         template<int R>
 65 |         using RCells = std::tuple<
 66 |                 Matrix<
 67 |                         double,
 68 |                         1,
 69 |                         UnpackInts<I + 1, Layers...>::value
 70 |                 >[R]...
 71 |         >;
 72 | 
 73 |         /// Temp states type
 74 |         template<int R>
 75 |         using Temps = std::tuple<
 76 |                 Matrix<
 77 |                         double,
 78 |                         1,
 79 |                         UnpackInts<I + 1, Layers...>::value
 80 |                 >[R][o]...
 81 |         >;
 82 |     };
 83 | }
 84 | 
 85 | /// The LSTM neural network class
 86 | template<int... Layers>
 87 | class LSTM : public NNParam
 88 | {
 89 |     static const int N = sizeof...(Layers);
 90 |     using expander = int[];
 91 | 
 92 | public:
 93 |     template<int R>
 94 |     using  InMatrix = Matrix<double,
 95 |                              R,
 96 |                              UnpackInts<0, Layers...>::value>;
 97 |     template<int R>
 98 |     using OutMatrix = Matrix<double,
 99 |                              R,
100 |                              UnpackInts<N - 1, Layers...>::value>;
101 | 
102 | public:
103 |     LSTM<Layers...>& init();
104 | 
105 |     template<class LX, class LY, class W, class T, class RW, class CY, class RC, class S>
106 |     void forward(LX& layerX, LY& layerY, W& weight, T& threshold, RW& rWeight, CY& cellY,
107 |                  RC& rCell, S& state, int t, int rIn);
108 |     template<class LX, class W, class T, class DX, class DY, class GD, class RW, class S,
109 |             class RD, class RC, class CY>
110 |     void backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY, GD& gDelta,
111 |                  RW& rWeight, S& state, RD& rDelta, RC& rCell, CY& cellY, int t, int r, int rIn);
112 | 
113 |     template<class IN, class OUT, std::size_t... I>
114 |     bool train(IN& input, OUT& output, int times, double nor, std::index_sequence<I...>);
115 |     template<class IN, class OUT>
116 |     bool train(IN& input, OUT& output, int times = 1, double nor = 1)
117 |     {   return train(input, output, times, nor, std::make_index_sequence<N - 1>());
118 |     }
119 | 
120 |     template<class IN, class OUT, std::size_t... I>
121 |     double simulate(IN& input, OUT& output, OUT& expect, double nor, std::index_sequence<I...>);
122 |     template<class IN, class OUT>
123 |     double simulate(IN& input, OUT& output, OUT& expect, double nor = 1)
124 |     {   return simulate(input, output, expect, nor, std::make_index_sequence<N - 1>());
125 |     }
126 | 
127 | public:
128 |     std::tuple<Matrix<double, 1, Layers>...> m_layers;
129 |     std::tuple<Matrix<double, 1, Layers>...> m_cells; /// redundance 1
130 |     std::tuple<Matrix<double, 1, Layers>...> m_deltas; /// Final delta of every output, redundance 1
131 |     typename lstm::Type<std::make_index_sequence<N - 1>, Layers...>::Weights m_weights;
132 |     typename lstm::Type<std::make_index_sequence<N - 1>, Layers...>::Thresholds m_thresholds;
133 |     typename lstm::Type<std::make_index_sequence<N - 1>, Layers...>::Thresholds m_gDeltas; /// All gates deltas, redundance 1
134 |     typename lstm::Type<std::make_index_sequence<N>, Layers...>::RWeights m_rWeights;  /// redundance 1
135 | };
136 | 
137 | }
138 | 
139 | #include "include/LSTM.inl"


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # [Meta-programming DeepLearning](https://github.com/bowdar/DeepLearning) 
  2 | Meta-programming neural network 是一个基于C++14实现的元编程神经网络库
  3 | Compile-time matrix constructions, headonly, no dependency, limitless layers, limitless nodes
  4 |  
  5 | ## Feature
  6 | * 支持任意深度和超大结点数
  7 | * 矩阵运算（CNN采用张量运算）
  8 | * 循环类网络输入输出支持多对单、单对多、多对多
  9 | * 源码Head-only并且无依赖
 10 | * 使用方法极其简单，适合程序局部应用ANN以及用来学习研究
 11 | 
 12 | ## Sample
 13 | ### 1) BPNN
 14 | ```cpp
 15 | #include "BPNN.hpp"
 16 | int main()
 17 | {
 18 |     /// 1. Create a 4 layers NN each layer nodes are 20, 30, 20 and 2
 19 |     ///    The first 20 is input layer and the last 2 is output
 20 |     typedef mtl::BPNN<20, 30, 20, 2> MyNN;
 21 |     MyNN bpnn;
 22 |     
 23 |     /// 2. Initialize, setup parameters and activate functions
 24 |     bpnn.init()
 25 |         .set_aberration(0.0001)
 26 |         .set_learnrate(0.8)
 27 |         .set_sigfunc(mtl::logsig)
 28 |         .set_dsigfunc(mtl::dlogsig);
 29 | 
 30 |     /// 3. Create input output matrixs, and then enter matrix datas your self
 31 |     MyNN::InMatrix inMx;
 32 |     MyNN::OutMatrix outMx;
 33 |     MyNN::OutMatrix expectMx;
 34 |     ///    enter matrix datas ...
 35 |     
 36 |     /// 4. Training, call train in your own way
 37 |     bpnn.train(inMx, outMx, 100);
 38 |     
 39 |     /// 5. Simulate
 40 |     bpnn.simulate(inMx, outMx, expectMx);
 41 | }
 42 | ```
 43 | 
 44 | ### 2) RNN
 45 | ```cpp
 46 | #include "RNN.hpp"
 47 | int main()
 48 | {
 49 |     /// 1. Create a 4 layers NN each layer nodes are 20, 30, 20 and 2
 50 |     ///    The first 20 is input layer and the last 2 is output
 51 |     typedef mtl::RNN<20, 30, 20, 2> MyRnn;
 52 |     MyRnn rnn;
 53 |     
 54 |     /// 2. Initialize, setup parameters and activate functions
 55 |     rnn.init()
 56 |        .set_aberration(0.0001)
 57 |        .set_learnrate(0.8)
 58 |        .set_sigfunc(mtl::logsig)
 59 |        .set_dsigfunc(mtl::dlogsig);
 60 | 
 61 |     /// 3. Create input output matrixs, and then enter matrix datas your self
 62 |     ///    RNN suport multi-in-out like M:1, 1:M and M:M also 1:1 which is meaningless
 63 |     MyRnn::InMatrix<10> inMx; /// 10 input a group, you can change it each training
 64 |     MyRnn::OutMatrix<2> outMx; /// 2 ouput a group
 65 |     MyRnn::OutMatrix<2> expectMx;
 66 |     ///    enter matrix datas ...
 67 |     
 68 |     /// 4. Training, call train in your own way
 69 |     rnn.train(inMx, outMx, 100);
 70 |     
 71 |     /// 5. Simulate
 72 |     rnn.simulate(inMx, outMx，expectMx);
 73 | }
 74 | ```
 75 | 
 76 | ### 3) LSTM
 77 | ```cpp
 78 | #include "LSTM.hpp"
 79 | int main()
 80 | {
 81 |     /// 1. Create a 4 layers NN each layer nodes are 20, 30, 20 and 2
 82 |     ///    The first 20 is input layer and the last 2 is output
 83 |     typedef mtl::LSTM<20, 30, 20, 2> MyLSTM;
 84 |     MyLSTM lstm;
 85 |     
 86 |     /// 2. Initialize, setup parameters, LSTM wouldn't setup activate functions
 87 |     lstm.init()
 88 |         .set_aberration(0.0001)
 89 |         .set_learnrate(0.8);
 90 | 
 91 |     /// 3. Create input output matrixs, and then enter matrix datas your self
 92 |     ///    RNN suport multi-in-out like M:1, 1:M and M:M also 1:1 which is meaningless
 93 |     MyLSTM::InMatrix<10> inMx;
 94 |     MyLSTM::OutMatrix<2> outMx;
 95 |     MyLSTM::OutMatrix<2> expectMx;
 96 |     ///    enter matrix datas ...
 97 |     
 98 |     /// 4. Training, call train in your own way
 99 |     lstm.train(inMx, outMx, 100);
100 |     
101 |     /// 5. Simulate
102 |     lstm.simulate(inMx, outMx，expectMx);
103 | }
104 | ```
105 | <a href="https://996.icu"><img src="https://img.shields.io/badge/link-996.icu-red.svg"></a>
106 | 


--------------------------------------------------------------------------------
/RNN.hpp:
--------------------------------------------------------------------------------
  1 | //-------------------------------------------------------------------------------
  2 | // @brief
  3 | //     Recurrent deep neural network
  4 | //     Support M:1, 1:M, M:M in and out certainly 1: 1
  5 | //
  6 | // @author
  7 | //     Millhaus.Chen @time 2017/09/02 16:34
  8 | //-------------------------------------------------------------------------------
  9 | #pragma once
 10 | 
 11 | #include "math/sigfunc.h"
 12 | #include "math/Matrix.hpp"
 13 | #include "util/UnpackArgs.hpp"
 14 | #include "util/TupleTool.hpp"
 15 | #include "include/Parameter.hpp"
 16 | 
 17 | #include <tuple>
 18 | #include <utility>
 19 | 
 20 | namespace mtl {
 21 | 
 22 | namespace rnn
 23 | {
 24 |     /// Type helper
 25 |     template<typename I, int... Layers>
 26 |     struct Type;
 27 |     template<std::size_t... I, int... Layers>
 28 |     struct Type<std::index_sequence<I...>, Layers...>
 29 |     {
 30 |         typedef /// Weights type
 31 |         std::tuple<
 32 |                 Matrix<
 33 |                         double,
 34 |                         UnpackInts<I, Layers...>::value,
 35 |                         UnpackInts<I + 1, Layers...>::value
 36 |                 >...
 37 |         > Weights;
 38 | 
 39 |         typedef /// Thresholds type
 40 |         std::tuple<
 41 |                 Matrix<
 42 |                         double,
 43 |                         1,
 44 |                         UnpackInts<I + 1, Layers...>::value
 45 |                 >...
 46 |         > Thresholds;
 47 | 
 48 |         typedef /// RWeights type
 49 |         std::tuple<
 50 |                 Matrix<
 51 |                         double,
 52 |                         UnpackInts<I, Layers...>::value,
 53 |                         UnpackInts<I, Layers...>::value
 54 |                 >...
 55 |         > RWeights;
 56 | 
 57 |         /// Temp states type
 58 |         template<int R>
 59 |         using Temps = std::tuple<
 60 |                 Matrix<
 61 |                         double,
 62 |                         1,
 63 |                         UnpackInts<I + 1, Layers...>::value
 64 |                 >[R]...
 65 |         >;
 66 |     };
 67 | }
 68 | /// The neural network class
 69 | template<int... Layers>
 70 | class RNN : public NNParam
 71 | {
 72 |     static const int N = sizeof...(Layers);
 73 |     using expander = int[];
 74 | 
 75 | public:
 76 |     template<int R>
 77 |     using  InMatrix = Matrix<double,
 78 |                              R,
 79 |                              UnpackInts<0, Layers...>::value>;
 80 |     template<int R>
 81 |     using OutMatrix = Matrix<double,
 82 |                              R,
 83 |                              UnpackInts<N - 1, Layers...>::value>;
 84 | 
 85 | public:
 86 |     RNN<Layers...>& init();
 87 | 
 88 |     template<class LX, class LY, class W, class T, class RW, class S>
 89 |     void forward(LX& layerX, LY& layerY, W& weight, T& threshold, RW& rWeight, S& state, int t, int rIn);
 90 |     template<class LX, class W, class T, class DX, class DY, class RW, class S, class RD>
 91 |     void backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY, RW& rWeight,
 92 |                  S& state, RD& rDelta, int t, int r, int rIn);
 93 | 
 94 |     template<class IN, class OUT, std::size_t... I>
 95 |     bool train(IN& input, OUT& output, int times, double nor, std::index_sequence<I...>);
 96 |     template<class IN, class OUT>
 97 |     bool train(IN& input, OUT& output, int times = 1, double nor = 1)
 98 |     {   return train(input, output, times, nor, std::make_index_sequence<N - 1>());
 99 |     }
100 | 
101 |     template<class IN, class OUT, std::size_t... I>
102 |     double simulate(IN& input, OUT& output, OUT& expect, double nor, std::index_sequence<I...>);
103 |     template<class IN, class OUT>
104 |     double simulate(IN& input, OUT& output, OUT& expect, double nor = 1)
105 |     {   return simulate(input, output, expect, nor, std::make_index_sequence<N - 1>());
106 |     }
107 | 
108 | public:
109 |     std::tuple<Matrix<double, 1, Layers>...> m_layers;
110 |     std::tuple<Matrix<double, 1, Layers>...> m_deltas; /// redundance 1
111 |     typename rnn::Type<std::make_index_sequence<N - 1>, Layers...>::Weights m_weights;
112 |     typename rnn::Type<std::make_index_sequence<N - 1>, Layers...>::Thresholds m_thresholds;
113 |     typename rnn::Type<std::make_index_sequence<N>, Layers...>::RWeights m_rWeights;  /// redundance 1
114 | };
115 | 
116 | }
117 | 
118 | #include "include/RNN.inl"


--------------------------------------------------------------------------------
/RNN_N.hpp:
--------------------------------------------------------------------------------
  1 | //-------------------------------------------------------------------------------
  2 | // @brief
  3 | //     Recurrent deep neural network
  4 | //     Different from RNN, the in and out of RNN_N is all in circulate and only
  5 | //  support 1:1 mode
  6 | //
  7 | // @author
  8 | //     Millhaus.Chen @time 2017/09/02 16:34
  9 | //-------------------------------------------------------------------------------
 10 | #pragma once
 11 | 
 12 | #include "math/sigfunc.h"
 13 | #include "math/Matrix.hpp"
 14 | #include "util/UnpackArgs.hpp"
 15 | #include "util/TupleTool.hpp"
 16 | #include "include/Parameter.hpp"
 17 | 
 18 | #include <tuple>
 19 | #include <utility>
 20 | 
 21 | namespace mtl {
 22 | 
 23 | /// Type helper
 24 | template<typename I, int... Layers> struct RNNType;
 25 | template<std::size_t... I, int... Layers>
 26 | struct RNNType<std::index_sequence<I...>, Layers...>
 27 | {
 28 |     typedef /// Weights type
 29 |     std::tuple<
 30 |             Matrix<
 31 |                     double,
 32 |                     UnpackInts<I, Layers...>::value,
 33 |                     UnpackInts<I + 1, Layers...>::value
 34 |             >...
 35 |     > Weights;
 36 | 
 37 |     typedef /// Thresholds type
 38 |     std::tuple<
 39 |             Matrix<
 40 |                     double,
 41 |                     1,
 42 |                     UnpackInts<I + 1, Layers...>::value
 43 |             >...
 44 |     > Thresholds;
 45 | 
 46 |     typedef /// RWeights type
 47 |     std::tuple<
 48 |             Matrix<
 49 |                     double,
 50 |                     UnpackInts<I, Layers...>::value,
 51 |                     UnpackInts<I, Layers...>::value
 52 |             >...
 53 |     > RWeights;
 54 | };
 55 | 
 56 | /// The neural network class
 57 | template<int... Layers>
 58 | class RNN_N : NNParam
 59 | {
 60 |     static const int N = sizeof...(Layers);
 61 |     using expander = int[];
 62 | public:
 63 |     using InMatrix = Matrix<double, 1, UnpackInts<0, Layers...>::value>;
 64 |     using OutMatrix = Matrix<double, 1, UnpackInts<N - 1, Layers...>::value>;
 65 | 
 66 | public:
 67 |     RNN_N<Layers...>& init();
 68 | 
 69 |     template<class LX, class LY, class W, class T, class RLY, class RW>
 70 |     void forward(LX& layerX, LY& layerY, W& weight, T& threshold, RLY& rLayerY, RW& rWeight);
 71 |     template<class LX, class W, class T, class DX, class DY, class RLY, class RWX, class RWY, class RDX>
 72 |     void backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY, RLY& rLayerY,
 73 |                  RWX& rWeightX, RWY& rWeightY, RDX& rDeltaX);
 74 | 
 75 |     template<std::size_t... I>
 76 |     bool train(const InMatrix& input, const OutMatrix& output, int times, double nor, std::index_sequence<I...>);
 77 |     bool train(const InMatrix& input, const OutMatrix& output, int times = 1, double nor = 1)
 78 |     {   return train(input, output, times, nor, std::make_index_sequence<N - 1>());
 79 |     }
 80 | 
 81 |     template<std::size_t... I>
 82 |     double simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor, std::index_sequence<I...>);
 83 |     double simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor = 1)
 84 |     {   return simulate(input, output, expect, nor, std::make_index_sequence<N - 1>());
 85 |     }
 86 | 
 87 | public:
 88 |     std::tuple<Matrix<double, 1, Layers>...> m_layers;
 89 |     std::tuple<Matrix<double, 1, Layers>...> m_rLayers;
 90 |     typename RNNType<std::make_index_sequence<N - 1>, Layers...>::Weights m_weights;
 91 |     typename RNNType<std::make_index_sequence<N - 1>, Layers...>::Thresholds m_thresholds;
 92 |     typename RNNType<std::make_index_sequence<N>, Layers...>::RWeights m_rWeights;  /// redundance 1
 93 |     std::tuple<Matrix<double, 1, Layers>...> m_deltas; /// redundance 1
 94 |     std::tuple<Matrix<double, 1, Layers>...> m_rDeltas; /// redundance 1
 95 |     OutMatrix m_aberrmx;
 96 | };
 97 | 
 98 | }
 99 | 
100 | #include "include/RNN_N.inl"


--------------------------------------------------------------------------------
/Samples.cpp:
--------------------------------------------------------------------------------
 1 | #include "BPNN.hpp"
 2 | #include "RNN.hpp"
 3 | #include "LSTM.hpp"
 4 | 
 5 | int main()
 6 | {
 7 |     /// -- BPNN --------------------------------------------------------------------
 8 |     /// 1. Create a 4 layers NN each layer nodes are 20, 30, 20 and 2
 9 |     ///    The first 20 is input layer and the last 2 is output
10 |     typedef mtl::BPNN<20, 30, 20, 2> MyNN;
11 |     MyNN bpnn;
12 |     
13 |     /// 2. Initialize, setup parameters and activate functions
14 |     bpnn.init()
15 |         .set_aberration(0.0001)
16 |         .set_learnrate(0.8)
17 |         .set_sigfunc(mtl::logsig)
18 |         .set_dsigfunc(mtl::dlogsig);
19 | 
20 |     /// 3. Create input output matrixs, and then enter matrix datas your self
21 |     MyNN::InMatrix inMx;
22 |     MyNN::OutMatrix outMx;
23 |     MyNN::OutMatrix expectMx;
24 |     ///    enter matrix datas ...
25 |     
26 |     /// 4. Training, call train in your own way
27 |     bpnn.train(inMx, outMx, 100);
28 |     
29 |     /// 5. Simulate
30 |     bpnn.simulate(inMx, outMx, expectMx);
31 | 
32 | 
33 |     /// -- RNN ---------------------------------------------------------------------
34 |     /// 1. Create a 4 layers NN each layer nodes are 20, 30, 20 and 2
35 |     ///    The first 20 is input layer and the last 2 is output
36 |     typedef mtl::RNN<20, 30, 20, 2> MyRnn;
37 |     MyRnn rnn;
38 |     
39 |     /// 2. Initialize, setup parameters and activate functions
40 |     rnn.init()
41 |        .set_aberration(0.0001)
42 |        .set_learnrate(0.8)
43 |        .set_sigfunc(mtl::logsig)
44 |        .set_dsigfunc(mtl::dlogsig);
45 | 
46 |     /// 3. Create input output matrixs, and then enter matrix datas your self
47 |     ///    RNN suport multi-in-out like M:1, 1:M and M:M also 1:1 which is meaningless
48 |     MyRnn::InMatrix<10> inMx; /// 10 input a group, you can change it each training
49 |     MyRnn::OutMatrix<2> outMx; /// 2 ouput a group
50 |     MyRnn::OutMatrix<2> expectMx;
51 |     ///    enter matrix datas ...
52 |     
53 |     /// 4. Training, call train in your own way
54 |     rnn.train(inMx, outMx, 100);
55 |     
56 |     /// 5. Simulate
57 |     rnn.simulate(inMx, outMx，expectMx);
58 | 
59 |     
60 |     /// -- LSTM --------------------------------------------------------------------
61 |     /// 1. Create a 4 layers NN each layer nodes are 20, 30, 20 and 2
62 |     ///    The first 20 is input layer and the last 2 is output
63 |     typedef mtl::LSTM<20, 30, 20, 2> MyLSTM;
64 |     MyLSTM lstm;
65 |     
66 |     /// 2. Initialize, setup parameters, LSTM wouldn't setup activate functions
67 |     lstm.init()
68 |         .set_aberration(0.0001)
69 |         .set_learnrate(0.8);
70 | 
71 |     /// 3. Create input output matrixs, and then enter matrix datas your self
72 |     ///    RNN suport multi-in-out like M:1, 1:M and M:M also 1:1 which is meaningless
73 |     MyLSTM::InMatrix<10> inMx;
74 |     MyLSTM::OutMatrix<2> outMx;
75 |     MyLSTM::OutMatrix<2> expectMx;
76 |     ///    enter matrix datas ...
77 |     
78 |     /// 4. Training, call train in your own way
79 |     lstm.train(inMx, outMx, 100);
80 |     
81 |     /// 5. Simulate
82 |     lstm.simulate(inMx, outMx，expectMx);
83 | }
84 | 
85 | 


--------------------------------------------------------------------------------
/include/BPNN.inl:
--------------------------------------------------------------------------------
  1 | //-------------------------------------------------------------------------------
  2 | // @author
  3 | //     Millhaus.Chen @time 2017/08/01 09:29
  4 | //-------------------------------------------------------------------------------
  5 | namespace mtl {
  6 | 
  7 | template<int... Layers>
  8 | BPNN<Layers...>& BPNN<Layers...>::init()
  9 | {
 10 |     mtl::for_each(m_weights, [](auto& weight) mutable
 11 |     {   weight.random(0, 1);
 12 |     });
 13 | 
 14 |     mtl::for_each(m_thresholds, [](auto& threshold) mutable
 15 |     {   threshold.random(0, 1);
 16 |     });
 17 | 
 18 |     return *this;
 19 | }
 20 | 
 21 | template<class T>
 22 | void _testPrint(T& matrix, const char* name)
 23 | {
 24 |     printf("%s = \n", name);
 25 |     for(const auto& r : matrix.data)
 26 |     {   for(const auto& e : r)
 27 |         {   printf("%f\t", (float)e);
 28 |         }
 29 |         printf("\n---------------------------------------------------\n");
 30 |     }
 31 | }
 32 | 
 33 | template<class T>
 34 | void _testPrint1(T& matrix, const char* name)
 35 | {
 36 |     for(const auto& r : matrix.data)
 37 |     {   for(const auto& e : r)
 38 |         {   printf("%.2f\t", (float)e);
 39 |         }
 40 |     }
 41 | }
 42 | 
 43 | template<int... Layers>
 44 | template<class LX, class LY, class W, class T>
 45 | void BPNN<Layers...>::forward(LX& layerX, LY& layerY, W& weight, T& threshold)
 46 | {
 47 |     layerY.multiply(layerX, weight); /// layerY = layerX * weight
 48 |     layerY.foreach([&layerX](auto& e){ return e / layerX.Col();}); /// 用于支持超大节点数
 49 |     layerY += threshold;
 50 |     layerY.foreach(m_sigfunc);
 51 | };
 52 | 
 53 | template<int... Layers>
 54 | template<class LX, class W, class T, class DX, class DY>
 55 | void BPNN<Layers...>::backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY)
 56 | {
 57 |     weight.adjustW(layerX, deltaY, m_learnrate);
 58 |     threshold.adjustT(deltaY, m_learnrate);
 59 |     /// 计算delta
 60 |     deltaX.mult_trans(weight, deltaY);
 61 |     layerX.foreach(m_dsigfunc);
 62 |     deltaX.hadamard(layerX);
 63 | };
 64 | 
 65 | template<int... Layers>
 66 | template<std::size_t... I>
 67 | bool BPNN<Layers...>::train(const InMatrix& input, const OutMatrix& output, int times, double nor, std::index_sequence<I...>)
 68 | {
 69 |     /// 1. 输入归一化
 70 |     auto& layer0 = std::get<0>(m_layers);
 71 |     layer0 = input;
 72 |     layer0.normalize(nor);
 73 |     auto& layerN = std::get<N - 1>(m_layers);
 74 |     auto& deltaN = std::get<N - 1>(m_deltas);
 75 |     for(int i = 0; i < times; ++i)
 76 |     {   /// 2. 正向传播
 77 |         expander {(forward(std::get<I>(m_layers),
 78 |                            std::get<I + 1>(m_layers),
 79 |                            std::get<I>(m_weights),
 80 |                            std::get<I>(m_thresholds)),
 81 |                            0)...};
 82 |         if(i == times - 1)
 83 |         {   _testPrint(output, "output");
 84 |             _testPrint(layerN, "train output");
 85 |         }
 86 |         /// 3. 判断误差
 87 |         double aberration = m_aberrmx.subtract(output, layerN).squariance() / 2;
 88 |         if (aberration < m_aberration) break;
 89 |         /// 4. 反向修正
 90 |         deltaN.hadamard(m_aberrmx, layerN.foreach(dlogsig));
 91 |         expander {(backward(std::get<N - I - 2>(m_layers),
 92 |                             std::get<N - I - 2>(m_weights),
 93 |                             std::get<N - I - 2>(m_thresholds),
 94 |                             std::get<N - I - 2>(m_deltas),
 95 |                             std::get<N - I - 1>(m_deltas)),
 96 |                             0)...};
 97 |     }
 98 |     return false;
 99 | }
100 | 
101 | template<int... Layers>
102 | template<std::size_t... I>
103 | double BPNN<Layers...>::simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor, std::index_sequence<I...>)
104 | {
105 |     /// 1. 输入归一化
106 |     auto& layer0 = std::get<0>(m_layers);
107 |     layer0 = input;
108 |     layer0.normalize(nor);
109 |     /// 2. 正向传播
110 |     expander {(forward(std::get<I>(m_layers),
111 |                        std::get<I + 1>(m_layers),
112 |                        std::get<I>(m_weights),
113 |                        std::get<I>(m_thresholds)),
114 |                        0)...};
115 |     /// 3. 输出结果
116 |     output = std::get<N - 1>(m_layers);
117 | 
118 |     /// 4. 判断误差
119 |     double aberration = m_aberrmx.subtract(expect, output).squariance() / 2;
120 | 
121 |     _testPrint(expect, "expect");
122 |     _testPrint(output, "simulate output");
123 | 
124 |     return aberration;
125 | }
126 | 
127 | }


--------------------------------------------------------------------------------
/include/CNN.inl:
--------------------------------------------------------------------------------
 1 | //-------------------------------------------------------------------------------
 2 | // @author
 3 | //     Millhaus.Chen @time 2017/10/07 11:01
 4 | //-------------------------------------------------------------------------------
 5 | namespace mtl {
 6 | 
 7 | template<int... Layers>
 8 | CNN<Layers...>& CNN<Layers...>::init()
 9 | {
10 |     mtl::for_each(m_weights, [](auto& weight) mutable
11 |     {   weight.random(0, 1);
12 |     });
13 | 
14 |     mtl::for_each(m_thresholds, [](auto& threshold) mutable
15 |     {   threshold.random(0, 1);
16 |     });
17 | 
18 |     return *this;
19 | }
20 | 
21 | template<int... Layers>
22 | template<class LX, class LY, class W, class T>
23 | void CNN<Layers...>::forward(LX& layerX, LY& layerY, W& weight, T& threshold)
24 | {
25 |     layerY.multiply(layerX, weight); /// layerY = layerX * weight
26 |     layerY.foreach([&layerX](auto& e){ return e / layerX.Col();}); /// 用于支持超大节点数
27 |     layerY += threshold;
28 |     layerY.foreach(m_sigfunc);
29 | };
30 | 
31 | template<int... Layers>
32 | template<class LX, class W, class T, class DX, class DY>
33 | void CNN<Layers...>::backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY)
34 | {
35 |     weight.adjustW(layerX, deltaY, m_learnrate);
36 |     threshold.adjustT(deltaY, m_learnrate);
37 |     /// 计算delta
38 |     deltaX.mult_trans(weight, deltaY);
39 |     layerX.foreach(m_dsigfunc);
40 |     deltaX.hadamard(layerX);
41 | };
42 | 
43 | template<int... Layers>
44 | template<std::size_t... I>
45 | bool CNN<Layers...>::train(const InMatrix& input, const OutMatrix& output, int times, double nor, std::index_sequence<I...>)
46 | {
47 |     /// 1. 输入归一化
48 |     auto& layer0 = std::get<0>(m_layers);
49 |     layer0 = input;
50 |     layer0.normalize(nor);
51 |     auto& layerN = std::get<N - 1>(m_layers);
52 |     auto& deltaN = std::get<N - 1>(m_deltas);
53 |     for(int i = 0; i < times; ++i)
54 |     {   /// 2. 正向传播
55 |         expander {(forward(std::get<I>(m_layers),
56 |                            std::get<I + 1>(m_layers),
57 |                            std::get<I>(m_weights),
58 |                            std::get<I>(m_thresholds)),
59 |                            0)...};
60 |         /// 3. 判断误差
61 |         double aberration = m_aberrmx.subtract(output, layerN).squariance() / 2;
62 |         if (aberration < m_aberration) break;
63 |         /// 4. 反向修正
64 |         deltaN.hadamard(m_aberrmx, layerN.foreach(dlogsig));
65 |         expander {(backward(std::get<N - I - 2>(m_layers),
66 |                             std::get<N - I - 2>(m_weights),
67 |                             std::get<N - I - 2>(m_thresholds),
68 |                             std::get<N - I - 2>(m_deltas),
69 |                             std::get<N - I - 1>(m_deltas)),
70 |                             0)...};
71 |     }
72 |     return false;
73 | }
74 | 
75 | template<int... Layers>
76 | template<std::size_t... I>
77 | double CNN<Layers...>::simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor, std::index_sequence<I...>)
78 | {
79 |     /// 1. 输入归一化
80 |     auto& layer0 = std::get<0>(m_layers);
81 |     layer0 = input;
82 |     layer0.normalize(nor);
83 |     /// 2. 正向传播
84 |     expander {(forward(std::get<I>(m_layers),
85 |                        std::get<I + 1>(m_layers),
86 |                        std::get<I>(m_weights),
87 |                        std::get<I>(m_thresholds)),
88 |                        0)...};
89 |     /// 3. 输出结果
90 |     output = std::get<N - 1>(m_layers);
91 | 
92 |     /// 4. 判断误差
93 |     double aberration = m_aberrmx.subtract(expect, output).squariance() / 2;
94 | 
95 |     return aberration;
96 | }
97 | 
98 | }


--------------------------------------------------------------------------------
/include/LSTM.inl:
--------------------------------------------------------------------------------
  1 | //-------------------------------------------------------------------------------
  2 | // @author
  3 | //     Millhaus.Chen @time 2017/09/10 11:03
  4 | //-------------------------------------------------------------------------------
  5 | namespace mtl {
  6 | 
  7 | template<int... Layers>
  8 | LSTM<Layers...>& LSTM<Layers...>::init()
  9 | {
 10 |     using namespace lstm;
 11 |     mtl::for_each(m_weights, [](auto& weight) mutable
 12 |     {   for(int g = 0; g < o; ++g) weight[g].random(0, 1);
 13 |     });
 14 |     mtl::for_each(m_thresholds, [](auto& threshold) mutable
 15 |     {   for(int g = 0; g < o; ++g) threshold[g].random(0, 1);
 16 |     });
 17 |     mtl::for_each(m_rWeights, [](auto& rWeight) mutable
 18 |     {   for(int g = 0; g < o; ++g) rWeight[g].random(0, 1);
 19 |     });
 20 | 
 21 |     return *this;
 22 | }
 23 | 
 24 | template<int... Layers>
 25 | template<class LX, class LY, class W, class T, class RW, class CY, class RC, class S>
 26 | void LSTM<Layers...>::forward(LX& layerX, LY& layerY, W& weight, T& threshold, RW& rWeight, CY& cellY,
 27 |              RC& rCell, S& state, int t, int rIn)
 28 | {
 29 |     using namespace lstm;
 30 | 
 31 |     /// formula : state_? = σ (∑[0,t]layerX × weight_? + threshold_?)
 32 |     auto formula = [&](unsigned char g, auto func)
 33 |     {   if(t < rIn) layerY.multiply(layerX, weight[g]);
 34 |         if(t > 0) layerY.mult_sum(state[t - 1][g], rWeight[g]);
 35 |         layerY += threshold[g];
 36 |         layerY.foreach(func);
 37 |     };
 38 | 
 39 |     formula(f, logsig);
 40 |     formula(i, logsig);
 41 |     formula(C, tansig);
 42 |     formula(o, logsig);
 43 | 
 44 |     /// cellY = t_f * rCellY + t_i * t_C;
 45 |     cellY.hadamard(state[t][f], rCell[t]);
 46 |     cellY.hadamard_sum(state[t][i], state[t][C]);
 47 |     rCell[t] = cellY;
 48 | 
 49 |     /// layerY = t_o * tanh(cellY);
 50 |     cellY.foreach(tansig);
 51 |     layerY.hadamard(state[t][o], cellY);
 52 | 
 53 |     state[t][o] = layerY;
 54 | };
 55 | 
 56 | template<int... Layers>
 57 | template<class LX, class W, class T, class DX, class DY, class GD, class RW, class S,
 58 |         class RD, class RC, class CY>
 59 | void LSTM<Layers...>::backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY, GD& gDelta,
 60 |               RW& rWeight, S& state, RD& rDelta, RC& rCell, CY& cellY, int t, int r, int rIn)
 61 | {
 62 |     using namespace lstm;
 63 | 
 64 |     /// 计算各个门当前delta分量
 65 |     gDelta[o].hadamard(deltaY, state[t][o]); /// temp parameter
 66 |     auto tempCY = cellY.foreach_n(dtansig);
 67 |     gDelta[f].hadamard(gDelta[o], tempCY).hadamard(rCell[t]).hadamard(state[t][f].foreach_n(dlogsig));
 68 |     gDelta[i].hadamard(gDelta[o], tempCY).hadamard(state[t][C]).hadamard(state[t][i].foreach_n(dlogsig));
 69 |     gDelta[C].hadamard(gDelta[o], tempCY).hadamard(rCell[t]).hadamard(state[t][i]).hadamard(state[t][C].foreach_n([](auto e){ return 1 - e * e;}));
 70 |     gDelta[o].hadamard(deltaY, cellY.foreach(tansig)).hadamard(state[t][o].foreach_n(dlogsig));
 71 | 
 72 |     /// 倒序计算循环过程中产生的delta（时间方向上的delta）
 73 |     auto delta_cal = [&](unsigned char g, auto func)
 74 |     {
 75 |         if(t >= r - 1)
 76 |         {   /// 倒数第一个时刻
 77 |             rDelta[t][g] = gDelta[g];
 78 |             gDelta[g].hadamard(rDelta[t][g], state[t][g]);
 79 |         } else if(t >= rIn)
 80 |         {   /// 对应输出的时刻
 81 |             rDelta[t][g] = gDelta[g];
 82 |             rDelta[t][g].mult_trans_sum(rWeight[g], state[t + 1][g].foreach(func));
 83 |             gDelta[g].hadamard_sum(rDelta[t][g], state[t][g]);
 84 |             rDelta[t][g] += rDelta[t + 1][g];
 85 |         } else
 86 |         {   /// 对应输入的时刻
 87 |             rDelta[t][g].mult_trans(rWeight[g], state[t + 1][g].foreach(func));
 88 |             gDelta[g].hadamard_sum(rDelta[t][g], state[t][g]);
 89 |             rDelta[t][g] += rDelta[t + 1][g];
 90 |         }
 91 |     };
 92 |     delta_cal(f, dlogsig);
 93 |     delta_cal(i, dlogsig);
 94 |     delta_cal(C, dtansig);
 95 |     delta_cal(o, dlogsig);
 96 | 
 97 |     /// 当计算完循环中的第一个delta后开始修正权重、阈值，并在最后计算上一层最后一个delta（深度方向上的deltaX）
 98 |     if(t == 0)
 99 |     {   deltaX.constant(0);
100 |         for(int g = 0; g < o; ++g)
101 |         {   rWeight[g].adjustW(state[t][g], rDelta[t][g], m_learnrate);
102 |             weight[g].adjustW(layerX, gDelta[g], m_learnrate);
103 |             threshold[g].adjustT(gDelta[g], m_learnrate);
104 |             deltaX.mult_trans_sum(weight[g], gDelta[g]);
105 |         }
106 |         deltaX.hadamard(layerX.foreach([](auto x)
107 |                         {   auto sigX = logsig(x);
108 |                             auto tanhX = tansig(x);
109 |                             auto stX = sigX * tanhX;
110 |                             auto dsigX = sigX * (1 - sigX);
111 |                             auto tanhstX = tanh(stX);
112 |                             /// 此处用到了链式求导法则和莱布尼茨公式
113 |                             return dsigX * tanh(stX) + sigX * (1 - tanhstX * tanhstX)
114 |                                        * (dsigX * tanhX + sigX * (1 - tanhX * tanhX));
115 |                         }));
116 |     }
117 | };
118 | 
119 | template<int... Layers>
120 | template<class IN, class OUT, std::size_t... I>
121 | bool LSTM<Layers...>::train(IN& input, OUT& output, int times, double nor, std::index_sequence<I...>)
122 | {
123 |     using namespace lstm;
124 |     /// lstm 需要创建临时矩阵，用来保存当前系列输入的states，也就是临时的layer集合还有delta集合和out集合
125 |     const int r = input.Row() + output.Row();
126 |     typename Type<std::make_index_sequence<N - 1>, Layers...>::template RCells<r> rCells;
127 |     typename Type<std::make_index_sequence<N - 1>, Layers...>::template Temps<r> states;
128 |     typename Type<std::make_index_sequence<N - 1>, Layers...>::template Temps<r> rDeltas;
129 |     OUT trainOut;
130 |     OUT aberration;
131 | 
132 |     auto& layer0 = std::get<0>(m_layers);
133 |     auto& layerN = std::get<N - 1>(m_layers);
134 |     auto& deltaN = std::get<N - 1>(m_deltas);
135 |     for(int i = 0; i < times; ++i)
136 |     {   /// 1. 正向传播
137 |         for(int t = 0; t < r; ++t)
138 |         {   /// 1.1 依次取input的每一层作为当前输入层
139 |             layer0.subset(input, t, 0);
140 |             layer0.normalize(nor);
141 |             expander {(forward(std::get<I>(m_layers),
142 |                                std::get<I + 1>(m_layers),
143 |                                std::get<I>(m_weights),
144 |                                std::get<I>(m_thresholds),
145 |                                std::get<I + 1>(m_rWeights),
146 |                                std::get<I + 1>(m_cells),
147 |                                std::get<I>(rCells),
148 |                                std::get<I>(states),
149 |                                t, input.Row()),
150 |                                0)...};
151 |             /// 1.2 计算出的out依次赋给output的每一层
152 |             if(t >= input.Row())
153 |             {   trainOut.set(layerN, t - input.Row(), 0);
154 |             }
155 |         }
156 |         /// 2. 判断误差
157 |         double error = aberration.subtract(output, trainOut).squariance() / 2;
158 |         if (error < m_aberration) break;
159 |         /// 3. 反向修正
160 |         for(int t = r - 1; t >= 0; --t)
161 |         {   if(t > input.Row() - 1)
162 |             {   deltaN.subset(aberration, t - input.Row(), 0);
163 |                 deltaN.hadamard(layerN.foreach(dlogsig));
164 |             }
165 |             expander {(backward(std::get<N - I - 2>(m_layers),
166 |                                 std::get<N - I - 2>(m_weights),
167 |                                 std::get<N - I - 2>(m_thresholds),
168 |                                 std::get<N - I - 2>(m_deltas),
169 |                                 std::get<N - I - 1>(m_deltas),
170 |                                 std::get<N - I - 2>(m_gDeltas),
171 |                                 std::get<N - I - 1>(m_rWeights),
172 |                                 std::get<N - I - 2>(states),
173 |                                 std::get<N - I - 2>(rDeltas),
174 |                                 std::get<N - I - 2>(rCells),
175 |                                 std::get<N - I - 1>(m_cells),
176 |                                 t, r, input.Row()),
177 |                                 0)...};
178 |         }
179 |     }
180 |     return false;
181 | }
182 | 
183 | template<int... Layers>
184 | template<class IN, class OUT, std::size_t... I>
185 | double LSTM<Layers...>::simulate(IN& input, OUT& output, OUT& expect, double nor, std::index_sequence<I...>)
186 | {
187 |     using namespace lstm;
188 |     /// lstm 需要创建临时矩阵，用来保存当前系列输入的states，也就是临时的layer集合还有delta集合和out集合
189 |     const int r = input.Row() + output.Row();
190 |     typename Type<std::make_index_sequence<N - 1>, Layers...>::template RCells<r> rCells;
191 |     typename Type<std::make_index_sequence<N - 1>, Layers...>::template Temps<r> states;
192 |     typename Type<std::make_index_sequence<N - 1>, Layers...>::template Temps<r> deltas;
193 |     OUT trainOut;
194 |     OUT aberration;
195 | 
196 |     auto& layer0 = std::get<0>(m_layers);
197 |     auto& layerN = std::get<N - 1>(m_layers);
198 |     /// 1. 正向传播
199 |     for(int t = 0; t < r; ++t)
200 |     {   /// 1.1 依次取input的每一层作为当前输入层
201 |         layer0.subset(input, t, 0);
202 |         layer0.normalize(nor);
203 |         expander {(forward(std::get<I>(m_layers),
204 |                            std::get<I + 1>(m_layers),
205 |                            std::get<I>(m_weights),
206 |                            std::get<I>(m_thresholds),
207 |                            std::get<I + 1>(m_rWeights),
208 |                            std::get<I + 1>(m_cells),
209 |                            std::get<I>(rCells),
210 |                            std::get<I>(states),
211 |                            t, input.Row()),
212 |                 0)...};
213 |         /// 1.2 计算出的out依次赋给output的每一层
214 |         if(t >= input.Row())
215 |         {   trainOut.set(layerN, t - input.Row(), 0);
216 |         }
217 |     }
218 | 
219 |     /// 2. 判断误差
220 |     double error = aberration.subtract(output, trainOut).squariance() / 2;
221 | 
222 |     return error;
223 | }
224 | 
225 | }


--------------------------------------------------------------------------------
/include/Parameter.hpp:
--------------------------------------------------------------------------------
 1 | //-------------------------------------------------------------------------------
 2 | // @brief
 3 | //     Parameter base class of NN classes
 4 | //
 5 | // @author
 6 | //     Millhaus.Chen @time 2017/10/01 10:51
 7 | //-------------------------------------------------------------------------------
 8 | #pragma once
 9 | 
10 | #include <functional>
11 | 
12 | namespace mtl {
13 | 
14 | class NNParam
15 | {
16 | public:
17 |     NNParam& set_aberration(double val)
18 |     {
19 |         m_aberration = val;
20 |         return *this;
21 |     }
22 | 
23 |     NNParam& set_learnrate(double val)
24 |     {
25 |         m_learnrate = val;
26 |         return *this;
27 |     }
28 | 
29 |     NNParam& set_sigfunc(std::function<double(double)> sigfunc)
30 |     {
31 |         m_sigfunc = sigfunc;
32 |         return *this;
33 |     }
34 | 
35 |     NNParam& set_dsigfunc(std::function<double(double)> dsigfunc)
36 |     {
37 |         m_dsigfunc = dsigfunc;
38 |         return *this;
39 |     }
40 | 
41 | protected:
42 |     double m_aberration = 0.001;
43 |     double m_learnrate = 0.1;
44 |     std::function<double(double)> m_sigfunc;
45 |     std::function<double(double)> m_dsigfunc;
46 | };
47 | 
48 | 
49 | }


--------------------------------------------------------------------------------
/include/RNN.inl:
--------------------------------------------------------------------------------
  1 | //-------------------------------------------------------------------------------
  2 | // @author
  3 | //     Millhaus.Chen @time 2017/09/02 16:34
  4 | //-------------------------------------------------------------------------------
  5 | namespace mtl {
  6 | 
  7 | template<int... Layers>
  8 | RNN<Layers...>& RNN<Layers...>::init()
  9 | {
 10 |     mtl::for_each(m_weights, [](auto& weight) mutable
 11 |     {   weight.random(0, 1);
 12 |     });
 13 |     mtl::for_each(m_thresholds, [](auto& threshold) mutable
 14 |     {   threshold.random(0, 1);
 15 |     });
 16 |     mtl::for_each(m_rWeights, [](auto& rWeight) mutable
 17 |     {   rWeight.random(0, 1);
 18 |     });
 19 | 
 20 |     return *this;
 21 | }
 22 | 
 23 | template<int... Layers>
 24 | template<class LX, class LY, class W, class T, class RW, class S>
 25 | void RNN<Layers...>::forward(LX& layerX, LY& layerY, W& weight, T& threshold, RW& rWeight, S& state, int t, int rIn)
 26 | {
 27 |     if(t < rIn) layerY.multiply(layerX, weight); /// 循环中只有输入部分才计算输入层
 28 |     if(t > 0) layerY.mult_sum(state[t - 1], rWeight); /// 循环中的第一个不累加上一个时刻的状态
 29 |     layerY += threshold;
 30 |     layerY.foreach(m_sigfunc);
 31 |     state[t] = layerY;
 32 | };
 33 | 
 34 | template<int... Layers>
 35 | template<class LX, class W, class T, class DX, class DY, class RW, class S, class RD>
 36 | void RNN<Layers...>::backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY,
 37 |                              RW& rWeight, S& state, RD& rDelta, int t, int r, int rIn)
 38 | {
 39 |     /// 倒序计算循环过程中产生的delta（时间方向上的delta）
 40 |     if(t >= r - 1)
 41 |     {   /// 倒数第一个时刻
 42 |         rDelta[t] = deltaY;
 43 |         deltaY.hadamard(rDelta[t], state[t]);
 44 |     } else if(t >= rIn)
 45 |     {   /// 对应输出的时刻
 46 |         rDelta[t] = deltaY;
 47 |         rDelta[t].mult_trans_sum(rWeight, state[t + 1].foreach(m_dsigfunc));
 48 |         deltaY.hadamard_sum(rDelta[t], state[t]);
 49 |         rDelta[t] += rDelta[t + 1];
 50 |     } else
 51 |     {   /// 对应输入的时刻
 52 |         rDelta[t].mult_trans(rWeight, state[t + 1].foreach(m_dsigfunc));
 53 |         deltaY.hadamard_sum(rDelta[t], state[t]);
 54 |         rDelta[t] += rDelta[t + 1];
 55 |     }
 56 |     /// 当计算完循环中的第一个delta后开始修正权重、阈值
 57 |     if(t == 0)
 58 |     {   rWeight.adjustW(state[t], rDelta[t], m_learnrate);
 59 |         weight.adjustW(layerX, deltaY, m_learnrate);
 60 |         threshold.adjustT(deltaY, m_learnrate);
 61 |         /// 计算上一层最后一个delta（深度方向上的deltaX）
 62 |         deltaX.mult_trans(weight, deltaY);
 63 |         deltaX.hadamard(layerX.foreach(m_dsigfunc));
 64 |     }
 65 | };
 66 | 
 67 | template<int... Layers>
 68 | template<class IN, class OUT, std::size_t... I>
 69 | bool RNN<Layers...>::train(IN& input, OUT& output, int times, double nor, std::index_sequence<I...>)
 70 | {
 71 |     using namespace rnn;
 72 |     /// rnn 需要创建临时矩阵，用来保存当前系列输入的states，也就是临时的layer集合还有delta集合和out集合
 73 |     const int r = input.Row() + output.Row();
 74 |     typename Type<std::make_index_sequence<N - 1>, Layers...>::template Temps<r> states;
 75 |     typename Type<std::make_index_sequence<N - 1>, Layers...>::template Temps<r> rDeltas;
 76 |     OUT trainOut;
 77 |     OUT aberration;
 78 | 
 79 |     auto& layer0 = std::get<0>(m_layers);
 80 |     auto& layerN = std::get<N - 1>(m_layers);
 81 |     auto& deltaN = std::get<N - 1>(m_deltas);
 82 |     for(int i = 0; i < times; ++i)
 83 |     {   /// 1. 正向传播
 84 |         for(int t = 0; t < r; ++t)
 85 |         {   /// 1.1 依次取input的每一层作为当前输入层
 86 |             layer0.subset(input, t, 0);
 87 |             layer0.normalize(nor);
 88 |             expander {(forward(std::get<I>(m_layers),
 89 |                                std::get<I + 1>(m_layers),
 90 |                                std::get<I>(m_weights),
 91 |                                std::get<I>(m_thresholds),
 92 |                                std::get<I + 1>(m_rWeights),
 93 |                                std::get<I>(states),
 94 |                                t, input.Row()),
 95 |                                0)...};
 96 |             /// 1.2 计算出的out依次赋给output的每一层
 97 |             if(t >= input.Row())
 98 |             {   trainOut.set(layerN, t - input.Row(), 0);
 99 |             }
100 |         }
101 |         /// 2. 判断误差，这里是多个输出的总误差
102 |         double error = aberration.subtract(output, trainOut).squariance() / 2;
103 |         if (error < m_aberration) break;
104 |         /// 3. 反向修正
105 |         for(int t = r - 1; t >= 0; --t)
106 |         {   if(t > input.Row() - 1)
107 |             {   deltaN.subset(aberration, t - input.Row(), 0);
108 |                 deltaN.hadamard(layerN.foreach(dlogsig));
109 |             }
110 |             expander {(backward(std::get<N - I - 2>(m_layers),
111 |                                 std::get<N - I - 2>(m_weights),
112 |                                 std::get<N - I - 2>(m_thresholds),
113 |                                 std::get<N - I - 2>(m_deltas),
114 |                                 std::get<N - I - 1>(m_deltas),
115 |                                 std::get<N - I - 1>(m_rWeights),
116 |                                 std::get<N - I - 2>(states),
117 |                                 std::get<N - I - 2>(rDeltas),
118 |                                 t, r, input.Row()),
119 |                                 0)...};
120 |         }
121 |     }
122 |     return false;
123 | }
124 | 
125 | template<int... Layers>
126 | template<class IN, class OUT, std::size_t... I>
127 | double RNN<Layers...>::simulate(IN& input, OUT& output, OUT& expect, double nor, std::index_sequence<I...>)
128 | {
129 |     const int r = input.Row() + output.Row();
130 |     typename rnn::Type<std::make_index_sequence<N - 1>, Layers...>::template Temps<r> rStates;
131 | 
132 |     /// 正向传播
133 |     auto& layer0 = std::get<0>(m_layers);
134 |     auto& layerN = std::get<N - 1>(m_layers);
135 |     for(int t = 0; t < r; ++t)
136 |     {   input.subset(layer0, t, 0);
137 |         layer0.normalize(nor);
138 |         expander {(forward(std::get<I>(m_layers),
139 |                            std::get<I + 1>(m_layers),
140 |                            std::get<I>(m_weights),
141 |                            std::get<I>(m_thresholds),
142 |                            std::get<I + 1>(m_rWeights),
143 |                            std::get<I>(rStates),
144 |                            t, input.Row()),
145 |                 0)...};
146 |         if(t >= input.Row())
147 |         {   output.set(layerN, t - input.Row(), 0);
148 |         }
149 |     }
150 | 
151 |     /// 返回误差
152 |     return output.subtract(expect, output).squariance() / 2;
153 | }
154 | 
155 | }


--------------------------------------------------------------------------------
/include/RNN_N.inl:
--------------------------------------------------------------------------------
  1 | //-------------------------------------------------------------------------------
  2 | // @author
  3 | //     Millhaus.Chen @time 2017/09/02 16:34
  4 | //-------------------------------------------------------------------------------
  5 | namespace mtl {
  6 | 
  7 | template<int... Layers>
  8 | RNN_N<Layers...>& RNN_N<Layers...>::init()
  9 | {
 10 |     mtl::for_each(m_weights, [](auto& weight) mutable
 11 |     {   weight.random(0, 1);
 12 |     });
 13 |     mtl::for_each(m_thresholds, [](auto& threshold) mutable
 14 |     {   threshold.random(0, 1);
 15 |     });
 16 |     mtl::for_each(m_rWeights, [](auto& rWeight) mutable
 17 |     {   rWeight.random(0, 1);
 18 |     });
 19 |     std::get<0>(m_rLayers).constant(0);
 20 |     std::get<N - 2>(m_rDeltas).constant(0);
 21 | 
 22 |     return *this;
 23 | }
 24 | 
 25 | template<int... Layers>
 26 | template<class LX, class LY, class W, class T, class RLY, class RW>
 27 | void RNN_N<Layers...>::forward(LX& layerX, LY& layerY, W& weight, T& threshold, RLY& rLayerY, RW& rWeight)
 28 | {
 29 |     layerY.multiply(layerX, weight); /// layerY = layerX * weight
 30 |     layerY.mult_sum(rLayerY, rWeight);
 31 |     layerY.foreach([&layerX](auto& e){ return e / layerX.Col();});
 32 |     layerY += threshold;
 33 |     layerY.foreach(m_sigfunc);
 34 |     rLayerY = layerY;
 35 | };
 36 | 
 37 | template<int... Layers>
 38 | template<class LX, class W, class T, class DX, class DY, class RLY, class RWX, class RWY, class RDX>
 39 | void RNN_N<Layers...>::backward(LX& layerX, W& weight, T& threshold, DX& deltaX, DY& deltaY, RLY& rLayerY,
 40 |                              RWX& rWeightX, RWY& rWeightY, RDX& rDeltaX)
 41 | {
 42 |     weight.adjustW(layerX, deltaY, m_learnrate);
 43 |     rWeightY.adjustW(rLayerY, deltaY, m_learnrate);
 44 |     threshold.adjustT(deltaY, m_learnrate);
 45 |     /// 计算delta
 46 |     deltaX.mult_trans(weight, deltaY);
 47 |     deltaX.mult_trans_sum(rWeightX, rDeltaX);
 48 |     layerX.foreach(m_dsigfunc);
 49 |     deltaX.hadamard(layerX);
 50 |     rDeltaX = deltaX;
 51 | };
 52 | 
 53 | template<int... Layers>
 54 | template<std::size_t... I>
 55 | bool RNN_N<Layers...>::train(const InMatrix& input, const OutMatrix& output, int times, double nor, std::index_sequence<I...>)
 56 | {
 57 |     /// 1. 输入归一化
 58 |     auto& layer0 = std::get<0>(m_layers);
 59 |     layer0 = input;
 60 |     layer0.normalize(nor);
 61 |     auto& layerN = std::get<N - 1>(m_layers);
 62 |     auto& deltaN = std::get<N - 1>(m_deltas);
 63 |     for(int i = 0; i < times; ++i)
 64 |     {   /// 2. 正向传播
 65 |         expander {(forward(std::get<I>(m_layers),
 66 |                            std::get<I + 1>(m_layers),
 67 |                            std::get<I>(m_weights),
 68 |                            std::get<I>(m_thresholds),
 69 |                            std::get<I + 1>(m_rLayers),
 70 |                            std::get<I + 1>(m_rWeights)),
 71 |                 0)...};
 72 |         /// 3. 判断误差
 73 |         double aberration = m_aberrmx.subtract(output, layerN).squariance() / 2;
 74 |         if (aberration < m_aberration) break;
 75 |         /// 4. 反向修正
 76 |         deltaN.hadamard(m_aberrmx, layerN.foreach(dlogsig));
 77 |         expander {(backward(std::get<N - I - 2>(m_layers),
 78 |                            std::get<N - I - 2>(m_weights),
 79 |                            std::get<N - I - 2>(m_thresholds),
 80 |                            std::get<N - I - 2>(m_deltas),
 81 |                            std::get<N - I - 1>(m_deltas),
 82 |                            std::get<N - I - 1>(m_rLayers),
 83 |                            std::get<N - I - 2>(m_rWeights),
 84 |                            std::get<N - I - 1>(m_rWeights),
 85 |                            std::get<N - I - 2>(m_rDeltas)),
 86 |                 0)...};
 87 |     }
 88 |     return false;
 89 | }
 90 | 
 91 | template<int... Layers>
 92 | template<std::size_t... I>
 93 | double RNN_N<Layers...>::simulate(const InMatrix& input, OutMatrix& output, OutMatrix& expect, double nor, std::index_sequence<I...>)
 94 | {
 95 |     /// 1. 输入归一化
 96 |     auto& layer0 = std::get<0>(m_layers);
 97 |     layer0 = input;
 98 |     layer0.normalize(nor);
 99 |     /// 2. 正向传播
100 |     expander {(forward(std::get<I>(m_layers),
101 |                        std::get<I + 1>(m_layers),
102 |                        std::get<I>(m_weights),
103 |                        std::get<I>(m_thresholds)),
104 |             0)...};
105 |     /// 3. 输出结果
106 |     output = std::get<N - 1>(m_layers);
107 | 
108 |     /// 4. 判断误差
109 |     double aberration = m_aberrmx.subtract(expect, output).squariance() / 2;
110 | 
111 |     return aberration;
112 | }
113 | 
114 | }


--------------------------------------------------------------------------------
/math/Matrix.hpp:
--------------------------------------------------------------------------------
  1 | //-------------------------------------------------------------------------------
  2 | // Matrix.hpp
  3 | //
  4 | // @brief
  5 | //     thread-unsafe
  6 | //
  7 | // @author
  8 | //     Millhaus.Chen @time 2017/07/27 14:21
  9 | //-------------------------------------------------------------------------------
 10 | #pragma once
 11 | 
 12 | #include <cstdlib>
 13 | #include <cmath>
 14 | 
 15 | namespace mtl {
 16 | 
 17 | template<typename DataType, int ROW, int COL>
 18 | class Matrix
 19 | {
 20 | public:
 21 |     /// 提供三个遍历函数，第一个会遍历并修改每个元素，第三个会产生一个新的矩阵
 22 |     template<typename F>
 23 |     Matrix& foreach(F func)
 24 |     {   for(auto& r : data)
 25 |         {   for(auto& e : r)
 26 |             {   e = func(e);
 27 |             }
 28 |         }
 29 |         return *this;
 30 |     }
 31 |     template<typename F>
 32 |     Matrix& foreach_c(F func)
 33 |     {   for(auto& r : data)
 34 |         {   for(auto& e : r)
 35 |             {   func(e);
 36 |             }
 37 |         }
 38 |         return *this;
 39 |     }
 40 |     template<typename F>
 41 |     Matrix foreach_n(F func)
 42 |     {   Matrix ret;
 43 |         for (int i = 0; i < ROW; ++i)
 44 |         {   for (int j = 0; j < COL; ++j)
 45 |             {   ret.data[i][j] = func(data[i][j]);
 46 |             }
 47 |         }
 48 |         return ret;
 49 |     }
 50 | 
 51 |     /// 平方和
 52 |     DataType squariance()
 53 |     {   DataType ret = 0;
 54 |         foreach_c([&ret](auto& e){ ret += e * e; });
 55 |         return ret;
 56 |     }
 57 | 
 58 |     /// 归一化
 59 |     void normaliz1()
 60 |     {   DataType sqc = squariance();
 61 |         if (sqc == 0) return;
 62 |         foreach_c([&sqc](auto& e){ e = e / sqc; });
 63 |     }
 64 |     void normalize(DataType max = 0)
 65 |     {   if(max == 0)
 66 |         {   foreach_c([&max](auto &e){ if (std::abs(e) > max) max = std::abs(e); });
 67 |             return;
 68 |         }
 69 |         foreach([max](auto& e){ return (e / max); });
 70 |     }
 71 | 
 72 |     /// 矩阵乘法
 73 |     template<int RC>
 74 |     Matrix& multiply(const Matrix<DataType, ROW, RC>& mX, const Matrix<DataType, RC, COL>& mY)
 75 |     {   for (int i = 0; i < ROW; ++i)
 76 |         {   for (int j = 0; j < COL; ++j)
 77 |             {   data[i][j] = 0;
 78 |                 for (int k = 0; k < RC; ++k)
 79 |                 {   data[i][j] += mX.data[i][k] * mY.data[k][j];
 80 |                 }
 81 |             }
 82 |         }
 83 |         return *this;
 84 |     }
 85 |     /// 矩阵乘法带累加
 86 |     template<int RC>
 87 |     Matrix& mult_sum(const Matrix<DataType, ROW, RC>& mX, const Matrix<DataType, RC, COL>& mY)
 88 |     {   for (int i = 0; i < ROW; ++i)
 89 |         {   for (int j = 0; j < COL; ++j)
 90 |             {   for (int k = 0; k < RC; ++k)
 91 |                 {   data[i][j] += mX.data[i][k] * mY.data[k][j];
 92 |                 }
 93 |             }
 94 |         }
 95 |         return *this;
 96 |     }
 97 |     /// 矩阵乘法带转置
 98 |     template<int RC>
 99 |     Matrix& mult_trans(const Matrix<DataType, COL, RC> &mX, const Matrix<DataType, ROW, RC> &mY)
100 |     {   for (int i = 0; i < ROW; ++i)
101 |         {   for (int j = 0; j < COL; ++j)
102 |             {   data[i][j] = 0;
103 |                 for (int k = 0; k < RC; ++k)
104 |                 {   data[i][j] += mX.data[j][k] * mY.data[i][k];
105 |                 }
106 |             }
107 |         }
108 |         return *this;
109 |     }
110 |     /// 矩阵乘法带转置带累加
111 |     template<int RC>
112 |     Matrix& mult_trans_sum(const Matrix<DataType, COL, RC>& mX, const Matrix<DataType, ROW, RC>& mY)
113 |     {   for (int i = 0; i < ROW; ++i)
114 |         {   for (int j = 0; j < COL; ++j)
115 |             {   for (int k = 0; k < RC; ++k)
116 |                 {   data[i][j] += mX.data[j][k] * mY.data[i][k];
117 |                 }
118 |             }
119 |         }
120 |         return *this;
121 |     }
122 |     /// Matrix(ROW, COL) * Matrix(COL, COL_A)
123 |     template<int COL_A>
124 |     Matrix<DataType, ROW, COL_A> operator*(const Matrix<DataType, COL, COL_A>& m) const
125 |     {   Matrix<DataType, ROW, COL_A> ret;
126 |         for (int i = 0; i < ROW; ++i)
127 |         {   for (int j = 0; j < COL_A; ++j)
128 |             {   ret.data[i][j] = 0;
129 |                 for (int k = 0; k < COL; ++k)
130 |                 {   ret.data[i][j] += data[i][k] * m.data[k][j];
131 |                 }
132 |             }
133 |         }
134 |         return ret;
135 |     }
136 | 
137 |     Matrix operator+(const Matrix& m) const
138 |     {   Matrix ret;
139 |         for (int i = 0; i < ROW; ++i)
140 |         {   for (int j = 0; j < COL; ++j)
141 |             {   ret.data[i][j] = data[i][j] + m.data[i][j];
142 |             }
143 |         }
144 |         return ret;
145 |     }
146 |     Matrix& operator+=(const Matrix& m)
147 |     {   for (int i = 0; i < ROW; ++i)
148 |         {   for (int j = 0; j < COL; ++j)
149 |             {   data[i][j] += m.data[i][j];
150 |             }
151 |         }
152 |         return *this;
153 |     }
154 | 
155 |     Matrix& subtract(const Matrix& mX, const Matrix& mY)
156 |     {   for (int i = 0; i < ROW; ++i)
157 |         {   for (int j = 0; j < COL; ++j)
158 |             {   data[i][j] = mX.data[i][j] - mY.data[i][j];
159 |             }
160 |         }
161 |         return *this;
162 |     }
163 |     Matrix operator-(const Matrix& m) const
164 |     {   Matrix ret;
165 |         for (int i = 0; i < ROW; ++i)
166 |         {   for (int j = 0; j < COL; ++j)
167 |             {   ret.data[i][j] = data[i][j] - m.data[i][j];
168 |             }
169 |         }
170 |         return ret;
171 |     }
172 |     Matrix& operator-=(const Matrix& m)
173 |     {   for (int i = 0; i < ROW; ++i)
174 |         {   for (int j = 0; j < COL; ++j)
175 |             {   data[i][j] -= m.data[i][j];
176 |             }
177 |         }
178 |         return *this;
179 |     }
180 | 
181 |     /// 转置
182 |     Matrix<DataType, COL, ROW> transpose() const
183 |     {   Matrix<DataType, COL, ROW> ret;
184 |         for (int i = 0; i < COL; ++i)
185 |         {   for (int j = 0; j < ROW; ++j)
186 |             {   ret.data[i][j] = data[j][i];
187 |             }
188 |         }
189 |         return ret;
190 |     }
191 | 
192 |     /// Hadamard product 哈达玛积
193 |     Matrix& hadamard(const Matrix& m)
194 |     {   for (int i = 0; i < ROW; ++i)
195 |         {   for (int j = 0; j < COL; ++j)
196 |             {   data[i][j] *= m.data[i][j];
197 |             }
198 |         }
199 |         return *this;
200 |     }
201 |     Matrix& hadamard(const Matrix& mX, const Matrix& mY)
202 |     {   for (int i = 0; i < ROW; ++i)
203 |         {   for (int j = 0; j < COL; ++j)
204 |             {   data[i][j] = mX.data[i][j] * mY.data[i][j];
205 |             }
206 |         }
207 |         return *this;
208 |     }
209 |     Matrix& hadamard_sum(const Matrix& mX, const Matrix& mY)
210 |     {   for (int i = 0; i < ROW; ++i)
211 |         {   for (int j = 0; j < COL; ++j)
212 |             {   data[i][j] += mX.data[i][j] * mY.data[i][j];
213 |             }
214 |         }
215 |         return *this;
216 |     }
217 | 
218 |     /// Kronecker product 克罗内克积（张量积）
219 |     template<int r, int c>
220 |     Matrix<DataType, ROW * r, COL * c> kronecker(const Matrix<DataType, r, c>& m) const
221 |     {   Matrix<DataType, ROW * r, COL * c> ret;
222 |         for (int i = 0; i < ROW; ++i)
223 |         {   for (int j = 0; j < COL; ++j)
224 |             {   for (int a = 0; a < r; ++a)
225 |                 {   for (int b = 0; b < c; ++b)
226 |                     {   ret.data[i * r + a][j * c + b] = data[i][j] * m.data[a][b];
227 |                     }
228 |                 }
229 |             }
230 |         }
231 |         return ret;
232 |     }
233 | 
234 |     /// min到max的随机矩阵
235 |     void random(DataType min, DataType max)
236 |     {   DataType len = (max - min) / (DataType)RAND_MAX;
237 |         foreach([len, min](auto& e){ return min + (DataType)rand() * len; });
238 |     }
239 | 
240 |     /// 构建常值矩阵
241 |     void constant(DataType val)
242 |     {   foreach([val](auto& e){ return val; });
243 |     }
244 | 
245 |     /// 调整权值，算法包括隐含转置的单维张量积
246 |     Matrix& adjustW(const Matrix<DataType, 1, ROW>& mX, const Matrix<DataType, 1, COL>& mY, double learnrate)
247 |     {   for (int i = 0; i < ROW; ++i)
248 |         {   for (int j = 0; j < COL; ++j)
249 |             {   data[i][j] += learnrate * mX.data[0][i] * mY.data[0][j];
250 |             }
251 |         }
252 |         return *this;
253 |     }
254 |     /// 调整阈值
255 |     Matrix& adjustT(const Matrix& m, double learnrate)
256 |     {   for (int i = 0; i < ROW; ++i)
257 |         {   for (int j = 0; j < COL; ++j)
258 |             {   data[i][j] += learnrate * m.data[i][j];
259 |             }
260 |         }
261 |         return *this;
262 |     }
263 | 
264 |     /// 取子集
265 |     template<int R, int C>
266 |     void subset(Matrix<DataType, R, C> m, int offsetR, int offsetC = 0)
267 |     {   for(int r = offsetR; r < R; ++r)
268 |         {   for (int c = offsetC; c < C; ++c)
269 |             {   data[r - offsetR][c - offsetC] = m.data[r][c];
270 |             }
271 |         }
272 |     };
273 |     /// 子集赋值，越界部分会忽略
274 |     template<int R, int C>
275 |     void set(Matrix<DataType, R, C> m, int offsetR = 0, int offsetC = 0)
276 |     {   for(int r = offsetR; r < R && r < ROW; ++r)
277 |         {   for (int c = offsetC; c < C && c < COL; ++c)
278 |             {   data[r][c] = m.data[r - offsetR][c - offsetC];
279 |             }
280 |         }
281 |     };
282 | 
283 | public:
284 |     constexpr int Row() { return ROW; }
285 |     constexpr int Col() { return COL; }
286 | 
287 | public:
288 |     DataType data[ROW][COL];
289 | };
290 | 
291 | }


--------------------------------------------------------------------------------
/math/sigfunc.h:
--------------------------------------------------------------------------------
 1 | //-------------------------------------------------------------------------------
 2 | // sigfunc.h
 3 | //
 4 | // @author
 5 | //     Millhaus.Chen @time 2017/08/02 14:35
 6 | //-------------------------------------------------------------------------------
 7 | #pragma once
 8 | 
 9 | #include <cmath>
10 | 
11 | namespace mtl {
12 | 
13 | inline double tansig(double val)
14 | {
15 |     return 2.0 / (1.0 + std::exp(-2.0 * val)) - 1.0;
16 | }
17 | 
18 | inline double dtansig(double val)
19 | {
20 |     double tanh = tansig(val);
21 |     return 1 - tanh * tanh;
22 | }
23 | 
24 | inline double logsig(double val)
25 | {
26 |     return 1.0 / (1.0 + std::exp(-val));
27 | }
28 | 
29 | inline double dlogsig(double val)
30 | {
31 |     double sigmoid = logsig(val);
32 |     return sigmoid * (1.0 - sigmoid);
33 | }
34 | 
35 | }


--------------------------------------------------------------------------------
/util/TupleTool.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <tuple>
 4 | 
 5 | namespace mtl
 6 | {
 7 | 
 8 | //-------------------------------------------------------------------------------
 9 | // @description
10 | //     代替 c++14 才有的std::get<Type>
11 | //
12 | // @added
13 | //     Millhaus.Chen @time 2015/10/29 13:20
14 | //
15 | // @modified
16 | //     增加get返回默认值的功能 Millhaus.Chen @time 2016/03/14 14:37
17 | //-------------------------------------------------------------------------------
18 | template<size_t N, typename T, typename ...Args>
19 | struct GetType;
20 | 
21 | template<size_t N, typename T, typename Head, typename ...Args>
22 | struct GetType<N, T, Head, Args...>
23 | {
24 |     enum
25 |     {
26 |         value = std::is_same<T, Head>::value ? N : GetType<N + 1, T, Args...>::value
27 |     };
28 | };
29 | 
30 | template<size_t N, typename T>
31 | struct GetType<N, T>
32 | {
33 |     enum
34 |     {
35 |         value = 0
36 |     };
37 | };
38 | 
39 | template<typename T, typename ...Args>
40 | T get(std::tuple<Args...> const &tuple, T &&def_val = T{})
41 | {
42 |     return std::get<GetType<0, T, Args..., T>::value>(
43 |             std::tuple_cat(tuple, std::make_tuple(def_val)));
44 | }
45 | //template<typename T, typename ...Args>
46 | //T get(std::tuple<Args...> const& tuple)
47 | //{
48 | //	return std::get<GetType<0, T, Args...>::value>(tuple);
49 | //}
50 | 
51 | 
52 | //-------------------------------------------------------------------------------
53 | // @description
54 | //     tuple_for_each
55 | //
56 | // @added
57 | //     Millhaus.Chen @time 2017/03/16 10:21
58 | //-------------------------------------------------------------------------------
59 | template<class F, class... Ts, std::size_t... Is>
60 | void for_each(std::tuple<Ts...>& tuple, F func, std::index_sequence<Is...>)
61 | {
62 |     using expander = int[];
63 |     (void) expander {0, ((void) func(std::get<Is>(tuple)), 0)...};
64 | }
65 | 
66 | template<class F, class... Ts>
67 | void for_each(std::tuple<Ts...>& tuple, F func)
68 | {
69 |     for_each(tuple, func, std::make_index_sequence<sizeof...(Ts)>());
70 | }
71 | 
72 | }


--------------------------------------------------------------------------------
/util/UnpackArgs.hpp:
--------------------------------------------------------------------------------
 1 | //-------------------------------------------------------------------------------
 2 | // UnpackArgs.hpp
 3 | //
 4 | // @author
 5 | //     Millhaus.Chen @time 2017/08/01 11:49
 6 | //-------------------------------------------------------------------------------
 7 | #pragma once
 8 | 
 9 | /// Unpack ints from variadic template
10 | /// The compile-time integer array, following RCInt is reverse of ints e.g.
11 | ///    G{5, 3, 2, 4, 2} the (0, G) is 2 and (4, G) is 5
12 | template<int N, int... Tail>
13 | struct RCInt;
14 | template<int N, int Tail>
15 | struct RCInt<N, Tail>
16 | {
17 |     enum { value = Tail };
18 | };
19 | template<int N, int Head, int... Tail>
20 | struct RCInt<N, Head, Tail...>
21 | {
22 |     enum { value = (N == sizeof...(Tail)) ? Head : RCInt<N, Tail...>::value };
23 | };
24 | template<int N, int... Ints>
25 | struct UnpackInts
26 | {
27 |     enum { value = RCInt<(int)sizeof...(Ints) - N - 1, Ints...>::value };
28 | };


--------------------------------------------------------------------------------