├── .gitignore
├── c
    ├── utils.h
    ├── HiddenLayer.h
    ├── DBN.h
    ├── SdA.h
    ├── dA.h
    ├── LogisticRegression.h
    ├── RBM.h
    ├── LogisticRegression.c
    ├── dA.c
    ├── RBM.c
    ├── DBN.c
    └── SdA.c
├── cpp
    ├── HiddenLayer.h
    ├── LogisticRegression.h
    ├── DBN.h
    ├── SdA.h
    ├── dA.h
    ├── utils.cpp
    ├── RBM.h
    ├── HiddenLayer.cpp
    ├── LogisticRegression.cpp
    ├── dA.cpp
    ├── RBM.cpp
    ├── DBN.cpp
    └── SdA.cpp
├── python
    ├── utils.py
    ├── HiddenLayer.py
    ├── CRBM.py
    ├── LogisticRegression.py
    ├── CDBN.py
    ├── dA.py
    ├── RBM.py
    ├── SdA.py
    └── DBN.py
├── README.md
├── java
    ├── DBN
    │   └── src
    │   │   ├── LogisticRegression.java
    │   │   ├── HiddenLayer.java
    │   │   ├── RBM.java
    │   │   └── DBN.java
    ├── SdA
    │   └── src
    │   │   ├── LogisticRegression.java
    │   │   ├── HiddenLayer.java
    │   │   ├── dA.java
    │   │   └── SdA.java
    ├── LogisticRegression
    │   └── src
    │   │   └── LogisticRegression.java
    ├── dA
    │   └── src
    │   │   └── dA.java
    └── RBM
    │   └── src
    │       └── RBM.java
└── scala
    ├── HiddenLayer.scala
    ├── LogisticRegression.scala
    ├── dA.scala
    ├── RBM.scala
    ├── DBN.scala
    └── SdA.scala


/.gitignore:
--------------------------------------------------------------------------------
 1 | .DS_Store
 2 | *.pyc
 3 | *.out
 4 | *.o
 5 | 
 6 | *.class
 7 | *.classpath
 8 | *.settings
 9 | .project
10 | .metadata
11 | 


--------------------------------------------------------------------------------
/c/utils.h:
--------------------------------------------------------------------------------
1 | #ifndef UTILS_H
2 | #define UTILS_H
3 | 
4 | double uniform(double, double);
5 | int binomial(int, double);
6 | double sigmoid(double);
7 | 
8 | #endif
9 | 


--------------------------------------------------------------------------------
/cpp/HiddenLayer.h:
--------------------------------------------------------------------------------
 1 | class HiddenLayer {
 2 | 
 3 | public:
 4 |   int N;
 5 |   int n_in;
 6 |   int n_out;
 7 |   double **W;
 8 |   double *b;
 9 |   HiddenLayer(int, int, int, double**, double*);
10 |   ~HiddenLayer();
11 |   double output(int*, double*, double);
12 |   void sample_h_given_v(int*, int*);
13 | };
14 | 


--------------------------------------------------------------------------------
/cpp/LogisticRegression.h:
--------------------------------------------------------------------------------
 1 | class LogisticRegression {
 2 | 
 3 | public:
 4 |   int N;  // num of inputs
 5 |   int n_in;
 6 |   int n_out;
 7 |   double **W;
 8 |   double *b;
 9 |   LogisticRegression(int, int, int);
10 |   ~LogisticRegression();
11 |   void train(int*, int*, double);
12 |   void softmax(double*);
13 |   void predict(int*, double*);
14 | };
15 | 


--------------------------------------------------------------------------------
/cpp/DBN.h:
--------------------------------------------------------------------------------
 1 | class DBN {
 2 | 
 3 | public:
 4 |   int N;
 5 |   int n_ins;
 6 |   int *hidden_layer_sizes;
 7 |   int n_outs;
 8 |   int n_layers;
 9 |   HiddenLayer **sigmoid_layers;
10 |   RBM **rbm_layers;
11 |   LogisticRegression *log_layer;
12 |   DBN(int, int, int*, int, int);
13 |   ~DBN();
14 |   void pretrain(int*, double, int, int);
15 |   void finetune(int*, int*, double, int);
16 |   void predict(int*, double*);
17 | };
18 | 


--------------------------------------------------------------------------------
/cpp/SdA.h:
--------------------------------------------------------------------------------
 1 | class SdA {
 2 | 
 3 | public:
 4 |   int N;
 5 |   int n_ins;
 6 |   int *hidden_layer_sizes;
 7 |   int n_outs;
 8 |   int n_layers;
 9 |   HiddenLayer **sigmoid_layers;
10 |   dA **dA_layers;
11 |   LogisticRegression *log_layer;
12 |   SdA(int, int, int*, int, int);
13 |   ~SdA();
14 |   void pretrain(int*, double, double, int);
15 |   void finetune(int*, int*, double, int);
16 |   void predict(int*, double*);
17 | };
18 | 


--------------------------------------------------------------------------------
/cpp/dA.h:
--------------------------------------------------------------------------------
 1 | class dA {
 2 | 
 3 | public:
 4 |   int N;
 5 |   int n_visible;
 6 |   int n_hidden;
 7 |   double **W;
 8 |   double *hbias;
 9 |   double *vbias;
10 |   dA(int, int, int , double**, double*, double*);
11 |   ~dA();
12 |   void get_corrupted_input(int*, int*, double);
13 |   void get_hidden_values(int*, double*);
14 |   void get_reconstructed_input(double*, double*);
15 |   void train(int*, double, double);
16 |   void reconstruct(int*, double*);
17 | };
18 | 


--------------------------------------------------------------------------------
/c/HiddenLayer.h:
--------------------------------------------------------------------------------
 1 | #ifndef HIDDENLAYER_H
 2 | #define HIDDENLAYER_H
 3 | 
 4 | typedef struct {
 5 |   int N;
 6 |   int n_in;
 7 |   int n_out;
 8 |   double **W;
 9 |   double *b;
10 | } HiddenLayer;
11 | 
12 | void HiddenLayer__construct(HiddenLayer*, int, int, int, double**, double*);
13 | void HiddenLayer__destruct(HiddenLayer*);
14 | double HiddenLayer_output(HiddenLayer*, int*, double*, double);
15 | void HiddenLayer_sample_h_given_v(HiddenLayer*, int*, int*);
16 | 
17 | #endif
18 | 


--------------------------------------------------------------------------------
/c/DBN.h:
--------------------------------------------------------------------------------
 1 | #ifndef DBN_H
 2 | #define DBN_H
 3 | 
 4 | typedef struct {
 5 |   int N;
 6 |   int n_ins;
 7 |   int *hidden_layer_sizes;
 8 |   int n_outs;
 9 |   int n_layers;
10 |   HiddenLayer *sigmoid_layers;
11 |   RBM *rbm_layers;
12 |   LogisticRegression log_layer;
13 | } DBN;
14 | 
15 | void DBN__construct(DBN*, int, int, int*, int, int);
16 | void DBN__destruct(DBN*);
17 | void DBN_pretrain(DBN*, int*, double, int, int);
18 | void DBN_finetune(DBN*, int*, int*, double, int);
19 | void DBN_predict(DBN*, int*, double*);
20 | 
21 | #endif
22 | 


--------------------------------------------------------------------------------
/c/SdA.h:
--------------------------------------------------------------------------------
 1 | #ifndef SDA_H
 2 | #define SDA_H
 3 | 
 4 | typedef struct {
 5 |   int N;
 6 |   int n_ins;
 7 |   int *hidden_layer_sizes;
 8 |   int n_outs;
 9 |   int n_layers;
10 |   HiddenLayer *sigmoid_layers;
11 |   dA *dA_layers;
12 |   LogisticRegression log_layer;
13 | } SdA;
14 | 
15 | void SdA__construct(SdA*, int, int, int*, int, int);
16 | void SdA__destruct(SdA*);
17 | void SdA_pretrain(SdA*, int*, double, double, int);
18 | void SdA_finetune(SdA*, int*, int*, double, int);
19 | void SdA_predict(SdA*, int*, double*);
20 | 
21 | #endif
22 | 


--------------------------------------------------------------------------------
/cpp/utils.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <math.h>
 3 | using namespace std;
 4 | 
 5 | double uniform(double min, double max) {
 6 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;
 7 | }
 8 | 
 9 | int binomial(int n, double p) {
10 |   if(p < 0 || p > 1) return 0;
11 |   
12 |   int c = 0;
13 |   double r;
14 |   
15 |   for(int i=0; i<n; i++) {
16 |     r = rand() / (RAND_MAX + 1.0);
17 |     if (r < p) c++;
18 |   }
19 | 
20 |   return c;
21 | }
22 | 
23 | double sigmoid(double x) {
24 |   return 1.0 / (1.0 + exp(-x));
25 | }
26 | 


--------------------------------------------------------------------------------
/c/dA.h:
--------------------------------------------------------------------------------
 1 | #ifndef DA_H
 2 | #define DA_H
 3 | 
 4 | typedef struct {
 5 |   int N;
 6 |   int n_visible;
 7 |   int n_hidden;
 8 |   double **W;
 9 |   double *hbias;
10 |   double *vbias;
11 | } dA;
12 | 
13 | void dA__construct(dA*, int, int, int, double**, double*, double*);
14 | void dA__destruct(dA*);
15 | void dA_get_corrupted_input(dA*, int*, int*, double);
16 | void dA_get_hidden_values(dA*, int*, double*);
17 | void dA_get_reconstructed_input(dA*, double*, double*);
18 | void dA_train(dA*, int*, double, double);
19 | void dA_reconstruct(dA*, int*, double*);
20 | 
21 | #endif
22 | 


--------------------------------------------------------------------------------
/cpp/RBM.h:
--------------------------------------------------------------------------------
 1 | class RBM {
 2 | 
 3 | public:
 4 |   int N;
 5 |   int n_visible;
 6 |   int n_hidden;
 7 |   double **W;
 8 |   double *hbias;
 9 |   double *vbias;
10 |   RBM(int, int, int, double**, double*, double*);
11 |   ~RBM();
12 |   void contrastive_divergence(int*, double, int);
13 |   void sample_h_given_v(int*, double*, int*);
14 |   void sample_v_given_h(int*, double*, int*);
15 |   double propup(int*, double*, double);
16 |   double propdown(int*, int, double);
17 |   void gibbs_hvh(int*, double*, int*, double*, int*);
18 |   void reconstruct(int*, double*);
19 | };
20 | 
21 | 


--------------------------------------------------------------------------------
/c/LogisticRegression.h:
--------------------------------------------------------------------------------
 1 | #ifndef LOGISTICREGRESSION_H
 2 | #define LOGISTICREGRESSION_H
 3 | 
 4 | typedef struct {
 5 |   int N;
 6 |   int n_in;
 7 |   int n_out;
 8 |   double **W;
 9 |   double *b;
10 | } LogisticRegression;
11 | 
12 | void LogisticRegression__construct(LogisticRegression*, int, int, int);
13 | void LogisticRegression__destruct(LogisticRegression*);
14 | void LogisticRegression_train(LogisticRegression*, int*, int*, double);
15 | void LogisticRegression_softmax(LogisticRegression*, double*);
16 | void LogisticRegression_predict(LogisticRegression*, int*, double*);
17 | 
18 | #endif
19 | 


--------------------------------------------------------------------------------
/python/utils.py:
--------------------------------------------------------------------------------
 1 | ''' '''
 2 | import numpy
 3 | 
 4 | 
 5 | numpy.seterr(all='ignore')
 6 | 
 7 | 
 8 | def sigmoid(x):
 9 |     return 1. / (1 + numpy.exp(-x))
10 | 
11 | 
12 | def softmax(x):
13 |     e = numpy.exp(x - numpy.max(x))  # prevent overflow
14 |     if e.ndim == 1:
15 |         return e / numpy.sum(e, axis=0)
16 |     else:  
17 |         return e / numpy.array([numpy.sum(e, axis=1)]).T  # ndim = 2
18 | 
19 | 
20 | # # probability density for the Gaussian dist
21 | # def gaussian(x, mean=0.0, scale=1.0):
22 | #     s = 2 * numpy.power(scale, 2)
23 | #     e = numpy.exp( - numpy.power((x - mean), 2) / s )
24 | 
25 | #     return e / numpy.square(numpy.pi * s)
26 | 
27 | 


--------------------------------------------------------------------------------
/c/RBM.h:
--------------------------------------------------------------------------------
 1 | #ifndef RBM_H
 2 | #define RBM_H
 3 | 
 4 | typedef struct {
 5 |   int N;
 6 |   int n_visible;
 7 |   int n_hidden;
 8 |   double **W;
 9 |   double *hbias;
10 |   double *vbias;
11 | } RBM;
12 | 
13 | void RBM__construct(RBM*, int, int, int, double**, double*, double*);
14 | void RBM__destruct(RBM*);
15 | void RBM_contrastive_divergence(RBM*, int*, double, int);
16 | void RBM_sample_h_given_v(RBM*, int*, double*, int*);
17 | void RBM_sample_v_given_h(RBM*, int*, double*, int*);
18 | double RBM_propup(RBM*, int*, double*, double);
19 | double RBM_propdown(RBM*, int*, int, double);
20 | void RBM_gibbs_hvh(RBM*, int*, double*, int*, double*, int*);
21 | void RBM_reconstruct(RBM*, int*, double*);
22 | 
23 | #endif
24 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Deep Learning (Python, C/C++, Java)
 2 | 
 3 | ### Classes :
 4 | 
 5 |   - DBN:  Deep Belief Nets
 6 | 
 7 |   - CDBN: Deep Belief Nets w/ continuous-valued inputs 
 8 | 
 9 |   - RBM:  Restricted Boltzmann Machine
10 | 
11 |   - CRBM: Restricted Boltzmann Machine w/ continuous-valued inputs
12 | 
13 |   - dA:   Denoising Autoencoders
14 | 
15 |   - SdA:  Stacked Denoising Autoencoders
16 | 
17 |   - LogisticRegression: Logistic Regression
18 | 
19 |   - HiddenLayer: Hidden Layer of Neural Networks
20 | 
21 | 
22 | 
23 | ### References :
24 |   - Y. Bengio, P. Lamblin, D. Popovici, H. Larochelle: Greedy Layer-Wise
25 |   Training of Deep Networks, Advances in Neural Information Processing
26 |   Systems 19, 2007
27 | 
28 |   - P. Vincent, H. Larochelle, Y. Bengio, P.A. Manzagol: Extracting and
29 |   Composing Robust Features with Denoising Autoencoders, ICML' 08, 1096-1103,
30 |   2008
31 | 
32 | 
33 |   - DeepLearningTutorials
34 |   https://github.com/lisa-lab/DeepLearningTutorials
35 | 
36 |   - Yusuke Sugomori: Stochastic Gradient Descent for Denoising Autoencoders,
37 |   http://yusugomori.com/docs/SGD_DA.pdf
38 | 
39 | 
40 | 
41 | ### Bug reports are deeply welcome.


--------------------------------------------------------------------------------
/java/DBN/src/LogisticRegression.java:
--------------------------------------------------------------------------------
 1 | 
 2 | public class LogisticRegression {
 3 | 	public int N;
 4 | 	public int n_in;
 5 | 	public int n_out;
 6 | 	public double[][] W;
 7 | 	public double[] b;
 8 | 	
 9 | 	public LogisticRegression(int N, int n_in, int n_out) {
10 | 		this.N = N;
11 | 		this.n_in = n_in;
12 | 		this.n_out = n_out;
13 | 		
14 | 		W = new double[this.n_out][this.n_in];
15 | 		b = new double[this.n_out];
16 | 	}
17 | 	
18 | 	public void train(int[] x, int[] y, double lr) {
19 | 		double[] p_y_given_x = new double[n_out];
20 | 		double[] dy = new double[n_out];
21 | 		
22 | 		for(int i=0; i<n_out; i++) {
23 | 			p_y_given_x[i] = 0;
24 | 			for(int j=0; j<n_in; j++) {
25 | 				p_y_given_x[i] += W[i][j] * x[j];
26 | 			}
27 | 			p_y_given_x[i] += b[i];
28 | 		}
29 | 		softmax(p_y_given_x);
30 | 		
31 | 		for(int i=0; i<n_out; i++) {
32 | 			dy[i] = y[i] - p_y_given_x[i];
33 | 			
34 | 			for(int j=0; j<n_in; j++) {
35 | 				W[i][j] += lr * dy[i] * x[j] / N;
36 | 			}
37 | 			
38 | 			b[i] += lr * dy[i] / N;
39 | 		}
40 | 	}
41 | 	
42 | 	public void softmax(double[] x) {
43 | 		double max = 0.0;
44 | 		double sum = 0.0;
45 | 		
46 | 		for(int i=0; i<n_out; i++) {
47 | 			if(max < x[i]) {
48 | 				max = x[i];
49 | 			}
50 | 		}
51 | 		
52 | 		for(int i=0; i<n_out; i++) {
53 | 			x[i] = Math.exp(x[i] - max);
54 | 			sum += x[i];
55 | 		}
56 | 		
57 | 		for(int i=0; i<n_out; i++) {
58 | 			x[i] /= sum;
59 | 		}
60 | 	}
61 | 	
62 | 	public void predict(int[] x, double[] y) {
63 | 		for(int i=0; i<n_out; i++) {
64 | 			y[i] = 0;
65 | 			for(int j=0; j<n_in; j++) {
66 | 				y[i] += W[i][j] * x[j];
67 | 			}
68 | 			y[i] += b[i];
69 | 		}
70 | 		
71 | 		softmax(y);
72 | 	}		
73 | }
74 | 


--------------------------------------------------------------------------------
/java/SdA/src/LogisticRegression.java:
--------------------------------------------------------------------------------
 1 | 
 2 | public class LogisticRegression {
 3 | 	public int N;
 4 | 	public int n_in;
 5 | 	public int n_out;
 6 | 	public double[][] W;
 7 | 	public double[] b;
 8 | 	
 9 | 	public LogisticRegression(int N, int n_in, int n_out) {
10 | 		this.N = N;
11 | 		this.n_in = n_in;
12 | 		this.n_out = n_out;
13 | 		
14 | 		W = new double[this.n_out][this.n_in];
15 | 		b = new double[this.n_out];
16 | 	}
17 | 	
18 | 	public void train(int[] x, int[] y, double lr) {
19 | 		double[] p_y_given_x = new double[n_out];
20 | 		double[] dy = new double[n_out];
21 | 		
22 | 		for(int i=0; i<n_out; i++) {
23 | 			p_y_given_x[i] = 0;
24 | 			for(int j=0; j<n_in; j++) {
25 | 				p_y_given_x[i] += W[i][j] * x[j];
26 | 			}
27 | 			p_y_given_x[i] += b[i];
28 | 		}
29 | 		softmax(p_y_given_x);
30 | 		
31 | 		for(int i=0; i<n_out; i++) {
32 | 			dy[i] = y[i] - p_y_given_x[i];
33 | 			
34 | 			for(int j=0; j<n_in; j++) {
35 | 				W[i][j] += lr * dy[i] * x[j] / N;
36 | 			}
37 | 			
38 | 			b[i] += lr * dy[i] / N;
39 | 		}
40 | 	}
41 | 	
42 | 	public void softmax(double[] x) {
43 | 		double max = 0.0;
44 | 		double sum = 0.0;
45 | 		
46 | 		for(int i=0; i<n_out; i++) {
47 | 			if(max < x[i]) {
48 | 				max = x[i];
49 | 			}
50 | 		}
51 | 		
52 | 		for(int i=0; i<n_out; i++) {
53 | 			x[i] = Math.exp(x[i] - max);
54 | 			sum += x[i];
55 | 		}
56 | 		
57 | 		for(int i=0; i<n_out; i++) {
58 | 			x[i] /= sum;
59 | 		}
60 | 	}
61 | 	
62 | 	public void predict(int[] x, double[] y) {
63 | 		for(int i=0; i<n_out; i++) {
64 | 			y[i] = 0;
65 | 			for(int j=0; j<n_in; j++) {
66 | 				y[i] += W[i][j] * x[j];
67 | 			}
68 | 			y[i] += b[i];
69 | 		}
70 | 		
71 | 		softmax(y);
72 | 	}		
73 | }
74 | 


--------------------------------------------------------------------------------
/cpp/HiddenLayer.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <math.h>
 3 | #include "HiddenLayer.h"
 4 | using namespace std;
 5 | 
 6 | double uniform(double min, double max) {
 7 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;
 8 | }
 9 | 
10 | int binomial(int n, double p) {
11 |   if(p < 0 || p > 1) return 0;
12 |   
13 |   int c = 0;
14 |   double r;
15 |   
16 |   for(int i=0; i<n; i++) {
17 |     r = rand() / (RAND_MAX + 1.0);
18 |     if (r < p) c++;
19 |   }
20 | 
21 |   return c;
22 | }
23 | 
24 | double sigmoid(double x) {
25 |   return 1.0 / (1.0 + exp(-x));
26 | }
27 | 
28 | 
29 | 
30 | HiddenLayer::HiddenLayer(int size, int in, int out, double **w, double *bp) {
31 |   N = size;
32 |   n_in = in;
33 |   n_out = out;
34 | 
35 |   if(w == NULL) {
36 |     W = new double*[n_out];
37 |     for(int i=0; i<n_out; i++) W[i] = new double[n_in];
38 |     double a = 1.0 / n_in;
39 | 
40 |     for(int i=0; i<n_out; i++) {
41 |       for(int j=0; j<n_in; j++) {
42 |         W[i][j] = uniform(-a, a);
43 |       }
44 |     }
45 |   } else {
46 |     W = w;
47 |   }
48 | 
49 |   if(bp == NULL) {
50 |     b = new double[n_out];
51 |   } else {
52 |     b = bp;
53 |   }
54 | }
55 | 
56 | HiddenLayer::~HiddenLayer() {
57 |   for(int i=0; i<n_out; i++) delete W[i];
58 |   delete[] W;
59 |   delete[] b;
60 | }
61 | 
62 | double HiddenLayer::output(int *input, double *w, double b) {
63 |   double linear_output = 0.0;
64 |   for(int j=0; j<n_in; j++) {
65 |     linear_output += w[j] * input[j];
66 |   }
67 |   linear_output += b;
68 |   return sigmoid(linear_output);
69 | }
70 | 
71 | void HiddenLayer::sample_h_given_v(int *input, int *sample) {
72 |   for(int i=0; i<n_out; i++) {
73 |     sample[i] = binomial(1, output(input, W[i], b[i]));
74 |   }
75 | }
76 | 
77 | 
78 | 
79 | int main() {
80 |   return 0;
81 | }
82 | 


--------------------------------------------------------------------------------
/scala/HiddenLayer.scala:
--------------------------------------------------------------------------------
 1 | import scala.util.Random
 2 | import scala.math
 3 | 
 4 | class HiddenLayer(val N: Int, val n_in: Int, val n_out: Int, _W: Array[Array[Double]], _b: Array[Double], var rng: Random=null) {
 5 | 
 6 | 
 7 |   def uniform(min: Double, max: Double): Double = {
 8 |     return rng.nextDouble() * (max - min) + min
 9 |   }
10 | 
11 |   def binomial(n: Int, p: Double): Int = {
12 |     if(p < 0 || p > 1) return 0
13 | 
14 |     var c: Int = 0
15 |     var r: Double = 0.0
16 | 
17 |     var i: Int = 0
18 | 
19 |     for(i <- 0 until n) {
20 |       r = rng.nextDouble()
21 |       if(r < p) c += 1
22 |     }
23 | 
24 |     return c
25 |   }
26 | 
27 |   def sigmoid(x: Double): Double = {
28 |     return 1.0 / (1.0 + math.pow(math.E, -x))
29 |   }
30 | 
31 | 
32 |   if(rng == null) rng = new Random(1234)
33 | 
34 |   var a: Double = 0.0
35 |   var W: Array[Array[Double]] = Array.ofDim[Double](n_out, n_in)
36 |   var b: Array[Double] = new Array[Double](n_out)
37 | 
38 |   var i: Int = 0
39 |   if(_W == null) {
40 |     a = 1.0 / n_in
41 | 
42 |     for(i <- 0 until n_out) {
43 |       for(j <- 0 until n_in) {
44 |         W(i)(j) = uniform(-a, a)
45 |       }
46 |     }
47 |   } else {
48 |     W = _W
49 |   }
50 | 
51 |   if(_b != null) b = _b
52 | 
53 | 
54 |   def output(input: Array[Int], w: Array[Double], b: Double): Double = {
55 |     var linear_output: Double = 0.0
56 | 
57 |     var j: Int = 0
58 |     for(j <- 0 until n_in) {
59 |       linear_output += w(j) * input(j)
60 |     }
61 |     linear_output += b
62 | 
63 |     return sigmoid(linear_output)
64 |   }
65 | 
66 |   def sample_h_given_v(input: Array[Int], sample: Array[Int]) {
67 |     var i: Int = 0
68 |     
69 |     for(i <- 0 until n_out) {
70 |       sample(i) = binomial(1, output(input, W(i), b(i)))
71 |     }
72 |   }
73 | }
74 | 


--------------------------------------------------------------------------------
/java/DBN/src/HiddenLayer.java:
--------------------------------------------------------------------------------
 1 | import java.util.Random;
 2 | 
 3 | public class HiddenLayer {
 4 | 	public int N;
 5 | 	public int n_in;
 6 | 	public int n_out;
 7 | 	public double[][] W;
 8 | 	public double[] b;
 9 | 	public Random rng;
10 | 	
11 | 	public double uniform(double min, double max) {
12 | 		return rng.nextDouble() * (max - min) + min;
13 | 	}
14 | 	
15 | 	public int binomial(int n, double p) {
16 | 		if(p < 0 || p > 1) return 0;
17 | 		
18 | 		int c = 0;
19 | 		double r;
20 | 		
21 | 		for(int i=0; i<n; i++) {
22 | 			r = rng.nextDouble();
23 | 			if (r < p) c++;
24 | 		}
25 | 		
26 | 		return c;
27 | 	}
28 | 	
29 | 	public static double sigmoid(double x) {
30 | 		return 1.0 / (1.0 + Math.pow(Math.E, -x));
31 | 	}
32 | 	
33 | 	
34 | 	
35 | 	public HiddenLayer(int N, int n_in, int n_out, double[][] W, double[] b, Random rng) {
36 | 		this.N = N;
37 | 		this.n_in = n_in;
38 | 		this.n_out = n_out;
39 | 		
40 | 		if(rng == null)	this.rng = new Random(1234);
41 | 		else this.rng = rng;
42 | 	
43 | 		if(W == null) {
44 | 			this.W = new double[n_out][n_in];
45 | 			double a = 1.0 / this.n_in;
46 | 			
47 | 			for(int i=0; i<n_out; i++) {
48 | 				for(int j=0; j<n_in; j++) {
49 | 					this.W[i][j] = uniform(-a, a);
50 | 				}
51 | 			}
52 | 		} else {
53 | 			this.W = W;
54 | 		}
55 | 		
56 | 		if(b == null) this.b = new double[n_out];
57 | 		else this.b = b;
58 | 	}
59 | 	
60 | 	public double output(int[] input, double[] w, double b) {
61 | 		double linear_output = 0.0;
62 | 		for(int j=0; j<n_in; j++) {
63 | 			linear_output += w[j] * input[j];
64 | 		}
65 | 		linear_output += b;
66 | 		return sigmoid(linear_output);
67 | 	}
68 | 	
69 | 	public void sample_h_given_v(int[] input, int[] sample) {
70 | 		for(int i=0; i<n_out; i++) {
71 | 			sample[i] = binomial(1, output(input, W[i], b[i]));
72 | 		}
73 | 	}
74 | }
75 | 


--------------------------------------------------------------------------------
/java/SdA/src/HiddenLayer.java:
--------------------------------------------------------------------------------
 1 | import java.util.Random;
 2 | 
 3 | public class HiddenLayer {
 4 | 	public int N;
 5 | 	public int n_in;
 6 | 	public int n_out;
 7 | 	public double[][] W;
 8 | 	public double[] b;
 9 | 	public Random rng;
10 | 	
11 | 	public double uniform(double min, double max) {
12 | 		return rng.nextDouble() * (max - min) + min;
13 | 	}
14 | 	
15 | 	public int binomial(int n, double p) {
16 | 		if(p < 0 || p > 1) return 0;
17 | 		
18 | 		int c = 0;
19 | 		double r;
20 | 		
21 | 		for(int i=0; i<n; i++) {
22 | 			r = rng.nextDouble();
23 | 			if (r < p) c++;
24 | 		}
25 | 		
26 | 		return c;
27 | 	}
28 | 	
29 | 	public static double sigmoid(double x) {
30 | 		return 1.0 / (1.0 + Math.pow(Math.E, -x));
31 | 	}
32 | 	
33 | 	
34 | 	
35 | 	public HiddenLayer(int N, int n_in, int n_out, double[][] W, double[] b, Random rng) {
36 | 		this.N = N;
37 | 		this.n_in = n_in;
38 | 		this.n_out = n_out;
39 | 		
40 | 		if(rng == null)	this.rng = new Random(1234);
41 | 		else this.rng = rng;
42 | 	
43 | 		if(W == null) {
44 | 			this.W = new double[n_out][n_in];
45 | 			double a = 1.0 / this.n_in;
46 | 			
47 | 			for(int i=0; i<n_out; i++) {
48 | 				for(int j=0; j<n_in; j++) {
49 | 					this.W[i][j] = uniform(-a, a);
50 | 				}
51 | 			}
52 | 		} else {
53 | 			this.W = W;
54 | 		}
55 | 		
56 | 		if(b == null) this.b = new double[n_out];
57 | 		else this.b = b;
58 | 	}
59 | 	
60 | 	public double output(int[] input, double[] w, double b) {
61 | 		double linear_output = 0.0;
62 | 		for(int j=0; j<n_in; j++) {
63 | 			linear_output += w[j] * input[j];
64 | 		}
65 | 		linear_output += b;
66 | 		return sigmoid(linear_output);
67 | 	}
68 | 	
69 | 	public void sample_h_given_v(int[] input, int[] sample) {
70 | 		for(int i=0; i<n_out; i++) {
71 | 			sample[i] = binomial(1, output(input, W[i], b[i]));
72 | 		}
73 | 	}
74 | }
75 | 


--------------------------------------------------------------------------------
/python/HiddenLayer.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | '''
 5 |  Hidden Layer
 6 | 
 7 |  References :
 8 |    - DeepLearningTutorials
 9 |    https://github.com/lisa-lab/DeepLearningTutorials
10 | 
11 | '''
12 | 
13 | import sys
14 | import numpy
15 | from utils import *
16 | 
17 | 
18 | class HiddenLayer(object):
19 |     def __init__(self, input, n_in, n_out,\
20 |                  W=None, b=None, numpy_rng=None, activation=numpy.tanh):
21 |         
22 |         if numpy_rng is None:
23 |             numpy_rng = numpy.random.RandomState(1234)
24 | 
25 |         if W is None:
26 |             a = 1. / n_in
27 |             initial_W = numpy.array(numpy_rng.uniform(  # initialize W uniformly
28 |                 low=-a,
29 |                 high=a,
30 |                 size=(n_in, n_out)))
31 | 
32 |             W = initial_W
33 | 
34 |         if b is None:
35 |             b = numpy.zeros(n_out)  # initialize bias 0
36 | 
37 | 
38 |         self.numpy_rng = numpy_rng
39 |         self.input = input
40 |         self.W = W
41 |         self.b = b
42 | 
43 |         self.activation = activation
44 | 
45 |         # self.params = [self.W, self.b]
46 | 
47 |     def output(self, input=None):
48 |         if input is not None:
49 |             self.input = input
50 |         
51 |         linear_output = numpy.dot(self.input, self.W) + self.b
52 | 
53 |         return (linear_output if self.activation is None
54 |                 else self.activation(linear_output))
55 | 
56 | 
57 |     def sample_h_given_v(self, input=None):
58 |         if input is not None:
59 |             self.input = input
60 | 
61 |         v_mean = self.output()
62 |         h_sample = self.numpy_rng.binomial(size=v_mean.shape,
63 |                                            n=1,
64 |                                            p=v_mean)
65 |         return h_sample
66 | 


--------------------------------------------------------------------------------
/python/CRBM.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | """
 5 |  RBM  w/ continuous-valued inputs (Linear Energy)
 6 | 
 7 |  References :
 8 |    - Y. Bengio, P. Lamblin, D. Popovici, H. Larochelle: Greedy Layer-Wise
 9 |    Training of Deep Networks, Advances in Neural Information Processing
10 |    Systems 19, 2007
11 | 
12 | """
13 | 
14 | import sys
15 | import numpy
16 | from RBM import RBM
17 | from utils import *
18 | 
19 | 
20 | class CRBM(RBM):
21 |     def propdown(self, h):
22 |         pre_activation = numpy.dot(h, self.W.T) + self.vbias
23 |         return pre_activation
24 |         
25 | 
26 | 
27 |     def sample_v_given_h(self, h0_sample):
28 |         a_h = self.propdown(h0_sample)
29 |         en = numpy.exp(-a_h)
30 |         ep = numpy.exp(a_h)
31 | 
32 |         v1_mean = 1 / (1 - en) - 1 / a_h
33 |         U = numpy.array(self.numpy_rng.uniform(
34 |             low=0,
35 |             high=1,
36 |             size=v1_mean.shape))
37 | 
38 |         v1_sample = numpy.log((1 - U * (1 - ep))) / a_h
39 | 
40 |         return [v1_mean, v1_sample]
41 | 
42 | 
43 | 
44 | def test_crbm(learning_rate=0.1, k=1, training_epochs=1000):
45 |     data = numpy.array([[0.4, 0.5, 0.5, 0.,  0.,  0.],
46 |                         [0.5, 0.3,  0.5, 0.,  0.,  0.],
47 |                         [0.4, 0.5, 0.5, 0.,  0.,  0.],
48 |                         [0.,  0.,  0.5, 0.3, 0.5, 0.],
49 |                         [0.,  0.,  0.5, 0.4, 0.5, 0.],
50 |                         [0.,  0.,  0.5, 0.5, 0.5, 0.]])
51 | 
52 | 
53 |     rng = numpy.random.RandomState(123)
54 | 
55 |     # construct CRBM
56 |     rbm = CRBM(input=data, n_visible=6, n_hidden=5, numpy_rng=rng)
57 | 
58 |     # train
59 |     for epoch in xrange(training_epochs):
60 |         rbm.contrastive_divergence(lr=learning_rate, k=k)
61 |         # cost = rbm.get_reconstruction_cross_entropy()
62 |         # print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
63 | 
64 | 
65 |     # test
66 |     v = numpy.array([[0.5, 0.5, 0., 0., 0., 0.],
67 |                      [0., 0., 0., 0.5, 0.5, 0.]])
68 | 
69 |     print rbm.reconstruct(v)
70 | 
71 | 
72 | if __name__ == "__main__":
73 |     test_crbm()
74 | 


--------------------------------------------------------------------------------
/python/LogisticRegression.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | '''
 5 |  Logistic Regression
 6 |  
 7 |  References :
 8 |    - Jason Rennie: Logistic Regression,
 9 |    http://qwone.com/~jason/writing/lr.pdf
10 |  
11 |    - DeepLearningTutorials
12 |    https://github.com/lisa-lab/DeepLearningTutorials
13 | 
14 | '''
15 | 
16 | import sys
17 | import numpy
18 | from utils import *
19 | 
20 | 
21 | class LogisticRegression(object):
22 |     def __init__(self, input, label, n_in, n_out):
23 |         self.x = input
24 |         self.y = label
25 |         self.W = numpy.zeros((n_in, n_out))  # initialize W 0
26 |         self.b = numpy.zeros(n_out)          # initialize bias 0
27 | 
28 |         # self.params = [self.W, self.b]
29 | 
30 |     def train(self, lr=0.1, input=None, L2_reg=0.00):
31 |         if input is not None:
32 |             self.x = input
33 | 
34 |         # p_y_given_x = sigmoid(numpy.dot(self.x, self.W) + self.b)
35 |         p_y_given_x = softmax(numpy.dot(self.x, self.W) + self.b)
36 |         d_y = self.y - p_y_given_x
37 |         
38 |         self.W += lr * numpy.dot(self.x.T, d_y) - lr * L2_reg * self.W
39 |         self.b += lr * numpy.mean(d_y, axis=0)
40 |         
41 |         # cost = self.negative_log_likelihood()
42 |         # return cost
43 | 
44 |     def negative_log_likelihood(self):
45 |         # sigmoid_activation = sigmoid(numpy.dot(self.x, self.W) + self.b)
46 |         sigmoid_activation = softmax(numpy.dot(self.x, self.W) + self.b)
47 | 
48 |         cross_entropy = - numpy.mean(
49 |             numpy.sum(self.y * numpy.log(sigmoid_activation) +
50 |             (1 - self.y) * numpy.log(1 - sigmoid_activation),
51 |                       axis=1))
52 | 
53 |         return cross_entropy
54 | 
55 | 
56 |     def predict(self, x):
57 |         # return sigmoid(numpy.dot(x, self.W) + self.b)
58 |         return softmax(numpy.dot(x, self.W) + self.b)
59 | 
60 | 
61 | def test_lr(learning_rate=0.01, n_epochs=200):
62 |     # training data
63 |     x = numpy.array([[1,1,1,0,0,0],
64 |                      [1,0,1,0,0,0],
65 |                      [1,1,1,0,0,0],
66 |                      [0,0,1,1,1,0],
67 |                      [0,0,1,1,0,0],
68 |                      [0,0,1,1,1,0]])
69 |     y = numpy.array([[1, 0],
70 |                      [1, 0],
71 |                      [1, 0],
72 |                      [0, 1],
73 |                      [0, 1],
74 |                      [0, 1]])
75 | 
76 | 
77 |     # construct LogisticRegression
78 |     classifier = LogisticRegression(input=x, label=y, n_in=6, n_out=2)
79 | 
80 |     # train
81 |     for epoch in xrange(n_epochs):
82 |         classifier.train(lr=learning_rate)
83 |         # cost = classifier.negative_log_likelihood()
84 |         # print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
85 |         learning_rate *= 0.95
86 | 
87 | 
88 |     # test
89 |     x = numpy.array([[1, 1, 0, 0, 0, 0],
90 |                      [0, 0, 0, 1, 1, 0],
91 |                      [1, 1, 1, 1, 1, 0]])
92 | 
93 |     print >> sys.stderr, classifier.predict(x)
94 | 
95 | 
96 | if __name__ == "__main__":
97 |     test_lr()
98 | 


--------------------------------------------------------------------------------
/java/LogisticRegression/src/LogisticRegression.java:
--------------------------------------------------------------------------------
  1 | 
  2 | public class LogisticRegression {
  3 | 	public int N;
  4 | 	public int n_in;
  5 | 	public int n_out;
  6 | 	public double[][] W;
  7 | 	public double[] b;
  8 | 	
  9 | 	public LogisticRegression(int N, int n_in, int n_out) {
 10 | 		this.N = N;
 11 | 		this.n_in = n_in;
 12 | 		this.n_out = n_out;
 13 | 		
 14 | 		W = new double[this.n_out][this.n_in];
 15 | 		b = new double[this.n_out];
 16 | 	}
 17 | 	
 18 | 	public void train(int[] x, int[] y, double lr) {
 19 | 		double[] p_y_given_x = new double[n_out];
 20 | 		double[] dy = new double[n_out];
 21 | 		
 22 | 		for(int i=0; i<n_out; i++) {
 23 | 			p_y_given_x[i] = 0;
 24 | 			for(int j=0; j<n_in; j++) {
 25 | 				p_y_given_x[i] += W[i][j] * x[j];
 26 | 			}
 27 | 			p_y_given_x[i] += b[i];
 28 | 		}
 29 | 		softmax(p_y_given_x);
 30 | 		
 31 | 		for(int i=0; i<n_out; i++) {
 32 | 			dy[i] = y[i] - p_y_given_x[i];
 33 | 			
 34 | 			for(int j=0; j<n_in; j++) {
 35 | 				W[i][j] += lr * dy[i] * x[j] / N;
 36 | 			}
 37 | 			
 38 | 			b[i] += lr * dy[i] / N;
 39 | 		}
 40 | 	}
 41 | 	
 42 | 	public void softmax(double[] x) {
 43 | 		double max = 0.0;
 44 | 		double sum = 0.0;
 45 | 		
 46 | 		for(int i=0; i<n_out; i++) if(max < x[i]) max = x[i];
 47 | 		
 48 | 		for(int i=0; i<n_out; i++) {
 49 | 			x[i] = Math.exp(x[i] - max);
 50 | 			sum += x[i];
 51 | 		}
 52 | 		
 53 | 		for(int i=0; i<n_out; i++) x[i] /= sum;
 54 | 	}
 55 | 	
 56 | 	public void predict(int[] x, double[] y) {
 57 | 		for(int i=0; i<n_out; i++) {
 58 | 			y[i] = 0;
 59 | 			for(int j=0; j<n_in; j++) {
 60 | 				y[i] += W[i][j] * x[j];
 61 | 			}
 62 | 			y[i] += b[i];
 63 | 		}
 64 | 		
 65 | 		softmax(y);
 66 | 	}	
 67 | 	
 68 | 	private static void test_lr() {
 69 | 		double learning_rate = 0.1;
 70 | 		double n_epochs = 500;
 71 | 		
 72 | 		int train_N = 6;
 73 | 		int test_N = 2;
 74 | 		int n_in = 6;
 75 | 		int n_out = 2;
 76 | 		
 77 | 		int[][] train_X = {
 78 | 			{1, 1, 1, 0, 0, 0},
 79 | 			{1, 0, 1, 0, 0, 0},
 80 | 			{1, 1, 1, 0, 0, 0},
 81 | 			{0, 0, 1, 1, 1, 0},
 82 | 			{0, 0, 1, 1, 0, 0},
 83 | 			{0, 0, 1, 1, 1, 0}
 84 | 		};
 85 | 		
 86 | 		int[][] train_Y = {
 87 | 			{1, 0},
 88 | 			{1, 0},
 89 | 			{1, 0},
 90 | 			{0, 1},
 91 | 			{0, 1},
 92 | 			{0, 1}
 93 | 		}; 
 94 | 		
 95 | 		// construct
 96 | 		LogisticRegression classifier = new LogisticRegression(train_N, n_in, n_out);
 97 | 		
 98 | 		// train 
 99 | 		for(int epoch=0; epoch<n_epochs; epoch++) {
100 | 			for(int i=0; i<train_N; i++) {
101 | 				classifier.train(train_X[i], train_Y[i], learning_rate);
102 | 			}
103 | 			//learning_rate *= 0.95;
104 | 		}
105 | 		
106 | 		// test data
107 | 		int[][] test_X = {
108 | 			{1, 0, 1, 0, 0, 0},
109 | 			{0, 0, 1, 1, 1, 0}
110 | 		};
111 | 		
112 | 		double[][] test_Y = new double[test_N][n_in];
113 | 		
114 | 		
115 | 		// test
116 | 		for(int i=0; i<test_N; i++) {
117 | 			classifier.predict(test_X[i], test_Y[i]);
118 | 			for(int j=0; j<n_out; j++) {
119 | 				System.out.print(test_Y[i][j] + " ");
120 | 			}
121 | 			System.out.println();
122 | 		}
123 | 	}
124 | 	
125 | 	public static void main(String[] args) {
126 | 		test_lr();
127 | 	}
128 | }
129 | 


--------------------------------------------------------------------------------
/scala/LogisticRegression.scala:
--------------------------------------------------------------------------------
  1 | // $ scalac LogisticRegression.scala
  2 | // $ scala LogisticRegression
  3 | 
  4 | import scala.math
  5 | 
  6 | class LogisticRegression(val N: Int, val n_in: Int, val n_out: Int) {
  7 | 
  8 |   val W: Array[Array[Double]] = Array.ofDim[Double](n_out, n_in)
  9 |   val b: Array[Double] = new Array[Double](n_out)
 10 | 
 11 |   def train(x: Array[Int], y: Array[Int], lr: Double) {
 12 |     val p_y_given_x: Array[Double] = new Array[Double](n_out)
 13 |     val dy: Array[Double] = new Array[Double](n_out)
 14 | 
 15 |     var i: Int = 0
 16 |     var j: Int = 0
 17 |     for(i <- 0 until n_out) {
 18 |       p_y_given_x(i) = 0
 19 |       for(j <- 0 until n_in) {
 20 |         p_y_given_x(i) += W(i)(j) * x(j)
 21 |       }
 22 |       p_y_given_x(i) += b(i)
 23 |     }
 24 |     softmax(p_y_given_x)
 25 | 
 26 |     for(i <- 0 until n_out) {
 27 |       dy(i) = y(i) - p_y_given_x(i)
 28 | 
 29 |       for(j <- 0 until n_in) {
 30 |         W(i)(j) += lr * dy(i) * x(j) / N
 31 |       }
 32 |       b(i) += lr * dy(i) / N
 33 |     }
 34 |   }
 35 | 
 36 | 
 37 |   def softmax(x: Array[Double]) {
 38 |     var max: Double = 0.0
 39 |     var sum: Double = 0.0
 40 | 
 41 |     var i: Int = 0
 42 |     for(i <- 0 until n_out) if(max < x(i)) max = x(i)
 43 | 
 44 |     for(i <- 0 until n_out) {
 45 |       x(i) = math.exp(x(i) - max)
 46 |       sum += x(i)
 47 |     }
 48 | 
 49 |     for(i <- 0 until n_out) x(i) /= sum
 50 |   }
 51 | 
 52 | 
 53 |   def predict(x: Array[Int], y: Array[Double]) {
 54 |     var i: Int = 0
 55 |     var j: Int = 0
 56 |     for(i <- 0 until n_out) {
 57 |       y(i) = 0
 58 |       for(j <- 0 until n_in) {
 59 |         y(i) += W(i)(j) * x(j)
 60 |       }
 61 |       y(i) += b(i)
 62 |     }
 63 |     softmax(y)
 64 |   }
 65 | 
 66 | }
 67 | 
 68 | 
 69 | object LogisticRegression {
 70 |   def test_lr() {
 71 |     val learning_rate: Double = 0.1
 72 |     val n_epochs: Int = 500
 73 | 
 74 |     val train_N: Int = 6
 75 |     val test_N: Int = 2
 76 |     val n_in: Int = 6
 77 |     val n_out: Int = 2
 78 | 
 79 |     val train_X: Array[Array[Int]] = Array(
 80 |       Array(1, 1, 1, 0, 0, 0),
 81 |       Array(1, 0, 1, 0, 0, 0),
 82 |       Array(1, 1, 1, 0, 0, 0),
 83 |       Array(0, 0, 1, 1, 1, 0),
 84 |       Array(0, 0, 1, 0, 1, 0),
 85 |       Array(0, 0, 1, 1, 1, 0)
 86 |     )
 87 | 
 88 |     val train_Y: Array[Array[Int]] = Array(
 89 |       Array(1, 0),
 90 |       Array(1, 0),
 91 |       Array(1, 0),
 92 |       Array(0, 1),
 93 |       Array(0, 1),
 94 |       Array(0, 1)
 95 |     )
 96 | 
 97 |     // construct
 98 |     val classifier = new LogisticRegression(train_N, n_in, n_out)
 99 | 
100 |     // train
101 |     var epoch: Int = 0
102 |     var i: Int = 0
103 |     for(epoch <- 0 until n_epochs) {
104 |       for(i <- 0 until train_N) {
105 |         classifier.train(train_X(i), train_Y(i), learning_rate)
106 |       }
107 |       // learning_rate *= 0.95
108 |     }
109 | 
110 |     // test data
111 |     val test_X: Array[Array[Int]] = Array(
112 |       Array(1, 0, 1, 0, 0, 0),
113 |       Array(0, 0, 1, 1, 1, 0)
114 |     )
115 | 
116 |     val test_Y: Array[Array[Double]] = Array.ofDim[Double](test_N, n_in)
117 | 
118 |     // test
119 |     var j: Int = 0
120 |     for(i <- 0 until test_N) {
121 |       classifier.predict(test_X(i), test_Y(i))
122 |       for(j <- 0 until n_out) {
123 |         printf("%.5f ", test_Y(i)(j))
124 |       }
125 |       println()
126 |     }
127 |   }
128 | 
129 |   def main(args: Array[String]) {
130 |     test_lr()
131 |   }
132 | 
133 | }
134 | 


--------------------------------------------------------------------------------
/java/SdA/src/dA.java:
--------------------------------------------------------------------------------
  1 | import java.util.Random;
  2 | 
  3 | public class dA {
  4 | 	public int N;
  5 | 	public int n_visible;
  6 | 	public int n_hidden;
  7 | 	public double[][] W;
  8 | 	public double[] hbias;
  9 | 	public double[] vbias;
 10 | 	public Random rng;
 11 | 	
 12 | 	
 13 | 	public double uniform(double min, double max) {
 14 | 		return rng.nextDouble() * (max - min) + min;
 15 | 	}
 16 | 	
 17 | 	public int binomial(int n, double p) {
 18 | 		if(p < 0 || p > 1) return 0;
 19 | 		
 20 | 		int c = 0;
 21 | 		double r;
 22 | 		
 23 | 		for(int i=0; i<n; i++) {
 24 | 			r = rng.nextDouble();
 25 | 			if (r < p) c++;
 26 | 		}
 27 | 		
 28 | 		return c;
 29 | 	}
 30 | 	
 31 | 	public static double sigmoid(double x) {
 32 | 		return 1.0 / (1.0 + Math.pow(Math.E, -x));
 33 | 	}
 34 | 
 35 | 	public dA(int N, int n_visible, int n_hidden, 
 36 | 			double[][] W, double[] hbias, double[] vbias, Random rng) {
 37 | 		this.N = N;
 38 | 		this.n_visible = n_visible;
 39 | 		this.n_hidden = n_hidden;
 40 | 
 41 | 		if(rng == null)	this.rng = new Random(1234);
 42 | 		else this.rng = rng;
 43 | 				
 44 | 		if(W == null) {
 45 | 			this.W = new double[this.n_hidden][this.n_visible];
 46 | 			double a = 1.0 / this.n_visible;
 47 | 			
 48 | 			for(int i=0; i<this.n_hidden; i++) {
 49 | 				for(int j=0; j<this.n_visible; j++) {
 50 | 					this.W[i][j] = uniform(-a, a); 
 51 | 				}
 52 | 			}	
 53 | 		} else {
 54 | 			this.W = W;
 55 | 		}
 56 | 		
 57 | 		if(hbias == null) {
 58 | 			this.hbias = new double[this.n_hidden];
 59 | 			for(int i=0; i<this.n_hidden; i++) this.hbias[i] = 0;
 60 | 		} else {
 61 | 			this.hbias = hbias;
 62 | 		}
 63 | 		
 64 | 		if(vbias == null) {
 65 | 			this.vbias = new double[this.n_visible];
 66 | 			for(int i=0; i<this.n_visible; i++) this.vbias[i] = 0;
 67 | 		} else {
 68 | 			this.vbias = vbias;
 69 | 		}	
 70 | 	}
 71 | 	
 72 | 	public void get_corrupted_input(int[] x, int[] tilde_x, double p) {
 73 | 		for(int i=0; i<n_visible; i++) {
 74 | 			if(x[i] == 0) {
 75 | 				tilde_x[i] = 0;
 76 | 			} else {
 77 | 				tilde_x[i] = binomial(1, p);
 78 | 			}
 79 | 		}
 80 | 	}
 81 | 	
 82 | 	// Encode
 83 | 	public void get_hidden_values(int[] x, double[] y) {
 84 | 		for(int i=0; i<n_hidden; i++) {
 85 | 			y[i] = 0;
 86 | 			for(int j=0; j<n_visible; j++) {
 87 | 				y[i] += W[i][j] * x[j];
 88 | 			}
 89 | 			y[i] += hbias[i];
 90 | 			y[i] = sigmoid(y[i]);
 91 | 		}
 92 | 	}
 93 | 	
 94 | 	// Decode
 95 | 	public void get_reconstructed_input(double[] y, double[] z) {
 96 | 		for(int i=0; i<n_visible; i++) {
 97 | 			z[i] = 0;
 98 | 			for(int j=0; j<n_hidden; j++) {
 99 | 				z[i] += W[j][i] * y[j];
100 | 			}
101 | 			z[i] += vbias[i];
102 | 			z[i] = sigmoid(z[i]);
103 | 		}
104 | 	}
105 | 	
106 | 	public void train(int[] x, double lr, double corruption_level) {
107 | 		int[] tilde_x = new int[n_visible];
108 | 		double[] y = new double[n_hidden];
109 | 		double[] z = new double[n_visible];
110 | 		
111 | 		double[] L_vbias = new double[n_visible];
112 | 		double[] L_hbias = new double[n_hidden];
113 | 		
114 | 		double p = 1 - corruption_level;
115 | 		
116 | 		get_corrupted_input(x, tilde_x, p);
117 | 		get_hidden_values(tilde_x, y);
118 | 		get_reconstructed_input(y, z);
119 | 		
120 | 		// vbias
121 | 		for(int i=0; i<n_visible; i++) {
122 | 			L_vbias[i] = x[i] - z[i];
123 | 			vbias[i] += lr * L_vbias[i] / N;
124 | 		}
125 | 		
126 | 		// hbias
127 | 		for(int i=0; i<n_hidden; i++) {
128 | 			L_hbias[i] = 0;
129 | 			for(int j=0; j<n_visible; j++) {
130 | 				L_hbias[i] += W[i][j] * L_vbias[j];
131 | 			}
132 | 			L_hbias[i] *= y[i] * (1 - y[i]);
133 | 			hbias[i] += lr * L_hbias[i] / N;
134 | 		}
135 | 		
136 | 		// W
137 | 		for(int i=0; i<n_hidden; i++) {
138 | 			for(int j=0; j<n_visible; j++) {
139 | 				W[i][j] += lr * (L_hbias[i] * tilde_x[j] + L_vbias[j] * y[i]) / N;
140 | 			}
141 | 		}
142 | 	}
143 | 	
144 | 	public void reconstruct(int[] x, double[] z) {
145 | 		double[] y = new double[n_hidden];
146 | 		
147 | 		get_hidden_values(x, y);
148 | 		get_reconstructed_input(y, z);
149 | 	}	
150 | }
151 | 


--------------------------------------------------------------------------------
/cpp/LogisticRegression.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <string>
  3 | #include <math.h>
  4 | #include "LogisticRegression.h"
  5 | using namespace std;
  6 | 
  7 | 
  8 | LogisticRegression::LogisticRegression(int size, int in, int out) {
  9 |   N = size;
 10 |   n_in = in;
 11 |   n_out = out;
 12 | 
 13 |   // initialize W, b
 14 |   W = new double*[n_out];
 15 |   for(int i=0; i<n_out; i++) W[i] = new double[n_in];
 16 |   b = new double[n_out];
 17 | 
 18 |   for(int i=0; i<n_out; i++) {
 19 |     for(int j=0; j<n_in; j++) {
 20 |       W[i][j] = 0;
 21 |     }
 22 |     b[i] = 0;
 23 |   }
 24 | }
 25 | 
 26 | LogisticRegression::~LogisticRegression() {
 27 |   for(int i=0; i<n_out; i++) delete[] W[i];
 28 |   delete[] W;
 29 |   delete[] b;
 30 | }
 31 | 
 32 | 
 33 | void LogisticRegression::train(int *x, int *y, double lr) {
 34 |   double *p_y_given_x = new double[n_out];
 35 |   double *dy = new double[n_out];
 36 | 
 37 |   for(int i=0; i<n_out; i++) {
 38 |     p_y_given_x[i] = 0;
 39 |     for(int j=0; j<n_in; j++) {
 40 |       p_y_given_x[i] += W[i][j] * x[j];
 41 |     }
 42 |     p_y_given_x[i] += b[i];
 43 |   }
 44 |   softmax(p_y_given_x);
 45 | 
 46 |   for(int i=0; i<n_out; i++) {
 47 |     dy[i] = y[i] - p_y_given_x[i];
 48 | 
 49 |     for(int j=0; j<n_in; j++) {
 50 |       W[i][j] += lr * dy[i] * x[j] / N;
 51 |     }
 52 | 
 53 |     b[i] += lr * dy[i] / N;
 54 |   }
 55 |   delete[] p_y_given_x;
 56 |   delete[] dy;
 57 | }
 58 | 
 59 | void LogisticRegression::softmax(double *x) {
 60 |   double max = 0.0;
 61 |   double sum = 0.0;
 62 |   
 63 |   for(int i=0; i<n_out; i++) if(max < x[i]) max = x[i];
 64 |   for(int i=0; i<n_out; i++) {
 65 |     x[i] = exp(x[i] - max);
 66 |     sum += x[i];
 67 |   } 
 68 | 
 69 |   for(int i=0; i<n_out; i++) x[i] /= sum;
 70 | }
 71 | 
 72 | void LogisticRegression::predict(int *x, double *y) {
 73 |   for(int i=0; i<n_out; i++) {
 74 |     y[i] = 0;
 75 |     for(int j=0; j<n_in; j++) {
 76 |       y[i] += W[i][j] * x[j];
 77 |     }
 78 |     y[i] += b[i];
 79 |   }
 80 | 
 81 |   softmax(y);
 82 | }
 83 | 
 84 | 
 85 | void test_lr() {
 86 |   srand(0);
 87 |   
 88 |   double learning_rate = 0.1;
 89 |   double n_epochs = 500;
 90 | 
 91 |   int train_N = 6;
 92 |   int test_N = 2;
 93 |   int n_in = 6;
 94 |   int n_out = 2;
 95 |   // int **train_X;
 96 |   // int **train_Y;
 97 |   // int **test_X;
 98 |   // double **test_Y;
 99 | 
100 |   // train_X = new int*[train_N];
101 |   // train_Y = new int*[train_N];
102 |   // for(i=0; i<train_N; i++){
103 |   //   train_X[i] = new int[n_in];
104 |   //   train_Y[i] = new int[n_out];
105 |   // };
106 | 
107 |   // test_X = new int*[test_N];
108 |   // test_Y = new double*[test_N];
109 |   // for(i=0; i<test_N; i++){
110 |   //   test_X[i] = new int[n_in];
111 |   //   test_Y[i] = new double[n_out];
112 |   // }
113 | 
114 | 
115 |   // training data
116 |   int train_X[6][6] = {
117 |     {1, 1, 1, 0, 0, 0},
118 |     {1, 0, 1, 0, 0, 0},
119 |     {1, 1, 1, 0, 0, 0},
120 |     {0, 0, 1, 1, 1, 0},
121 |     {0, 0, 1, 1, 0, 0},
122 |     {0, 0, 1, 1, 1, 0}
123 |   };
124 | 
125 |   int train_Y[6][2] = {
126 |     {1, 0},
127 |     {1, 0},
128 |     {1, 0},
129 |     {0, 1},
130 |     {0, 1},
131 |     {0, 1}
132 |   };
133 | 
134 | 
135 |   // construct LogisticRegression
136 |   LogisticRegression classifier(train_N, n_in, n_out);
137 | 
138 | 
139 |   // train online
140 |   for(int epoch=0; epoch<n_epochs; epoch++) {
141 |     for(int i=0; i<train_N; i++) {
142 |       classifier.train(train_X[i], train_Y[i], learning_rate);
143 |     }
144 |     // learning_rate *= 0.95;
145 |   }
146 | 
147 | 
148 |   // test data
149 |   int test_X[2][6] = {
150 |     {1, 0, 1, 0, 0, 0},
151 |     {0, 0, 1, 1, 1, 0}
152 |   };
153 | 
154 |   double test_Y[2][2];
155 | 
156 | 
157 |   // test
158 |   for(int i=0; i<test_N; i++) {
159 |     classifier.predict(test_X[i], test_Y[i]);
160 |     for(int j=0; j<n_out; j++) {
161 |       cout << test_Y[i][j] << " ";
162 |     }
163 |     cout << endl;
164 |   }
165 | 
166 | }
167 | 
168 | 
169 | int main() {
170 |   test_lr();
171 |   return 0;
172 | }
173 | 


--------------------------------------------------------------------------------
/c/LogisticRegression.c:
--------------------------------------------------------------------------------
  1 | #include <stdio.h>
  2 | #include <stdlib.h>
  3 | #include <math.h>
  4 | #include "LogisticRegression.h"
  5 | 
  6 | void test_lr(void);
  7 | 
  8 | 
  9 | void LogisticRegression__construct(LogisticRegression *this, int N, int n_in, int n_out) {
 10 |   int i, j;
 11 |   this->N = N;
 12 |   this->n_in = n_in;
 13 |   this->n_out = n_out;
 14 | 
 15 |   this->W = (double **)malloc(sizeof(double*) * n_out);
 16 |   this->W[0] = (double *)malloc(sizeof(double) * n_in * n_out);
 17 |   for(i=0; i<n_out; i++) this->W[i] = this->W[0] + i * n_in;
 18 |   this->b = (double *)malloc(sizeof(double) * n_out);
 19 | 
 20 |   for(i=0; i<n_out; i++) {
 21 |     for(j=0; j<n_in; j++) {
 22 |       this->W[i][j] = 0;
 23 |     }
 24 |     this->b[i] = 0;
 25 |   }
 26 | }
 27 | 
 28 | void LogisticRegression__destruct(LogisticRegression *this) {
 29 |   free(this->W[0]);
 30 |   free(this->W);
 31 |   free(this->b);
 32 | }
 33 | 
 34 | void LogisticRegression_train(LogisticRegression *this, int *x, int *y, double lr) {
 35 |   int i,j;
 36 |   double *p_y_given_x = (double *)malloc(sizeof(double) * this->n_out);
 37 |   double *dy = (double *)malloc(sizeof(double) * this->n_out);
 38 | 
 39 |   for(i=0; i<this->n_out; i++) {
 40 |     p_y_given_x[i] = 0;
 41 |     for(j=0; j<this->n_in; j++) {
 42 |       p_y_given_x[i] += this->W[i][j] * x[j];
 43 |     }
 44 |     p_y_given_x[i] += this->b[i];
 45 |   }
 46 |   LogisticRegression_softmax(this, p_y_given_x);
 47 | 
 48 |   for(i=0; i<this->n_out; i++) {
 49 |     dy[i] = y[i] - p_y_given_x[i];
 50 | 
 51 |     for(j=0; j<this->n_in; j++) {
 52 |       this->W[i][j] += lr * dy[i] * x[j] / this->N;
 53 |     }
 54 | 
 55 |     this->b[i] += lr * dy[i] / this->N;
 56 |   }
 57 | 
 58 |   free(p_y_given_x);
 59 |   free(dy);
 60 | }
 61 | 
 62 | void LogisticRegression_softmax(LogisticRegression *this, double *x) {
 63 |   int i;
 64 |   double max = 0.0;
 65 |   double sum = 0.0;
 66 | 
 67 |   for(i=0; i<this->n_out; i++) if(max < x[i]) max = x[i];
 68 |   for(i=0; i<this->n_out; i++) {
 69 |     x[i] = exp(x[i] - max);
 70 |     sum += x[i];
 71 |   }
 72 | 
 73 |   for(i=0; i<this->n_out; i++) x[i] /= sum;
 74 | }
 75 | 
 76 | void LogisticRegression_predict(LogisticRegression *this, int *x, double *y) {
 77 |   int i,j;
 78 | 
 79 |   for(i=0; i<this->n_out; i++) {
 80 |     y[i] = 0;
 81 |     for(j=0; j<this->n_in; j++) {
 82 |       y[i] += this->W[i][j] * x[j];
 83 |     }
 84 |     y[i] += this->b[i];
 85 |   }
 86 | 
 87 |   LogisticRegression_softmax(this, y);
 88 | }
 89 | 
 90 | 
 91 | 
 92 | 
 93 | void test_lr(void) {
 94 |   int i, j, epoch;
 95 | 
 96 |   double learning_rate = 0.1;
 97 |   double n_epochs = 500;
 98 | 
 99 |   int train_N = 6;
100 |   int test_N = 2;
101 |   int n_in = 6;
102 |   int n_out = 2;
103 | 
104 | 
105 |   // training data
106 |   int train_X[6][6] = {
107 |     {1, 1, 1, 0, 0, 0},
108 |     {1, 0, 1, 0, 0, 0},
109 |     {1, 1, 1, 0, 0, 0},
110 |     {0, 0, 1, 1, 1, 0},
111 |     {0, 0, 1, 1, 0, 0},
112 |     {0, 0, 1, 1, 1, 0}
113 |   };
114 | 
115 |   int train_Y[6][2] = {
116 |     {1, 0},
117 |     {1, 0},
118 |     {1, 0},
119 |     {0, 1},
120 |     {0, 1},
121 |     {0, 1}
122 |   };
123 | 
124 | 
125 |   // construct LogisticRegression
126 |   LogisticRegression classifier;
127 |   LogisticRegression__construct(&classifier, train_N, n_in, n_out);
128 | 
129 | 
130 |   // train
131 |   for(epoch=0; epoch<n_epochs; epoch++) {
132 |     for(i=0; i<train_N; i++) {
133 |       LogisticRegression_train(&classifier, train_X[i], train_Y[i], learning_rate);
134 |     }
135 |     // learning_rate *= 0.95;
136 |   }
137 | 
138 | 
139 |   // test data
140 |   int test_X[2][6] = {
141 |     {1, 0, 1, 0, 0, 0},
142 |     {0, 0, 1, 1, 1, 0}
143 |   };
144 | 
145 |   double test_Y[2][2];
146 | 
147 | 
148 |   // test
149 |   for(i=0; i<test_N; i++) {
150 |     LogisticRegression_predict(&classifier, test_X[i], test_Y[i]);
151 |     for(j=0; j<n_out; j++) {
152 |       printf("%f ", test_Y[i][j]);
153 |     }
154 |     printf("\n");
155 |   }
156 | 
157 | 
158 | 
159 |   // destruct LogisticRegression
160 |   LogisticRegression__destruct(&classifier);
161 | }
162 | 
163 | 
164 | 
165 | 
166 | int main(void) {
167 |   test_lr();
168 |   
169 |   return 0;
170 | }
171 | 


--------------------------------------------------------------------------------
/java/DBN/src/RBM.java:
--------------------------------------------------------------------------------
  1 | import java.util.Random;
  2 | 
  3 | public class RBM {
  4 | 	public int N;
  5 | 	public int n_visible;
  6 | 	public int n_hidden;
  7 | 	public double[][] W;
  8 | 	public double[] hbias;
  9 | 	public double[] vbias;
 10 | 	public Random rng;
 11 | 	
 12 | 	public double uniform(double min, double max) {
 13 | 		return rng.nextDouble() * (max - min) + min;
 14 | 	}
 15 | 	
 16 | 	public int binomial(int n, double p) {
 17 | 		if(p < 0 || p > 1) return 0;
 18 | 		
 19 | 		int c = 0;
 20 | 		double r;
 21 | 		
 22 | 		for(int i=0; i<n; i++) {
 23 | 			r = rng.nextDouble();
 24 | 			if (r < p) c++;
 25 | 		}
 26 | 		
 27 | 		return c;
 28 | 	}
 29 | 	
 30 | 	public static double sigmoid(double x) {
 31 | 		return 1.0 / (1.0 + Math.pow(Math.E, -x));
 32 | 	}
 33 | 	
 34 | 	
 35 | 	public RBM(int N, int n_visible, int n_hidden, 
 36 | 			double[][] W, double[] hbias, double[] vbias, Random rng) {
 37 | 		this.N = N;
 38 | 		this.n_visible = n_visible;
 39 | 		this.n_hidden = n_hidden;
 40 | 	
 41 | 		if(rng == null)	this.rng = new Random(1234);
 42 | 		else this.rng = rng;
 43 | 		
 44 | 		if(W == null) {
 45 | 			this.W = new double[this.n_hidden][this.n_visible];
 46 | 			double a = 1.0 / this.n_visible;
 47 | 			
 48 | 			for(int i=0; i<this.n_hidden; i++) {
 49 | 				for(int j=0; j<this.n_visible; j++) {
 50 | 					this.W[i][j] = uniform(-a, a); 
 51 | 				}
 52 | 			}	
 53 | 		} else {
 54 | 			this.W = W;
 55 | 		}
 56 | 		
 57 | 		if(hbias == null) {
 58 | 			this.hbias = new double[this.n_hidden];
 59 | 			for(int i=0; i<this.n_hidden; i++) this.hbias[i] = 0;
 60 | 		} else {
 61 | 			this.hbias = hbias;
 62 | 		}
 63 | 		
 64 | 		if(vbias == null) {
 65 | 			this.vbias = new double[this.n_visible];
 66 | 			for(int i=0; i<this.n_visible; i++) this.vbias[i] = 0;
 67 | 		} else {
 68 | 			this.vbias = vbias;
 69 | 		}
 70 | 	}
 71 | 	
 72 | 	
 73 | 	public void contrastive_divergence(int[] input, double lr, int k) {
 74 | 		double[] ph_mean = new double[n_hidden];
 75 | 		int[] ph_sample = new int[n_hidden];
 76 | 		double[] nv_means = new double[n_visible];
 77 | 		int[] nv_samples = new int[n_visible];
 78 | 		double[] nh_means = new double[n_hidden];
 79 | 		int[] nh_samples = new int[n_hidden];
 80 | 		
 81 | 		/* CD-k */
 82 | 		sample_h_given_v(input, ph_mean, ph_sample);
 83 | 		
 84 | 		for(int step=0; step<k; step++) {
 85 | 			if(step == 0) {
 86 | 				gibbs_hvh(ph_sample, nv_means, nv_samples, nh_means, nh_samples);
 87 | 			} else {
 88 | 				gibbs_hvh(nh_samples, nv_means, nv_samples, nh_means, nh_samples);
 89 | 			}
 90 | 		}
 91 | 		
 92 | 		for(int i=0; i<n_hidden; i++) {
 93 | 			for(int j=0; j<n_visible; j++) {
 94 | 				// W[i][j] += lr *(ph_sample[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
 95 | 				W[i][j] += lr *(ph_mean[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
 96 | 			}
 97 | 			hbias[i] += lr * (ph_sample[i] - nh_means[i]) / N;
 98 | 		}
 99 | 		
100 | 
101 | 		for(int i=0; i<n_visible; i++) {
102 | 			vbias[i] += lr * (input[i] - nv_samples[i]) / N;
103 | 		}
104 | 
105 | 	}
106 | 	
107 | 	
108 | 	public void sample_h_given_v(int[] v0_sample, double[] mean, int[] sample) {
109 | 		for(int i=0; i<n_hidden; i++) {
110 | 			mean[i] = propup(v0_sample, W[i], hbias[i]);
111 | 			sample[i] = binomial(1, mean[i]);
112 | 		}
113 | 	}
114 | 
115 | 	public void sample_v_given_h(int[] h0_sample, double[] mean, int[] sample) {
116 | 		for(int i=0; i<n_visible; i++) {
117 | 			mean[i] = propdown(h0_sample, i, vbias[i]);
118 | 			sample[i] = binomial(1, mean[i]);
119 | 		}
120 | 	}
121 | 	
122 | 	public double propup(int[] v, double[] w, double b) {
123 | 		double pre_sigmoid_activation = 0.0;
124 | 		for(int j=0; j<n_visible; j++) {
125 | 			pre_sigmoid_activation += w[j] * v[j];
126 | 		}
127 | 		pre_sigmoid_activation += b;
128 | 		return sigmoid(pre_sigmoid_activation);
129 | 	}
130 | 	
131 | 	public double propdown(int[] h, int i, double b) {
132 | 	  double pre_sigmoid_activation = 0.0;
133 | 	  for(int j=0; j<n_hidden; j++) {
134 | 	    pre_sigmoid_activation += W[j][i] * h[j];
135 | 	  }
136 | 	  pre_sigmoid_activation += b;
137 | 	  return sigmoid(pre_sigmoid_activation);
138 | 	}
139 | 	
140 | 	public void gibbs_hvh(int[] h0_sample, double[] nv_means, int[] nv_samples, double[] nh_means, int[] nh_samples) {
141 | 	  sample_v_given_h(h0_sample, nv_means, nv_samples);
142 | 	  sample_h_given_v(nv_samples, nh_means, nh_samples);
143 | 	}
144 | 
145 | 
146 | 	public void reconstruct(int[] v, double[] reconstructed_v) {
147 | 	  double[] h = new double[n_hidden];
148 | 	  double pre_sigmoid_activation;
149 | 	
150 | 	  for(int i=0; i<n_hidden; i++) {
151 | 	    h[i] = propup(v, W[i], hbias[i]);
152 | 	  }
153 | 	
154 | 	  for(int i=0; i<n_visible; i++) {
155 | 	    pre_sigmoid_activation = 0.0;
156 | 	    for(int j=0; j<n_hidden; j++) {
157 | 	      pre_sigmoid_activation += W[j][i] * h[j];
158 | 	    }
159 | 	    pre_sigmoid_activation += vbias[i];
160 | 	
161 | 	    reconstructed_v[i] = sigmoid(pre_sigmoid_activation);
162 | 	  }	
163 | 	}	
164 | }
165 | 


--------------------------------------------------------------------------------
/python/CDBN.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | '''
  5 |  DBN  w/ continuous-valued inputs (Linear Energy)
  6 | 
  7 |  References :
  8 |    - Y. Bengio, P. Lamblin, D. Popovici, H. Larochelle: Greedy Layer-Wise
  9 |    Training of Deep Networks, Advances in Neural Information Processing
 10 |    Systems 19, 2007
 11 | 
 12 | '''
 13 | 
 14 | import sys
 15 | import numpy
 16 | from HiddenLayer import HiddenLayer
 17 | from LogisticRegression import LogisticRegression
 18 | from RBM import RBM
 19 | from CRBM import CRBM
 20 | from DBN import DBN
 21 | from utils import *
 22 | 
 23 | 
 24 | 
 25 |  
 26 | class CDBN(DBN):
 27 |     def __init__(self, input=None, label=None,\
 28 |                  n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2,\
 29 |                  numpy_rng=None):
 30 |         
 31 |         self.x = input
 32 |         self.y = label
 33 | 
 34 |         self.sigmoid_layers = []
 35 |         self.rbm_layers = []
 36 |         self.n_layers = len(hidden_layer_sizes)  # = len(self.rbm_layers)
 37 | 
 38 |         if numpy_rng is None:
 39 |             numpy_rng = numpy.random.RandomState(1234)
 40 | 
 41 |         
 42 |         assert self.n_layers > 0
 43 | 
 44 | 
 45 |         # construct multi-layer
 46 |         for i in xrange(self.n_layers):
 47 |             # layer_size
 48 |             if i == 0:
 49 |                 input_size = n_ins
 50 |             else:
 51 |                 input_size = hidden_layer_sizes[i - 1]
 52 | 
 53 |             # layer_input
 54 |             if i == 0:
 55 |                 layer_input = self.x
 56 |             else:
 57 |                 layer_input = self.sigmoid_layers[-1].sample_h_given_v()
 58 |                 
 59 |             # construct sigmoid_layer
 60 |             sigmoid_layer = HiddenLayer(input=layer_input,
 61 |                                         n_in=input_size,
 62 |                                         n_out=hidden_layer_sizes[i],
 63 |                                         numpy_rng=numpy_rng,
 64 |                                         activation=sigmoid)
 65 |             self.sigmoid_layers.append(sigmoid_layer)
 66 | 
 67 |             # construct rbm_layer
 68 |             if i == 0:
 69 |                 rbm_layer = CRBM(input=layer_input,     # continuous-valued inputs
 70 |                                  n_visible=input_size,
 71 |                                  n_hidden=hidden_layer_sizes[i],
 72 |                                  W=sigmoid_layer.W,     # W, b are shared
 73 |                                  hbias=sigmoid_layer.b)
 74 |             else:
 75 |                 rbm_layer = RBM(input=layer_input,
 76 |                                 n_visible=input_size,
 77 |                                 n_hidden=hidden_layer_sizes[i],
 78 |                                 W=sigmoid_layer.W,     # W, b are shared
 79 |                                 hbias=sigmoid_layer.b)
 80 |                 
 81 |             self.rbm_layers.append(rbm_layer)
 82 | 
 83 | 
 84 |         # layer for output using Logistic Regression
 85 |         self.log_layer = LogisticRegression(input=self.sigmoid_layers[-1].sample_h_given_v(),
 86 |                                             label=self.y,
 87 |                                             n_in=hidden_layer_sizes[-1],
 88 |                                             n_out=n_outs)
 89 | 
 90 |         # finetune cost: the negative log likelihood of the logistic regression layer
 91 |         self.finetune_cost = self.log_layer.negative_log_likelihood()
 92 | 
 93 | 
 94 | 
 95 | def test_cdbn(pretrain_lr=0.1, pretraining_epochs=1000, k=1, \
 96 |              finetune_lr=0.1, finetune_epochs=200):
 97 | 
 98 |     x = numpy.array([[0.4, 0.5, 0.5, 0.,  0.,  0.],
 99 |                      [0.5, 0.3,  0.5, 0.,  0.,  0.],
100 |                      [0.4, 0.5, 0.5, 0.,  0.,  0.],
101 |                      [0.,  0.,  0.5, 0.3, 0.5, 0.],
102 |                      [0.,  0.,  0.5, 0.4, 0.5, 0.],
103 |                      [0.,  0.,  0.5, 0.5, 0.5, 0.]])
104 |     
105 |     y = numpy.array([[1, 0],
106 |                      [1, 0],
107 |                      [1, 0],
108 |                      [0, 1],
109 |                      [0, 1],
110 |                      [0, 1]])
111 | 
112 |     
113 |     rng = numpy.random.RandomState(123)
114 | 
115 |     # construct DBN
116 |     dbn = CDBN(input=x, label=y, n_ins=6, hidden_layer_sizes=[5, 5], n_outs=2, numpy_rng=rng)
117 | 
118 |     # pre-training (TrainUnsupervisedDBN)
119 |     dbn.pretrain(lr=pretrain_lr, k=1, epochs=pretraining_epochs)
120 |     
121 |     # fine-tuning (DBNSupervisedFineTuning)
122 |     dbn.finetune(lr=finetune_lr, epochs=finetune_epochs)
123 | 
124 | 
125 |     # test
126 |     x = numpy.array([[0.5, 0.5, 0., 0., 0., 0.],
127 |                      [0., 0., 0., 0.5, 0.5, 0.],
128 |                      [0.5, 0.5, 0.5, 0.5, 0.5, 0.]])
129 | 
130 |     
131 |     print dbn.predict(x)
132 | 
133 | 
134 | 
135 | 
136 | if __name__ == "__main__":
137 |     test_cdbn()
138 | 


--------------------------------------------------------------------------------
/python/dA.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | """
  5 |  Denoising Autoencoders (dA)
  6 | 
  7 |  References :
  8 |    - P. Vincent, H. Larochelle, Y. Bengio, P.A. Manzagol: Extracting and
  9 |    Composing Robust Features with Denoising Autoencoders, ICML'08, 1096-1103,
 10 |    2008
 11 | 
 12 |    - DeepLearningTutorials
 13 |    https://github.com/lisa-lab/DeepLearningTutorials
 14 | 
 15 |    - Yusuke Sugomori: Stochastic Gradient Descent for Denoising Autoencoders,
 16 |    http://yusugomori.com/docs/SGD_DA.pdf
 17 | 
 18 | """
 19 | 
 20 | 
 21 | import sys
 22 | import numpy
 23 | from utils import *
 24 | 
 25 | 
 26 | class dA(object):
 27 |     def __init__(self, input=None, n_visible=2, n_hidden=3, \
 28 |         W=None, hbias=None, vbias=None, numpy_rng=None):
 29 | 
 30 |         self.n_visible = n_visible  # num of units in visible (input) layer
 31 |         self.n_hidden = n_hidden    # num of units in hidden layer
 32 | 
 33 |         if numpy_rng is None:
 34 |             numpy_rng = numpy.random.RandomState(1234)
 35 |             
 36 |         if W is None:
 37 |             a = 1. / n_visible
 38 |             initial_W = numpy.array(numpy_rng.uniform(  # initialize W uniformly
 39 |                 low=-a,
 40 |                 high=a,
 41 |                 size=(n_visible, n_hidden)))
 42 | 
 43 |             W = initial_W
 44 | 
 45 |         if hbias is None:
 46 |             hbias = numpy.zeros(n_hidden)  # initialize h bias 0
 47 | 
 48 |         if vbias is None:
 49 |             vbias = numpy.zeros(n_visible)  # initialize v bias 0
 50 | 
 51 |         self.numpy_rng = numpy_rng
 52 |         self.x = input
 53 |         self.W = W
 54 |         self.W_prime = self.W.T
 55 |         self.hbias = hbias
 56 |         self.vbias = vbias
 57 | 
 58 |         # self.params = [self.W, self.hbias, self.vbias]
 59 | 
 60 | 
 61 |         
 62 |     def get_corrupted_input(self, input, corruption_level):
 63 |         assert corruption_level < 1
 64 | 
 65 |         return self.numpy_rng.binomial(size=input.shape,
 66 |                                        n=1,
 67 |                                        p=1-corruption_level) * input
 68 | 
 69 |     # Encode
 70 |     def get_hidden_values(self, input):
 71 |         return sigmoid(numpy.dot(input, self.W) + self.hbias)
 72 | 
 73 |     # Decode
 74 |     def get_reconstructed_input(self, hidden):
 75 |         return sigmoid(numpy.dot(hidden, self.W_prime) + self.vbias)
 76 | 
 77 | 
 78 |     def train(self, lr=0.1, corruption_level=0.3, input=None):
 79 |         if input is not None:
 80 |             self.x = input
 81 | 
 82 |         x = self.x
 83 |         tilde_x = self.get_corrupted_input(x, corruption_level)
 84 |         y = self.get_hidden_values(tilde_x)
 85 |         z = self.get_reconstructed_input(y)
 86 | 
 87 |         L_h2 = x - z
 88 |         L_h1 = numpy.dot(L_h2, self.W) * y * (1 - y)
 89 | 
 90 |         L_vbias = L_h2
 91 |         L_hbias = L_h1
 92 |         L_W =  numpy.dot(tilde_x.T, L_h1) + numpy.dot(L_h2.T, y)
 93 | 
 94 | 
 95 |         self.W += lr * L_W
 96 |         self.hbias += lr * numpy.mean(L_hbias, axis=0)
 97 |         self.vbias += lr * numpy.mean(L_vbias, axis=0)
 98 | 
 99 | 
100 | 
101 |     def negative_log_likelihood(self, corruption_level=0.3):
102 |         tilde_x = self.get_corrupted_input(self.x, corruption_level)
103 |         y = self.get_hidden_values(tilde_x)
104 |         z = self.get_reconstructed_input(y)
105 | 
106 |         cross_entropy = - numpy.mean(
107 |             numpy.sum(self.x * numpy.log(z) +
108 |             (1 - self.x) * numpy.log(1 - z),
109 |                       axis=1))
110 | 
111 |         return cross_entropy
112 | 
113 | 
114 |     def reconstruct(self, x):
115 |         y = self.get_hidden_values(x)
116 |         z = self.get_reconstructed_input(y)
117 |         return z
118 | 
119 | 
120 | 
121 | def test_dA(learning_rate=0.1, corruption_level=0.3, training_epochs=50):
122 |     data = numpy.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
123 |                         [1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
124 |                         [1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
125 |                         [1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
126 |                         [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
127 |                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
128 |                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1],
129 |                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1],
130 |                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1],
131 |                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0]])
132 |     
133 |     rng = numpy.random.RandomState(123)
134 | 
135 |     # construct dA
136 |     da = dA(input=data, n_visible=20, n_hidden=5, numpy_rng=rng)
137 | 
138 |     # train
139 |     for epoch in xrange(training_epochs):
140 |         da.train(lr=learning_rate, corruption_level=corruption_level)
141 |         # cost = da.negative_log_likelihood(corruption_level=corruption_level)
142 |         # print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
143 |         # learning_rate *= 0.95
144 | 
145 | 
146 |     # test
147 |     x = numpy.array([[1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1],
148 |                      [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0]])
149 | 
150 |     print da.reconstruct(x)
151 | 
152 | 
153 | 
154 | if __name__ == "__main__":
155 |     test_dA()
156 | 


--------------------------------------------------------------------------------
/cpp/dA.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <math.h>
  3 | #include "dA.h"
  4 | using namespace std;
  5 | 
  6 | 
  7 | double uniform(double min, double max) {
  8 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;
  9 | }
 10 | 
 11 | int binomial(int n, double p) {
 12 |   if(p < 0 || p > 1) return 0;
 13 |   
 14 |   int c = 0;
 15 |   double r;
 16 |   
 17 |   for(int i=0; i<n; i++) {
 18 |     r = rand() / (RAND_MAX + 1.0);
 19 |     if (r < p) c++;
 20 |   }
 21 | 
 22 |   return c;
 23 | }
 24 | 
 25 | double sigmoid(double x) {
 26 |   return 1.0 / (1.0 + exp(-x));
 27 | }
 28 | 
 29 | 
 30 | dA::dA(int size, int n_v, int n_h, double **w, double *hb, double *vb) {
 31 |   N = size;
 32 |   n_visible = n_v;
 33 |   n_hidden = n_h;
 34 | 
 35 |   if(w == NULL) {
 36 |     W = new double*[n_hidden];
 37 |     for(int i=0; i<n_hidden; i++) W[i] = new double[n_visible];
 38 |     double a = 1.0 / n_visible;
 39 | 
 40 |     for(int i=0; i<n_hidden; i++) {
 41 |       for(int j=0; j<n_visible; j++) {
 42 |         W[i][j] = uniform(-a, a);
 43 |       }
 44 |     }
 45 |   } else {
 46 |     W = w;
 47 |   }
 48 | 
 49 |   if(hb == NULL) {
 50 |     hbias = new double[n_hidden];
 51 |     for(int i=0; i<n_hidden; i++) hbias[i] = 0;
 52 |   } else {
 53 |     hbias = hb;
 54 |   }
 55 | 
 56 |   if(vb == NULL) {
 57 |     vbias = new double[n_visible];
 58 |     for(int i=0; i<n_visible; i++) vbias[i] = 0;
 59 |   } else {
 60 |     vbias = vb;
 61 |   }
 62 | }
 63 | 
 64 | dA::~dA() {
 65 |   for(int i=0; i<n_hidden; i++) delete[] W[i];
 66 |   delete[] W;
 67 |   delete[] hbias;
 68 |   delete[] vbias;
 69 | }
 70 | 
 71 | void dA::get_corrupted_input(int *x, int *tilde_x, double p) {
 72 |   for(int i=0; i<n_visible; i++) {
 73 |     if(x[i] == 0) {
 74 |       tilde_x[i] = 0;
 75 |     } else {
 76 |       tilde_x[i] = binomial(1, p);
 77 |     }
 78 |   }
 79 | }
 80 | 
 81 | // Encode
 82 | void dA::get_hidden_values(int *x, double *y) {
 83 |   for(int i=0; i<n_hidden; i++) {
 84 |     y[i] = 0;
 85 |     for(int j=0; j<n_visible; j++) {
 86 |       y[i] += W[i][j] * x[j];
 87 |     }
 88 |     y[i] += hbias[i];
 89 |     y[i] = sigmoid(y[i]);
 90 |   }
 91 | }
 92 | 
 93 | // Decode
 94 | void dA::get_reconstructed_input(double *y, double *z) {
 95 |   for(int i=0; i<n_visible; i++) {
 96 |     z[i] = 0;
 97 |     for(int j=0; j<n_hidden; j++) {
 98 |       z[i] += W[j][i] * y[j];
 99 |     }
100 |     z[i] += vbias[i];
101 |     z[i] = sigmoid(z[i]);
102 |   }
103 | }
104 | 
105 | void dA::train(int *x, double lr, double corruption_level) {
106 |   int *tilde_x = new int[n_visible];
107 |   double *y = new double[n_hidden];
108 |   double *z = new double[n_visible];
109 | 
110 |   double *L_vbias = new double[n_visible];
111 |   double *L_hbias = new double[n_hidden];
112 | 
113 |   double p = 1 - corruption_level;
114 | 
115 |   get_corrupted_input(x, tilde_x, p);
116 |   get_hidden_values(tilde_x, y);
117 |   get_reconstructed_input(y, z);
118 |   
119 |   // vbias
120 |   for(int i=0; i<n_visible; i++) {
121 |     L_vbias[i] = x[i] - z[i];
122 |     vbias[i] += lr * L_vbias[i] / N;
123 |   }
124 | 
125 |   // hbias
126 |   for(int i=0; i<n_hidden; i++) {
127 |     L_hbias[i] = 0;
128 |     for(int j=0; j<n_visible; j++) {
129 |       L_hbias[i] += W[i][j] * L_vbias[j];
130 |     }
131 |     L_hbias[i] *= y[i] * (1 - y[i]);
132 | 
133 |     hbias[i] += lr * L_hbias[i] / N;
134 |   }
135 |   
136 |   // W
137 |   for(int i=0; i<n_hidden; i++) {
138 |     for(int j=0; j<n_visible; j++) {
139 |       W[i][j] += lr * (L_hbias[i] * tilde_x[j] + L_vbias[j] * y[i]) / N;
140 |     }
141 |   }
142 | 
143 |   delete[] L_hbias;
144 |   delete[] L_vbias;
145 |   delete[] z;
146 |   delete[] y;
147 |   delete[] tilde_x;
148 | }
149 | 
150 | void dA::reconstruct(int *x, double *z) {
151 |   double *y = new double[n_hidden];
152 | 
153 |   get_hidden_values(x, y);
154 |   get_reconstructed_input(y, z);
155 | 
156 |   delete[] y;
157 | }
158 | 
159 | 
160 | 
161 | void test_dA() {
162 |   srand(0);
163 |   
164 |   double learning_rate = 0.1;
165 |   double corruption_level = 0.3;
166 |   int training_epochs = 100;
167 | 
168 |   int train_N = 10;
169 |   int test_N = 2;
170 |   int n_visible = 20;
171 |   int n_hidden = 5;
172 | 
173 |   // training data
174 |   int train_X[10][20] = {
175 |     {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
176 |     {1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
177 |     {1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
178 |     {1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
179 |     {0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
180 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
181 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1},
182 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1},
183 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1},
184 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0}
185 |   };
186 | 
187 |   // construct dA
188 |   dA da(train_N, n_visible, n_hidden, NULL, NULL, NULL);
189 | 
190 |   // train
191 |   for(int epoch=0; epoch<training_epochs; epoch++) {
192 |     for(int i=0; i<train_N; i++) {
193 |       da.train(train_X[i], learning_rate, corruption_level);
194 |     }
195 |   }
196 | 
197 |   // test data
198 |   int test_X[2][20] = {
199 |     {1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
200 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0}
201 |   };
202 |   double reconstructed_X[2][20];
203 | 
204 | 
205 |   // test
206 |   for(int i=0; i<test_N; i++) {
207 |     da.reconstruct(test_X[i], reconstructed_X[i]);
208 |     for(int j=0; j<n_visible; j++) {
209 |       printf("%.5f ", reconstructed_X[i][j]);
210 |     }
211 |     cout << endl;
212 |   }
213 | 
214 |   cout << endl;
215 | }
216 | 
217 | 
218 | 
219 | int main() {
220 |   test_dA();
221 |   return 0;
222 | }
223 | 


--------------------------------------------------------------------------------
/java/dA/src/dA.java:
--------------------------------------------------------------------------------
  1 | import java.util.Random;
  2 | 
  3 | public class dA {
  4 | 	public int N;
  5 | 	public int n_visible;
  6 | 	public int n_hidden;
  7 | 	public double[][] W;
  8 | 	public double[] hbias;
  9 | 	public double[] vbias;
 10 | 	public Random rng;
 11 | 	
 12 | 	
 13 | 	public double uniform(double min, double max) {
 14 | 		return rng.nextDouble() * (max - min) + min;
 15 | 	}
 16 | 	
 17 | 	public int binomial(int n, double p) {
 18 | 		if(p < 0 || p > 1) return 0;
 19 | 		
 20 | 		int c = 0;
 21 | 		double r;
 22 | 		
 23 | 		for(int i=0; i<n; i++) {
 24 | 			r = rng.nextDouble();
 25 | 			if (r < p) c++;
 26 | 		}
 27 | 		
 28 | 		return c;
 29 | 	}
 30 | 	
 31 | 	public static double sigmoid(double x) {
 32 | 		return 1.0 / (1.0 + Math.pow(Math.E, -x));
 33 | 	}
 34 | 
 35 | 	public dA(int N, int n_visible, int n_hidden, 
 36 | 			double[][] W, double[] hbias, double[] vbias, Random rng) {
 37 | 		this.N = N;
 38 | 		this.n_visible = n_visible;
 39 | 		this.n_hidden = n_hidden;
 40 | 
 41 | 		if(rng == null)	this.rng = new Random(1234);
 42 | 		else this.rng = rng;
 43 | 				
 44 | 		if(W == null) {
 45 | 			this.W = new double[this.n_hidden][this.n_visible];
 46 | 			double a = 1.0 / this.n_visible;
 47 | 			
 48 | 			for(int i=0; i<this.n_hidden; i++) {
 49 | 				for(int j=0; j<this.n_visible; j++) {
 50 | 					this.W[i][j] = uniform(-a, a); 
 51 | 				}
 52 | 			}	
 53 | 		} else {
 54 | 			this.W = W;
 55 | 		}
 56 | 		
 57 | 		if(hbias == null) {
 58 | 			this.hbias = new double[this.n_hidden];
 59 | 			for(int i=0; i<this.n_hidden; i++) this.hbias[i] = 0;
 60 | 		} else {
 61 | 			this.hbias = hbias;
 62 | 		}
 63 | 		
 64 | 		if(vbias == null) {
 65 | 			this.vbias = new double[this.n_visible];
 66 | 			for(int i=0; i<this.n_visible; i++) this.vbias[i] = 0;
 67 | 		} else {
 68 | 			this.vbias = vbias;
 69 | 		}	
 70 | 	}
 71 | 	
 72 | 	public void get_corrupted_input(int[] x, int[] tilde_x, double p) {
 73 | 		for(int i=0; i<n_visible; i++) {
 74 | 			if(x[i] == 0) {
 75 | 				tilde_x[i] = 0;
 76 | 			} else {
 77 | 				tilde_x[i] = binomial(1, p);
 78 | 			}
 79 | 		}
 80 | 	}
 81 | 	
 82 | 	// Encode
 83 | 	public void get_hidden_values(int[] x, double[] y) {
 84 | 		for(int i=0; i<n_hidden; i++) {
 85 | 			y[i] = 0;
 86 | 			for(int j=0; j<n_visible; j++) {
 87 | 				y[i] += W[i][j] * x[j];
 88 | 			}
 89 | 			y[i] += hbias[i];
 90 | 			y[i] = sigmoid(y[i]);
 91 | 		}
 92 | 	}
 93 | 	
 94 | 	// Decode
 95 | 	public void get_reconstructed_input(double[] y, double[] z) {
 96 | 		for(int i=0; i<n_visible; i++) {
 97 | 			z[i] = 0;
 98 | 			for(int j=0; j<n_hidden; j++) {
 99 | 				z[i] += W[j][i] * y[j];
100 | 			}
101 | 			z[i] += vbias[i];
102 | 			z[i] = sigmoid(z[i]);
103 | 		}
104 | 	}
105 | 	
106 | 	public void train(int[] x, double lr, double corruption_level) {
107 | 		int[] tilde_x = new int[n_visible];
108 | 		double[] y = new double[n_hidden];
109 | 		double[] z = new double[n_visible];
110 | 		
111 | 		double[] L_vbias = new double[n_visible];
112 | 		double[] L_hbias = new double[n_hidden];
113 | 		
114 | 		double p = 1 - corruption_level;
115 | 		
116 | 		get_corrupted_input(x, tilde_x, p);
117 | 		get_hidden_values(tilde_x, y);
118 | 		get_reconstructed_input(y, z);
119 | 		
120 | 		// vbias
121 | 		for(int i=0; i<n_visible; i++) {
122 | 			L_vbias[i] = x[i] - z[i];
123 | 			vbias[i] += lr * L_vbias[i] / N;
124 | 		}
125 | 		
126 | 		// hbias
127 | 		for(int i=0; i<n_hidden; i++) {
128 | 			L_hbias[i] = 0;
129 | 			for(int j=0; j<n_visible; j++) {
130 | 				L_hbias[i] += W[i][j] * L_vbias[j];
131 | 			}
132 | 			L_hbias[i] *= y[i] * (1 - y[i]);
133 | 			hbias[i] += lr * L_hbias[i] / N;
134 | 		}
135 | 		
136 | 		// W
137 | 		for(int i=0; i<n_hidden; i++) {
138 | 			for(int j=0; j<n_visible; j++) {
139 | 				W[i][j] += lr * (L_hbias[i] * tilde_x[j] + L_vbias[j] * y[i]) / N;
140 | 			}
141 | 		}
142 | 	}
143 | 	
144 | 	public void reconstruct(int[] x, double[] z) {
145 | 		double[] y = new double[n_hidden];
146 | 		
147 | 		get_hidden_values(x, y);
148 | 		get_reconstructed_input(y, z);
149 | 	}
150 | 	
151 | 	
152 | 	private static void test_dA() {
153 | 		Random rng = new Random(123);
154 | 		
155 | 		double learning_rate = 0.1;
156 | 		double corruption_level = 0.3;
157 | 		int training_epochs = 100;
158 | 		
159 | 		int train_N = 10;
160 | 		int test_N = 2;
161 | 		int n_visible = 20;
162 | 		int n_hidden = 5;
163 | 		
164 | 		int[][] train_X = {
165 | 			{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
166 | 			{1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
167 | 			{1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
168 | 			{1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
169 | 			{0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
170 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
171 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1},
172 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1},
173 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1},
174 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0}		
175 | 		};
176 | 		
177 | 		dA da = new dA(train_N, n_visible, n_hidden, null, null, null, rng);
178 | 		
179 | 		// train
180 | 		for(int epoch=0; epoch<training_epochs; epoch++) {
181 | 			for(int i=0; i<train_N; i++) {
182 | 				da.train(train_X[i], learning_rate, corruption_level);
183 | 			}
184 | 		}
185 | 		
186 | 		// test data
187 | 		int[][] test_X = {
188 | 			{1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
189 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0}				
190 | 		};
191 | 		
192 | 		double[][] reconstructed_X = new double[test_N][n_visible];
193 | 		
194 | 		// test
195 | 		for(int i=0; i<test_N; i++) {
196 | 			da.reconstruct(test_X[i], reconstructed_X[i]);
197 | 			for(int j=0; j<n_visible; j++) {
198 | 				System.out.printf("%.5f ", reconstructed_X[i][j]);
199 | 			}
200 | 			System.out.println();
201 | 		}
202 | 	}
203 | 	
204 | 	public static void main(String[] args) {
205 | 		test_dA();
206 | 	}
207 | }
208 | 


--------------------------------------------------------------------------------
/python/RBM.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | """
  5 |  Restricted Boltzmann Machine (RBM)
  6 | 
  7 |  References :
  8 |    - Y. Bengio, P. Lamblin, D. Popovici, H. Larochelle: Greedy Layer-Wise
  9 |    Training of Deep Networks, Advances in Neural Information Processing
 10 |    Systems 19, 2007
 11 | 
 12 | 
 13 |    - DeepLearningTutorials
 14 |    https://github.com/lisa-lab/DeepLearningTutorials
 15 | 
 16 | """
 17 | 
 18 | import sys
 19 | import numpy
 20 | from utils import *
 21 | 
 22 | class RBM(object):
 23 |     def __init__(self, input=None, n_visible=2, n_hidden=3, \
 24 |         W=None, hbias=None, vbias=None, numpy_rng=None):
 25 |         
 26 |         self.n_visible = n_visible  # num of units in visible (input) layer
 27 |         self.n_hidden = n_hidden    # num of units in hidden layer
 28 | 
 29 |         if numpy_rng is None:
 30 |             numpy_rng = numpy.random.RandomState(1234)
 31 | 
 32 | 
 33 |         if W is None:
 34 |             a = 1. / n_visible
 35 |             initial_W = numpy.array(numpy_rng.uniform(  # initialize W uniformly
 36 |                 low=-a,
 37 |                 high=a,
 38 |                 size=(n_visible, n_hidden)))
 39 | 
 40 |             W = initial_W
 41 | 
 42 |         if hbias is None:
 43 |             hbias = numpy.zeros(n_hidden)  # initialize h bias 0
 44 | 
 45 |         if vbias is None:
 46 |             vbias = numpy.zeros(n_visible)  # initialize v bias 0
 47 | 
 48 | 
 49 |         self.numpy_rng = numpy_rng
 50 |         self.input = input
 51 |         self.W = W
 52 |         self.hbias = hbias
 53 |         self.vbias = vbias
 54 | 
 55 |         # self.params = [self.W, self.hbias, self.vbias]
 56 | 
 57 | 
 58 |     def contrastive_divergence(self, lr=0.1, k=1, input=None):
 59 |         if input is not None:
 60 |             self.input = input
 61 |         
 62 |         ''' CD-k '''
 63 |         ph_mean, ph_sample = self.sample_h_given_v(self.input)
 64 | 
 65 |         chain_start = ph_sample
 66 | 
 67 |         for step in xrange(k):
 68 |             if step == 0:
 69 |                 nv_means, nv_samples,\
 70 |                 nh_means, nh_samples = self.gibbs_hvh(chain_start)
 71 |             else:
 72 |                 nv_means, nv_samples,\
 73 |                 nh_means, nh_samples = self.gibbs_hvh(nh_samples)
 74 | 
 75 |         # chain_end = nv_samples
 76 | 
 77 | 
 78 |         self.W += lr * (numpy.dot(self.input.T, ph_sample)
 79 |                         - numpy.dot(nv_samples.T, nh_means))
 80 |         self.vbias += lr * numpy.mean(self.input - nv_samples, axis=0)
 81 |         self.hbias += lr * numpy.mean(ph_sample - nh_means, axis=0)
 82 | 
 83 |         # cost = self.get_reconstruction_cross_entropy()
 84 |         # return cost
 85 | 
 86 | 
 87 |     def sample_h_given_v(self, v0_sample):
 88 |         h1_mean = self.propup(v0_sample)
 89 |         h1_sample = self.numpy_rng.binomial(size=h1_mean.shape,   # discrete: binomial
 90 |                                        n=1,
 91 |                                        p=h1_mean)
 92 | 
 93 |         return [h1_mean, h1_sample]
 94 | 
 95 | 
 96 |     def sample_v_given_h(self, h0_sample):
 97 |         v1_mean = self.propdown(h0_sample)
 98 |         v1_sample = self.numpy_rng.binomial(size=v1_mean.shape,   # discrete: binomial
 99 |                                             n=1,
100 |                                             p=v1_mean)
101 |         
102 |         return [v1_mean, v1_sample]
103 | 
104 |     def propup(self, v):
105 |         pre_sigmoid_activation = numpy.dot(v, self.W) + self.hbias
106 |         return sigmoid(pre_sigmoid_activation)
107 | 
108 |     def propdown(self, h):
109 |         pre_sigmoid_activation = numpy.dot(h, self.W.T) + self.vbias
110 |         return sigmoid(pre_sigmoid_activation)
111 | 
112 | 
113 |     def gibbs_hvh(self, h0_sample):
114 |         v1_mean, v1_sample = self.sample_v_given_h(h0_sample)
115 |         h1_mean, h1_sample = self.sample_h_given_v(v1_sample)
116 | 
117 |         return [v1_mean, v1_sample,
118 |                 h1_mean, h1_sample]
119 |     
120 | 
121 |     def get_reconstruction_cross_entropy(self):
122 |         pre_sigmoid_activation_h = numpy.dot(self.input, self.W) + self.hbias
123 |         sigmoid_activation_h = sigmoid(pre_sigmoid_activation_h)
124 |         
125 |         pre_sigmoid_activation_v = numpy.dot(sigmoid_activation_h, self.W.T) + self.vbias
126 |         sigmoid_activation_v = sigmoid(pre_sigmoid_activation_v)
127 | 
128 |         cross_entropy =  - numpy.mean(
129 |             numpy.sum(self.input * numpy.log(sigmoid_activation_v) +
130 |             (1 - self.input) * numpy.log(1 - sigmoid_activation_v),
131 |                       axis=1))
132 |         
133 |         return cross_entropy
134 | 
135 |     def reconstruct(self, v):
136 |         h = sigmoid(numpy.dot(v, self.W) + self.hbias)
137 |         reconstructed_v = sigmoid(numpy.dot(h, self.W.T) + self.vbias)
138 |         return reconstructed_v
139 | 
140 | 
141 | 
142 | 
143 | 
144 | def test_rbm(learning_rate=0.1, k=1, training_epochs=1000):
145 |     data = numpy.array([[1,1,1,0,0,0],
146 |                         [1,0,1,0,0,0],
147 |                         [1,1,1,0,0,0],
148 |                         [0,0,1,1,1,0],
149 |                         [0,0,1,1,0,0],
150 |                         [0,0,1,1,1,0]])
151 | 
152 | 
153 |     rng = numpy.random.RandomState(123)
154 | 
155 |     # construct RBM
156 |     rbm = RBM(input=data, n_visible=6, n_hidden=2, numpy_rng=rng)
157 | 
158 |     # train
159 |     for epoch in xrange(training_epochs):
160 |         rbm.contrastive_divergence(lr=learning_rate, k=k)
161 |         # cost = rbm.get_reconstruction_cross_entropy()
162 |         # print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
163 | 
164 | 
165 |     # test
166 |     v = numpy.array([[1, 1, 0, 0, 0, 0],
167 |                      [0, 0, 0, 1, 1, 0]])
168 | 
169 |     print rbm.reconstruct(v)
170 | 
171 | 
172 | 
173 | if __name__ == "__main__":
174 |     test_rbm()
175 | 


--------------------------------------------------------------------------------
/cpp/RBM.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <math.h>
  3 | #include "RBM.h"
  4 | using namespace std;
  5 | 
  6 | double uniform(double min, double max) {
  7 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;
  8 | }
  9 | 
 10 | int binomial(int n, double p) {
 11 |   if(p < 0 || p > 1) return 0;
 12 |   
 13 |   int c = 0;
 14 |   double r;
 15 |   
 16 |   for(int i=0; i<n; i++) {
 17 |     r = rand() / (RAND_MAX + 1.0);
 18 |     if (r < p) c++;
 19 |   }
 20 | 
 21 |   return c;
 22 | }
 23 | 
 24 | double sigmoid(double x) {
 25 |   return 1.0 / (1.0 + exp(-x));
 26 | }
 27 | 
 28 | 
 29 | RBM::RBM(int size, int n_v, int n_h, double **w, double *hb, double *vb) {
 30 |   N = size;
 31 |   n_visible = n_v;
 32 |   n_hidden = n_h;
 33 | 
 34 |   if(w == NULL) {
 35 |     W = new double*[n_hidden];
 36 |     for(int i=0; i<n_hidden; i++) W[i] = new double[n_visible];
 37 |     double a = 1.0 / n_visible;
 38 | 
 39 |     for(int i=0; i<n_hidden; i++) {
 40 |       for(int j=0; j<n_visible; j++) {
 41 |         W[i][j] = uniform(-a, a);
 42 |       }
 43 |     }
 44 |   } else {
 45 |     W = w;
 46 |   }
 47 | 
 48 |   if(hb == NULL) {
 49 |     hbias = new double[n_hidden];
 50 |     for(int i=0; i<n_hidden; i++) hbias[i] = 0;
 51 |   } else {
 52 |     hbias = hb;
 53 |   }
 54 | 
 55 |   if(vb == NULL) {
 56 |     vbias = new double[n_visible];
 57 |     for(int i=0; i<n_visible; i++) vbias[i] = 0;
 58 |   } else {
 59 |     vbias = vb;
 60 |   }
 61 | }
 62 | 
 63 | RBM::~RBM() {
 64 |   for(int i=0; i<n_hidden; i++) delete[] W[i];
 65 |   delete[] W;
 66 |   delete[] hbias;
 67 |   delete[] vbias;
 68 | }
 69 | 
 70 | 
 71 | void RBM::contrastive_divergence(int *input, double lr, int k) {
 72 |   double *ph_mean = new double[n_hidden];
 73 |   int *ph_sample = new int[n_hidden];
 74 |   double *nv_means = new double[n_visible];
 75 |   int *nv_samples = new int[n_visible];
 76 |   double *nh_means = new double[n_hidden];
 77 |   int *nh_samples = new int[n_hidden];
 78 | 
 79 |   /* CD-k */
 80 |   sample_h_given_v(input, ph_mean, ph_sample);
 81 | 
 82 |   for(int step=0; step<k; step++) {
 83 |     if(step == 0) {
 84 |       gibbs_hvh(ph_sample, nv_means, nv_samples, nh_means, nh_samples);
 85 |     } else {
 86 |       gibbs_hvh(nh_samples, nv_means, nv_samples, nh_means, nh_samples);
 87 |     }
 88 |   }
 89 | 
 90 |   for(int i=0; i<n_hidden; i++) {
 91 |     for(int j=0; j<n_visible; j++) {
 92 |       // W[i][j] += lr * (ph_sample[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
 93 |       W[i][j] += lr * (ph_mean[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
 94 |     }
 95 |     hbias[i] += lr * (ph_sample[i] - nh_means[i]) / N;
 96 |   }
 97 | 
 98 |   for(int i=0; i<n_visible; i++) {
 99 |     vbias[i] += lr * (input[i] - nv_samples[i]) / N;
100 |   }
101 | 
102 |   delete[] ph_mean;
103 |   delete[] ph_sample;
104 |   delete[] nv_means;
105 |   delete[] nv_samples;
106 |   delete[] nh_means;
107 |   delete[] nh_samples;
108 | }
109 | 
110 | void RBM::sample_h_given_v(int *v0_sample, double *mean, int *sample) {
111 |   for(int i=0; i<n_hidden; i++) {
112 |     mean[i] = propup(v0_sample, W[i], hbias[i]);
113 |     sample[i] = binomial(1, mean[i]);
114 |   }
115 | }
116 | 
117 | void RBM::sample_v_given_h(int *h0_sample, double *mean, int *sample) {
118 |   for(int i=0; i<n_visible; i++) {
119 |     mean[i] = propdown(h0_sample, i, vbias[i]);
120 |     sample[i] = binomial(1, mean[i]);
121 |   }
122 | }
123 | 
124 | double RBM::propup(int *v, double *w, double b) {
125 |   double pre_sigmoid_activation = 0.0;
126 |   for(int j=0; j<n_visible; j++) {
127 |     pre_sigmoid_activation += w[j] * v[j];
128 |   }
129 |   pre_sigmoid_activation += b;
130 |   return sigmoid(pre_sigmoid_activation);
131 | }
132 | 
133 | double RBM::propdown(int *h, int i, double b) {
134 |   double pre_sigmoid_activation = 0.0;
135 |   for(int j=0; j<n_hidden; j++) {
136 |     pre_sigmoid_activation += W[j][i] * h[j];
137 |   }
138 |   pre_sigmoid_activation += b;
139 |   return sigmoid(pre_sigmoid_activation);
140 | }
141 | 
142 | void RBM::gibbs_hvh(int *h0_sample, double *nv_means, int *nv_samples, \
143 |                     double *nh_means, int *nh_samples) {
144 |   sample_v_given_h(h0_sample, nv_means, nv_samples);
145 |   sample_h_given_v(nv_samples, nh_means, nh_samples);
146 | }
147 | 
148 | void RBM::reconstruct(int *v, double *reconstructed_v) {
149 |   double *h = new double[n_hidden];
150 |   double pre_sigmoid_activation;
151 | 
152 |   for(int i=0; i<n_hidden; i++) {
153 |     h[i] = propup(v, W[i], hbias[i]);
154 |   }
155 | 
156 |   for(int i=0; i<n_visible; i++) {
157 |     pre_sigmoid_activation = 0.0;
158 |     for(int j=0; j<n_hidden; j++) {
159 |       pre_sigmoid_activation += W[j][i] * h[j];
160 |     }
161 |     pre_sigmoid_activation += vbias[i];
162 | 
163 |     reconstructed_v[i] = sigmoid(pre_sigmoid_activation);
164 |   }
165 | 
166 |   delete[] h;
167 | }
168 | 
169 | 
170 | void test_rbm() {
171 |   srand(0);
172 | 
173 |   double learning_rate = 0.1;
174 |   int training_epochs = 1000;
175 |   int k = 1;
176 |   
177 |   int train_N = 6;
178 |   int test_N = 2;
179 |   int n_visible = 6;
180 |   int n_hidden = 3;
181 | 
182 |   // training data
183 |   int train_X[6][6] = {
184 |     {1, 1, 1, 0, 0, 0},
185 |     {1, 0, 1, 0, 0, 0},
186 |     {1, 1, 1, 0, 0, 0},
187 |     {0, 0, 1, 1, 1, 0},
188 |     {0, 0, 1, 0, 1, 0},
189 |     {0, 0, 1, 1, 1, 0}
190 |   };
191 | 
192 | 
193 |   // construct RBM
194 |   RBM rbm(train_N, n_visible, n_hidden, NULL, NULL, NULL);
195 | 
196 |   // train
197 |   for(int epoch=0; epoch<training_epochs; epoch++) {
198 |     for(int i=0; i<train_N; i++) {
199 |       rbm.contrastive_divergence(train_X[i], learning_rate, k);
200 |     }
201 |   }
202 | 
203 |   // test data
204 |   int test_X[2][6] = {
205 |     {1, 1, 0, 0, 0, 0},
206 |     {0, 0, 0, 1, 1, 0}
207 |   };
208 |   double reconstructed_X[2][6];
209 | 
210 | 
211 |   // test
212 |   for(int i=0; i<test_N; i++) {
213 |     rbm.reconstruct(test_X[i], reconstructed_X[i]);
214 |     for(int j=0; j<n_visible; j++) {
215 |       printf("%.5f ", reconstructed_X[i][j]);
216 |     }
217 |     cout << endl;
218 |   }
219 | 
220 | }
221 | 
222 | 
223 | 
224 | int main() {
225 |   test_rbm();
226 |   return 0;
227 | }
228 | 


--------------------------------------------------------------------------------
/java/RBM/src/RBM.java:
--------------------------------------------------------------------------------
  1 | import java.util.Random;
  2 | 
  3 | public class RBM {
  4 | 	public int N;
  5 | 	public int n_visible;
  6 | 	public int n_hidden;
  7 | 	public double[][] W;
  8 | 	public double[] hbias;
  9 | 	public double[] vbias;
 10 | 	public Random rng;
 11 | 	
 12 | 	public double uniform(double min, double max) {
 13 | 		return rng.nextDouble() * (max - min) + min;
 14 | 	}
 15 | 	
 16 | 	public int binomial(int n, double p) {
 17 | 		if(p < 0 || p > 1) return 0;
 18 | 		
 19 | 		int c = 0;
 20 | 		double r;
 21 | 		
 22 | 		for(int i=0; i<n; i++) {
 23 | 			r = rng.nextDouble();
 24 | 			if (r < p) c++;
 25 | 		}
 26 | 		
 27 | 		return c;
 28 | 	}
 29 | 	
 30 | 	public static double sigmoid(double x) {
 31 | 		return 1.0 / (1.0 + Math.pow(Math.E, -x));
 32 | 	}
 33 | 	
 34 | 	
 35 | 	public RBM(int N, int n_visible, int n_hidden, 
 36 | 			double[][] W, double[] hbias, double[] vbias, Random rng) {
 37 | 		this.N = N;
 38 | 		this.n_visible = n_visible;
 39 | 		this.n_hidden = n_hidden;
 40 | 	
 41 | 		if(rng == null)	this.rng = new Random(1234);
 42 | 		else this.rng = rng;
 43 | 		
 44 | 		if(W == null) {
 45 | 			this.W = new double[this.n_hidden][this.n_visible];
 46 | 			double a = 1.0 / this.n_visible;
 47 | 			
 48 | 			for(int i=0; i<this.n_hidden; i++) {
 49 | 				for(int j=0; j<this.n_visible; j++) {
 50 | 					this.W[i][j] = uniform(-a, a); 
 51 | 				}
 52 | 			}	
 53 | 		} else {
 54 | 			this.W = W;
 55 | 		}
 56 | 		
 57 | 		if(hbias == null) {
 58 | 			this.hbias = new double[this.n_hidden];
 59 | 			for(int i=0; i<this.n_hidden; i++) this.hbias[i] = 0;
 60 | 		} else {
 61 | 			this.hbias = hbias;
 62 | 		}
 63 | 		
 64 | 		if(vbias == null) {
 65 | 			this.vbias = new double[this.n_visible];
 66 | 			for(int i=0; i<this.n_visible; i++) this.vbias[i] = 0;
 67 | 		} else {
 68 | 			this.vbias = vbias;
 69 | 		}
 70 | 	}
 71 | 	
 72 | 	
 73 | 	public void contrastive_divergence(int[] input, double lr, int k) {
 74 | 		double[] ph_mean = new double[n_hidden];
 75 | 		int[] ph_sample = new int[n_hidden];
 76 | 		double[] nv_means = new double[n_visible];
 77 | 		int[] nv_samples = new int[n_visible];
 78 | 		double[] nh_means = new double[n_hidden];
 79 | 		int[] nh_samples = new int[n_hidden];
 80 | 		
 81 | 		/* CD-k */
 82 | 		sample_h_given_v(input, ph_mean, ph_sample);
 83 | 		
 84 | 		for(int step=0; step<k; step++) {
 85 | 			if(step == 0) {
 86 | 				gibbs_hvh(ph_sample, nv_means, nv_samples, nh_means, nh_samples);
 87 | 			} else {
 88 | 				gibbs_hvh(nh_samples, nv_means, nv_samples, nh_means, nh_samples);
 89 | 			}
 90 | 		}
 91 | 		
 92 | 		for(int i=0; i<n_hidden; i++) {
 93 | 			for(int j=0; j<n_visible; j++) {
 94 | 				// W[i][j] += lr * (ph_sample[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
 95 | 				W[i][j] += lr * (ph_mean[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
 96 | 			}
 97 | 			hbias[i] += lr * (ph_sample[i] - nh_means[i]) / N;
 98 | 		}
 99 | 		
100 | 
101 | 		for(int i=0; i<n_visible; i++) {
102 | 			vbias[i] += lr * (input[i] - nv_samples[i]) / N;
103 | 		}
104 | 
105 | 	}
106 | 	
107 | 	
108 | 	public void sample_h_given_v(int[] v0_sample, double[] mean, int[] sample) {
109 | 		for(int i=0; i<n_hidden; i++) {
110 | 			mean[i] = propup(v0_sample, W[i], hbias[i]);
111 | 			sample[i] = binomial(1, mean[i]);
112 | 		}
113 | 	}
114 | 
115 | 	public void sample_v_given_h(int[] h0_sample, double[] mean, int[] sample) {
116 | 		for(int i=0; i<n_visible; i++) {
117 | 			mean[i] = propdown(h0_sample, i, vbias[i]);
118 | 			sample[i] = binomial(1, mean[i]);
119 | 		}
120 | 	}
121 | 	
122 | 	public double propup(int[] v, double[] w, double b) {
123 | 		double pre_sigmoid_activation = 0.0;
124 | 		for(int j=0; j<n_visible; j++) {
125 | 			pre_sigmoid_activation += w[j] * v[j];
126 | 		}
127 | 		pre_sigmoid_activation += b;
128 | 		return sigmoid(pre_sigmoid_activation);
129 | 	}
130 | 	
131 | 	public double propdown(int[] h, int i, double b) {
132 | 	  double pre_sigmoid_activation = 0.0;
133 | 	  for(int j=0; j<n_hidden; j++) {
134 | 	    pre_sigmoid_activation += W[j][i] * h[j];
135 | 	  }
136 | 	  pre_sigmoid_activation += b;
137 | 	  return sigmoid(pre_sigmoid_activation);
138 | 	}
139 | 	
140 | 	public void gibbs_hvh(int[] h0_sample, double[] nv_means, int[] nv_samples, double[] nh_means, int[] nh_samples) {
141 | 	  sample_v_given_h(h0_sample, nv_means, nv_samples);
142 | 	  sample_h_given_v(nv_samples, nh_means, nh_samples);
143 | 	}
144 | 
145 | 
146 | 	public void reconstruct(int[] v, double[] reconstructed_v) {
147 | 	  double[] h = new double[n_hidden];
148 | 	  double pre_sigmoid_activation;
149 | 	
150 | 	  for(int i=0; i<n_hidden; i++) {
151 | 	    h[i] = propup(v, W[i], hbias[i]);
152 | 	  }
153 | 	
154 | 	  for(int i=0; i<n_visible; i++) {
155 | 	    pre_sigmoid_activation = 0.0;
156 | 	    for(int j=0; j<n_hidden; j++) {
157 | 	      pre_sigmoid_activation += W[j][i] * h[j];
158 | 	    }
159 | 	    pre_sigmoid_activation += vbias[i];
160 | 	
161 | 	    reconstructed_v[i] = sigmoid(pre_sigmoid_activation);
162 | 	  }	
163 | 	}
164 | 
165 | 	
166 | 	
167 | 	private static void test_rbm() {
168 | 		Random rng = new Random(123);
169 | 
170 | 		double learning_rate = 0.1;
171 | 		int training_epochs = 1000;
172 | 		int k = 1;
173 | 		  
174 | 		int train_N = 6;
175 | 		int test_N = 2;
176 | 		int n_visible = 6;
177 | 		int n_hidden = 3;
178 | 
179 | 		// training data
180 | 		int[][] train_X = {
181 | 			{1, 1, 1, 0, 0, 0},
182 | 			{1, 0, 1, 0, 0, 0},
183 | 			{1, 1, 1, 0, 0, 0},
184 | 			{0, 0, 1, 1, 1, 0},
185 | 			{0, 0, 1, 0, 1, 0},
186 | 			{0, 0, 1, 1, 1, 0}
187 | 		};
188 | 
189 | 		
190 | 		
191 | 		RBM rbm = new RBM(train_N, n_visible, n_hidden, null, null, null, rng);
192 | 
193 | 		// train
194 | 		for(int epoch=0; epoch<training_epochs; epoch++) {
195 | 			for(int i=0; i<train_N; i++) {
196 | 				rbm.contrastive_divergence(train_X[i], learning_rate, k);
197 | 			}
198 | 		}
199 | 
200 | 		// test data
201 | 		int[][] test_X = {
202 | 			{1, 1, 0, 0, 0, 0},
203 | 			{0, 0, 0, 1, 1, 0}
204 | 		};
205 | 		
206 | 		double[][] reconstructed_X = new double[test_N][n_visible];
207 | 
208 | 		for(int i=0; i<test_N; i++) {
209 | 			rbm.reconstruct(test_X[i], reconstructed_X[i]);
210 | 			for(int j=0; j<n_visible; j++) {
211 | 				System.out.printf("%.5f ", reconstructed_X[i][j]);
212 | 			}
213 | 			System.out.println();
214 | 		}
215 | 	}
216 | 	
217 | 	public static void main(String[] args) {
218 | 		test_rbm();
219 | 	}
220 | 	
221 | }
222 | 


--------------------------------------------------------------------------------
/scala/dA.scala:
--------------------------------------------------------------------------------
  1 | // $ scalac dA.scala
  2 | // $ scala dA
  3 | 
  4 | import scala.util.Random
  5 | import scala.math
  6 | 
  7 | class dA(val N: Int, val n_visible: Int, val n_hidden: Int,
  8 |          _W: Array[Array[Double]]=null, _hbias: Array[Double]=null, _vbias: Array[Double]=null,
  9 |          var rng: Random=null) {
 10 |   
 11 |   var W: Array[Array[Double]] = Array.ofDim[Double](n_hidden, n_visible)
 12 |   var hbias: Array[Double] = new Array[Double](n_hidden)
 13 |   var vbias: Array[Double] = new Array[Double](n_visible)
 14 | 
 15 | 
 16 |   if(rng == null) rng = new Random(1234)
 17 | 
 18 |   if(_W == null) {
 19 |     var i: Int = 0
 20 |     var j: Int = 0
 21 | 
 22 |     val a: Double = 1 / n_visible
 23 |     for(i <- 0 until n_hidden)
 24 |       for(j <- 0 until n_visible)
 25 |         W(i)(j) = uniform(-a, a)
 26 | 
 27 |   } else {
 28 |     W = _W
 29 |   }
 30 | 
 31 |   if(_hbias == null) {
 32 |     var i: Int = 0
 33 |     for(i <- 0 until n_hidden) hbias(i) = 0
 34 |   } else {
 35 |     hbias = _hbias
 36 |   }
 37 | 
 38 |   if(_vbias == null) {
 39 |     var i: Int = 0
 40 |     for(i <- 0 until n_visible) vbias(i) = 0
 41 |   } else {
 42 |     vbias = _vbias
 43 |   }
 44 | 
 45 | 
 46 |   def uniform(min: Double, max: Double): Double = rng.nextDouble() * (max - min) + min
 47 |   def binomial(n: Int, p: Double): Int = {
 48 |     if(p < 0 || p > 1) return 0
 49 |     
 50 |     var c: Int = 0
 51 |     var r: Double = 0
 52 | 
 53 |     var i: Int = 0
 54 |     for(i <- 0 until n) {
 55 |       r = rng.nextDouble()
 56 |       if(r < p) c += 1
 57 |     }
 58 | 
 59 |     c
 60 |   }
 61 | 
 62 |   def sigmoid(x: Double): Double = 1.0 / (1.0 + math.pow(math.E, -x))
 63 | 
 64 | 
 65 |   def get_corrupted_input(x: Array[Int], tilde_x: Array[Int], p: Double) {
 66 |     var i: Int = 0;
 67 |     for(i <- 0 until n_visible) {
 68 |       if(x(i) == 0) {
 69 |         tilde_x(i) = 0;
 70 |       } else {
 71 |         tilde_x(i) = binomial(1, p)
 72 |       }
 73 |     }
 74 |   }
 75 | 
 76 |   // Encode
 77 |   def get_hidden_values(x: Array[Int], y: Array[Double]) {
 78 |     var i: Int = 0
 79 |     var j: Int = 0
 80 |     for(i <- 0 until n_hidden) {
 81 |       y(i) = 0
 82 |       for(j <- 0 until n_visible) {
 83 |         y(i) += W(i)(j) * x(j)
 84 |       }
 85 |       y(i) += hbias(i)
 86 |       y(i) = sigmoid(y(i))
 87 |     }
 88 |   }
 89 | 
 90 |   // Decode
 91 |   def get_reconstructed_input(y: Array[Double], z: Array[Double]) {
 92 |     var i: Int = 0
 93 |     var j: Int = 0
 94 |     for(i <- 0 until n_visible) {
 95 |       z(i) = 0
 96 |       for(j <- 0 until n_hidden) {
 97 |         z(i) += W(j)(i) * y(j)
 98 |       }
 99 |       z(i) += vbias(i)
100 |       z(i) = sigmoid(z(i))
101 |     }
102 |   }
103 | 
104 |   def train(x: Array[Int], lr: Double, corruption_level: Double) {
105 |     var i: Int = 0
106 |     var j: Int = 0
107 | 
108 |     val tilde_x: Array[Int] = new Array[Int](n_visible)
109 |     val y: Array[Double] = new Array[Double](n_hidden)
110 |     val z: Array[Double] = new Array[Double](n_visible)
111 | 
112 |     val L_vbias: Array[Double] = new Array[Double](n_visible)
113 |     val L_hbias: Array[Double] = new Array[Double](n_hidden)
114 | 
115 |     val p: Double = 1 - corruption_level
116 | 
117 |     get_corrupted_input(x, tilde_x, p)
118 |     get_hidden_values(tilde_x, y)
119 |     get_reconstructed_input(y, z)
120 | 
121 |     // vbias
122 |     for(i <- 0 until n_visible) {
123 |       L_vbias(i) = x(i) - z(i)
124 |       vbias(i) += lr * L_vbias(i) / N
125 |     }
126 | 
127 |     // hbias
128 |     for(i <- 0 until n_hidden) {
129 |       L_hbias(i) = 0
130 |       for(j <- 0 until n_visible) {
131 |         L_hbias(i) += W(i)(j) * L_vbias(j)
132 |       }
133 |       L_hbias(i) *= y(i) * (1 - y(i))
134 |       hbias(i) += lr * L_hbias(i) / N
135 |     }
136 | 
137 |     // W
138 |     for(i <- 0 until n_hidden) {
139 |       for(j <- 0 until n_visible) {
140 |         W(i)(j) += lr * (L_hbias(i) * tilde_x(j) + L_vbias(j) * y(i)) / N
141 |       }
142 |     }
143 |   }
144 | 
145 |   def reconstruct(x: Array[Int], z: Array[Double]) {
146 |     val y: Array[Double] = new Array[Double](n_hidden)
147 | 
148 |     get_hidden_values(x, y)
149 |     get_reconstructed_input(y, z)
150 |   }
151 | 
152 | }
153 | 
154 | 
155 | object dA {
156 |   def test_dA() {
157 |     val rng: Random = new Random(123)
158 |     var learning_rate: Double = 0.1
159 |     val corruption_level: Double = 0.3
160 |     val training_epochs: Int = 500
161 | 
162 |     val train_N: Int = 10
163 |     val test_N: Int = 2
164 | 
165 |     val n_visible: Int = 20
166 |     val n_hidden: Int = 5
167 | 
168 |     val train_X: Array[Array[Int]] = Array(
169 |       Array(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
170 |       Array(1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
171 |       Array(1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
172 |       Array(1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
173 |       Array(0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
174 |       Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
175 |       Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1),
176 |       Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1),
177 |       Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1),
178 |       Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0)
179 |     )
180 | 
181 |     val da: dA = new dA(train_N, n_visible, n_hidden, rng=rng)
182 | 
183 |     var i: Int = 0
184 |     var j: Int = 0
185 | 
186 |     // train
187 |     var epoch: Int = 0
188 |     for(epoch <- 0 until training_epochs) {
189 |       for(i <- 0 until train_N) {
190 |         da.train(train_X(i), learning_rate, corruption_level)
191 |       }
192 |     }
193 | 
194 | 
195 |     // test data
196 |     val test_X: Array[Array[Int]] = Array(
197 |       Array(1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
198 |       Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0)
199 |     )
200 | 
201 |     val reconstructed_X: Array[Array[Double]] = Array.ofDim[Double](test_N, n_visible)
202 |     for(i <- 0 until test_N) {
203 |       da.reconstruct(test_X(i), reconstructed_X(i))
204 |       for(j <- 0 until n_visible) {
205 |         printf("%.5f ", reconstructed_X(i)(j))
206 |       }
207 |       println()
208 |     }
209 |   }
210 | 
211 |   def main(args: Array[String]) {
212 |     test_dA()
213 |   }
214 |   
215 | }
216 | 


--------------------------------------------------------------------------------
/python/SdA.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | 
  5 | """
  6 |  Stacked Denoising Autoencoders (SdA)
  7 | 
  8 |  References :
  9 |    - P. Vincent, H. Larochelle, Y. Bengio, P.A. Manzagol: Extracting and
 10 |    Composing Robust Features with Denoising Autoencoders, ICML' 08, 1096-1103,
 11 |    2008
 12 | 
 13 |    - DeepLearningTutorials
 14 |    https://github.com/lisa-lab/DeepLearningTutorials
 15 |    
 16 | """
 17 | 
 18 | import sys
 19 | import numpy
 20 | from HiddenLayer import HiddenLayer
 21 | from LogisticRegression import LogisticRegression
 22 | from dA import dA
 23 | from utils import *
 24 | 
 25 | 
 26 | class SdA(object):
 27 |     def __init__(self, input=None, label=None,\
 28 |                  n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2,\
 29 |                  numpy_rng=None):
 30 |         
 31 |         self.x = input
 32 |         self.y = label
 33 | 
 34 |         self.sigmoid_layers = []
 35 |         self.dA_layers = []
 36 |         self.n_layers = len(hidden_layer_sizes)  # = len(self.rbm_layers)
 37 | 
 38 |         if numpy_rng is None:
 39 |             numpy_rng = numpy.random.RandomState(1234)
 40 | 
 41 |         
 42 |         assert self.n_layers > 0
 43 | 
 44 | 
 45 |         # construct multi-layer
 46 |         for i in xrange(self.n_layers):
 47 |             # layer_size
 48 |             if i == 0:
 49 |                 input_size = n_ins
 50 |             else:
 51 |                 input_size = hidden_layer_sizes[i - 1]
 52 | 
 53 |             # layer_input
 54 |             if i == 0:
 55 |                 layer_input = self.x
 56 |             else:
 57 |                 layer_input = self.sigmoid_layers[-1].sample_h_given_v()
 58 |                 
 59 |             # construct sigmoid_layer
 60 |             sigmoid_layer = HiddenLayer(input=layer_input,
 61 |                                         n_in=input_size,
 62 |                                         n_out=hidden_layer_sizes[i],
 63 |                                         numpy_rng=numpy_rng,
 64 |                                         activation=sigmoid)
 65 |             self.sigmoid_layers.append(sigmoid_layer)
 66 | 
 67 | 
 68 |             # construct dA_layers
 69 |             dA_layer = dA(input=layer_input,
 70 |                           n_visible=input_size,
 71 |                           n_hidden=hidden_layer_sizes[i],
 72 |                           W=sigmoid_layer.W,
 73 |                           hbias=sigmoid_layer.b)
 74 |             self.dA_layers.append(dA_layer)
 75 | 
 76 | 
 77 |         # layer for output using Logistic Regression
 78 |         self.log_layer = LogisticRegression(input=self.sigmoid_layers[-1].sample_h_given_v(),
 79 |                                             label=self.y,
 80 |                                             n_in=hidden_layer_sizes[-1],
 81 |                                             n_out=n_outs)
 82 | 
 83 |         # finetune cost: the negative log likelihood of the logistic regression layer
 84 |         self.finetune_cost = self.log_layer.negative_log_likelihood()
 85 | 
 86 | 
 87 |     def pretrain(self, lr=0.1, corruption_level=0.3, epochs=100):
 88 |         for i in xrange(self.n_layers):
 89 |             if i == 0:
 90 |                 layer_input = self.x
 91 |             else:
 92 |                 layer_input = self.sigmoid_layers[i-1].sample_h_given_v(layer_input)
 93 | 
 94 |             da = self.dA_layers[i]
 95 | 
 96 |             for epoch in xrange(epochs):
 97 |                 da.train(lr=lr, corruption_level=corruption_level, input=layer_input)
 98 | 
 99 |     def finetune(self, lr=0.1, epochs=100):
100 |         layer_input = self.sigmoid_layers[-1].sample_h_given_v()
101 | 
102 |         # train log_layer
103 |         epoch = 0
104 | 
105 |         while epoch < epochs:
106 |             self.log_layer.train(lr=lr, input=layer_input)
107 |             # self.finetune_cost = self.log_layer.negative_log_likelihood()
108 |             # print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, self.finetune_cost
109 |             
110 |             lr *= 0.95
111 |             epoch += 1
112 | 
113 | 
114 |     def predict(self, x):
115 |         layer_input = x
116 |         
117 |         for i in xrange(self.n_layers):
118 |             sigmoid_layer = self.sigmoid_layers[i]
119 |             layer_input = sigmoid_layer.output(input=layer_input)
120 | 
121 |         out = self.log_layer.predict(layer_input)
122 |         return out
123 | 
124 | 
125 | 
126 | def test_SdA(pretrain_lr=0.1, pretraining_epochs=1000, corruption_level=0.3, \
127 |              finetune_lr=0.1, finetune_epochs=200):
128 |     x = numpy.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
129 |                      [1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
130 |                      [1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
131 |                      [1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
132 |                      [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
133 |                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
134 |                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1],
135 |                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1],
136 |                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1],
137 |                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0]])
138 | 
139 |     y = numpy.array([[1, 0],
140 |                      [1, 0],
141 |                      [1, 0],
142 |                      [1, 0],
143 |                      [1, 0],
144 |                      [0, 1],
145 |                      [0, 1],
146 |                      [0, 1],
147 |                      [0, 1],
148 |                      [0, 1]])
149 | 
150 | 
151 |     rng = numpy.random.RandomState(123)
152 | 
153 |     # construct SdA
154 |     sda = SdA(input=x, label=y, \
155 |               n_ins=20, hidden_layer_sizes=[15, 15], n_outs=2, numpy_rng=rng)
156 | 
157 |     # pre-training
158 |     sda.pretrain(lr=pretrain_lr, corruption_level=corruption_level, epochs=pretraining_epochs)
159 | 
160 |     # fine-tuning
161 |     sda.finetune(lr=finetune_lr, epochs=finetune_epochs)
162 | 
163 | 
164 |     # test
165 |     x = numpy.array([[1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
166 |                      [1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
167 |                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1],
168 |                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1]])
169 | 
170 |     print sda.predict(x)
171 |     
172 |     
173 | 
174 | if __name__ == "__main__":
175 |     test_SdA()
176 | 


--------------------------------------------------------------------------------
/c/dA.c:
--------------------------------------------------------------------------------
  1 | #include <stdio.h>
  2 | #include <stdlib.h>
  3 | #include <math.h>
  4 | #include "dA.h"
  5 | #include "utils.h"
  6 | 
  7 | 
  8 | void test_dbn(void);
  9 | 
 10 | 
 11 | double uniform(double min, double max) {
 12 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;  
 13 | }
 14 | 
 15 | int binomial(int n, double p) {
 16 |   if(p < 0 || p > 1) return 0;
 17 | 
 18 |   int i;
 19 |   int c = 0;
 20 |   double r;
 21 | 
 22 |   for(i=0; i<n; i++) {
 23 |     r = rand() / (RAND_MAX + 1.0);
 24 |     if (r < p) c++;
 25 |   }
 26 | 
 27 |   return c;
 28 | }
 29 | 
 30 | double sigmoid(double x) {
 31 |   return 1.0 / (1.0 + exp(-x));
 32 | }
 33 | 
 34 | 
 35 | void dA__construct(dA* this, int N, int n_visible, int n_hidden, \
 36 |                    double **W, double *hbias, double *vbias) {
 37 |   int i, j;
 38 |   double a = 1.0 / n_visible;
 39 |   
 40 |   this->N = N;
 41 |   this->n_visible = n_visible;
 42 |   this->n_hidden = n_hidden;
 43 | 
 44 |   if(W == NULL) {
 45 |     this->W = (double **)malloc(sizeof(double*) * n_hidden);
 46 |     this->W[0] = (double *)malloc(sizeof(double) * n_visible * n_hidden);
 47 |     for(i=0; i<n_hidden; i++) this->W[i] = this->W[0] + i * n_visible;
 48 | 
 49 |     for(i=0; i<n_hidden; i++) {
 50 |       for(j=0; j<n_visible; j++) {
 51 |         this->W[i][j] = uniform(-a, a);
 52 |       }
 53 |     }
 54 |   } else {
 55 |     this->W = W;
 56 |   }
 57 | 
 58 |   if(hbias == NULL) {
 59 |     this->hbias = (double *)malloc(sizeof(double) * n_hidden);
 60 |     for(i=0; i<n_hidden; i++) this->hbias[i] = 0;
 61 |   } else {
 62 |     this->hbias = hbias;
 63 |   }
 64 | 
 65 |   if(vbias == NULL) {
 66 |     this->vbias = (double *)malloc(sizeof(double) * n_visible);
 67 |     for(i=0; i<n_visible; i++) this->vbias[i] = 0;
 68 |   } else {
 69 |     this->vbias = vbias;
 70 |   }
 71 | }
 72 | 
 73 | void dA__destruct(dA* this) {
 74 |   free(this->W[0]);
 75 |   free(this->W);
 76 |   free(this->hbias);
 77 |   free(this->vbias);
 78 | }
 79 | 
 80 | void dA_get_corrupted_input(dA* this, int *x, int *tilde_x, double p) {
 81 |   int i;
 82 |   for(i=0; i<this->n_visible; i++) {
 83 |     if(x[i] == 0) {
 84 |       tilde_x[i] = 0;
 85 |     } else {
 86 |       tilde_x[i] = binomial(1, p);
 87 |     }
 88 |   }
 89 | }
 90 | 
 91 | // Encode
 92 | void dA_get_hidden_values(dA* this, int *x, double *y) {
 93 |   int i,j;
 94 |   for(i=0; i<this->n_hidden; i++) {
 95 |     y[i] = 0;
 96 |     for(j=0; j<this->n_visible; j++) {
 97 |       y[i] += this->W[i][j] * x[j];
 98 |     }
 99 |     y[i] += this->hbias[i];
100 |     y[i] = sigmoid(y[i]);
101 |   }
102 | }
103 | 
104 | // Decode
105 | void dA_get_reconstructed_input(dA* this, double *y, double *z) {
106 |   int i, j;
107 |   for(i=0; i<this->n_visible; i++) {
108 |     z[i] = 0;
109 |     for(j=0; j<this->n_hidden; j++) {
110 |       z[i] += this->W[j][i] * y[j];
111 |     }
112 |     z[i] += this->vbias[i];
113 |     z[i] = sigmoid(z[i]);
114 |   }
115 | }
116 | 
117 | 
118 | void dA_train(dA* this, int *x, double lr, double corruption_level) {
119 |   int i, j;
120 |   
121 |   int *tilde_x = (int *)malloc(sizeof(int) * this->n_visible);
122 |   double *y = (double *)malloc(sizeof(double) * this->n_hidden);
123 |   double *z = (double *)malloc(sizeof(double) * this->n_visible);
124 | 
125 |   double *L_vbias = (double *)malloc(sizeof(double) * this->n_visible);
126 |   double *L_hbias = (double *)malloc(sizeof(double) * this->n_hidden);
127 | 
128 |   double p = 1 - corruption_level;
129 | 
130 |   dA_get_corrupted_input(this, x, tilde_x, p);
131 |   dA_get_hidden_values(this, tilde_x, y);
132 |   dA_get_reconstructed_input(this, y, z);
133 | 
134 |   // vbias
135 |   for(i=0; i<this->n_visible; i++) {
136 |     L_vbias[i] = x[i] - z[i];
137 |     this->vbias[i] += lr * L_vbias[i] / this->N;
138 |   }
139 | 
140 |   // hbias
141 |   for(i=0; i<this->n_hidden; i++) {
142 |     L_hbias[i] = 0;
143 |     for(j=0; j<this->n_visible; j++) {
144 |       L_hbias[i] += this->W[i][j] * L_vbias[j];
145 |     }
146 |     L_hbias[i] *= y[i] * (1 - y[i]);
147 | 
148 |     this->hbias[i] += lr * L_hbias[i] / this->N;
149 |   }
150 | 
151 |   // W
152 |   for(i=0; i<this->n_hidden; i++) {
153 |     for(j=0; j<this->n_visible; j++) {
154 |       this->W[i][j] += lr * (L_hbias[i] * tilde_x[j] + L_vbias[j] * y[i]) / this->N;
155 |     }
156 |   }
157 | 
158 |   free(L_hbias);
159 |   free(L_vbias);
160 |   free(z);
161 |   free(y);
162 |   free(tilde_x);
163 | }
164 | 
165 | void dA_reconstruct(dA* this, int *x, double *z) {
166 |   int i;
167 |   double *y = (double *)malloc(sizeof(double) * this->n_hidden);
168 | 
169 |   dA_get_hidden_values(this, x, y);
170 |   dA_get_reconstructed_input(this, y, z);
171 | 
172 |   free(y);
173 | }
174 | 
175 | void test_dbn(void) {
176 |   srand(0);
177 |   int i, j, epoch;
178 |   
179 |   double learning_rate = 0.1;
180 |   double corruption_level = 0.3;
181 |   int training_epochs = 100;
182 | 
183 |   int train_N = 10;
184 |   int test_N = 2;
185 |   int n_visible = 20;
186 |   int n_hidden = 5;
187 | 
188 |   // training data
189 |   int train_X[10][20] = {
190 |     {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
191 |     {1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
192 |     {1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
193 |     {1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
194 |     {0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
195 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
196 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1},
197 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1},
198 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1},
199 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0}
200 |   };
201 | 
202 |   // construct dA  
203 |   dA da;
204 |   dA__construct(&da, train_N, n_visible, n_hidden, NULL, NULL, NULL);
205 | 
206 |   // train
207 |   for(epoch=0; epoch<training_epochs; epoch++) {
208 |     for(i=0; i<train_N; i++) {
209 |       dA_train(&da, train_X[i], learning_rate, corruption_level);
210 |     }
211 |   }
212 | 
213 |   // test data
214 |   int test_X[2][20] = {
215 |     {1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
216 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0}
217 |   };
218 |   double reconstructed_X[2][20];
219 | 
220 |   
221 |   // test
222 |   for(i=0; i<test_N; i++) {
223 |     dA_reconstruct(&da, test_X[i], reconstructed_X[i]);
224 |     for(j=0; j<n_visible; j++) {
225 |       printf("%.5f ", reconstructed_X[i][j]);
226 |     }
227 |     printf("\n");
228 |   }
229 | 
230 | 
231 |   // destruct dA
232 |   dA__destruct(&da);
233 | }
234 | 
235 | 
236 | int main(void) {
237 |   test_dbn();
238 |   return 0;
239 | }
240 | 


--------------------------------------------------------------------------------
/python/DBN.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | '''
  5 |  Deep Belief Nets (DBN)
  6 | 
  7 |  References :
  8 |    - Y. Bengio, P. Lamblin, D. Popovici, H. Larochelle: Greedy Layer-Wise
  9 |    Training of Deep Networks, Advances in Neural Information Processing
 10 |    Systems 19, 2007
 11 | 
 12 | 
 13 |    - DeepLearningTutorials
 14 |    https://github.com/lisa-lab/DeepLearningTutorials
 15 | 
 16 | '''
 17 | 
 18 | import sys
 19 | import numpy
 20 | from HiddenLayer import HiddenLayer
 21 | from LogisticRegression import LogisticRegression
 22 | from RBM import RBM
 23 | from utils import *
 24 | 
 25 | 
 26 | class DBN(object):
 27 |     def __init__(self, input=None, label=None,\
 28 |                  n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2,\
 29 |                  numpy_rng=None):
 30 |         
 31 |         self.x = input
 32 |         self.y = label
 33 | 
 34 |         self.sigmoid_layers = []
 35 |         self.rbm_layers = []
 36 |         self.n_layers = len(hidden_layer_sizes)  # = len(self.rbm_layers)
 37 | 
 38 |         if numpy_rng is None:
 39 |             numpy_rng = numpy.random.RandomState(1234)
 40 | 
 41 |         
 42 |         assert self.n_layers > 0
 43 | 
 44 | 
 45 |         # construct multi-layer
 46 |         for i in xrange(self.n_layers):
 47 |             # layer_size
 48 |             if i == 0:
 49 |                 input_size = n_ins
 50 |             else:
 51 |                 input_size = hidden_layer_sizes[i - 1]
 52 | 
 53 |             # layer_input
 54 |             if i == 0:
 55 |                 layer_input = self.x
 56 |             else:
 57 |                 layer_input = self.sigmoid_layers[-1].sample_h_given_v()
 58 |                 
 59 |             # construct sigmoid_layer
 60 |             sigmoid_layer = HiddenLayer(input=layer_input,
 61 |                                         n_in=input_size,
 62 |                                         n_out=hidden_layer_sizes[i],
 63 |                                         numpy_rng=numpy_rng,
 64 |                                         activation=sigmoid)
 65 |             self.sigmoid_layers.append(sigmoid_layer)
 66 | 
 67 | 
 68 |             # construct rbm_layer
 69 |             rbm_layer = RBM(input=layer_input,
 70 |                             n_visible=input_size,
 71 |                             n_hidden=hidden_layer_sizes[i],
 72 |                             W=sigmoid_layer.W,     # W, b are shared
 73 |                             hbias=sigmoid_layer.b)
 74 |             self.rbm_layers.append(rbm_layer)
 75 | 
 76 | 
 77 |         # layer for output using Logistic Regression
 78 |         self.log_layer = LogisticRegression(input=self.sigmoid_layers[-1].sample_h_given_v(),
 79 |                                             label=self.y,
 80 |                                             n_in=hidden_layer_sizes[-1],
 81 |                                             n_out=n_outs)
 82 | 
 83 |         # finetune cost: the negative log likelihood of the logistic regression layer
 84 |         self.finetune_cost = self.log_layer.negative_log_likelihood()
 85 | 
 86 | 
 87 | 
 88 |     def pretrain(self, lr=0.1, k=1, epochs=100):
 89 |         # pre-train layer-wise
 90 |         for i in xrange(self.n_layers):
 91 |             if i == 0:
 92 |                 layer_input = self.x
 93 |             else:
 94 |                 layer_input = self.sigmoid_layers[i-1].sample_h_given_v(layer_input)
 95 |             rbm = self.rbm_layers[i]
 96 |             
 97 |             for epoch in xrange(epochs):
 98 |                 rbm.contrastive_divergence(lr=lr, k=k, input=layer_input)
 99 |                 # cost = rbm.get_reconstruction_cross_entropy()
100 |                 # print >> sys.stderr, \
101 |                 #        'Pre-training layer %d, epoch %d, cost ' %(i, epoch), cost
102 | 
103 |     # def pretrain(self, lr=0.1, k=1, epochs=100):
104 |     #     # pre-train layer-wise
105 |     #     for i in xrange(self.n_layers):
106 |     #         rbm = self.rbm_layers[i]
107 |             
108 |     #         for epoch in xrange(epochs):
109 |     #             layer_input = self.x
110 |     #             for j in xrange(i):
111 |     #                 layer_input = self.sigmoid_layers[j].sample_h_given_v(layer_input)
112 |             
113 |     #             rbm.contrastive_divergence(lr=lr, k=k, input=layer_input)
114 |     #             # cost = rbm.get_reconstruction_cross_entropy()
115 |     #             # print >> sys.stderr, \
116 |     #             #        'Pre-training layer %d, epoch %d, cost ' %(i, epoch), cost
117 | 
118 | 
119 |     def finetune(self, lr=0.1, epochs=100):
120 |         layer_input = self.sigmoid_layers[-1].sample_h_given_v()
121 | 
122 |         # train log_layer
123 |         epoch = 0
124 |         done_looping = False
125 |         while (epoch < epochs) and (not done_looping):
126 |             self.log_layer.train(lr=lr, input=layer_input)
127 |             # self.finetune_cost = self.log_layer.negative_log_likelihood()
128 |             # print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, self.finetune_cost
129 |             
130 |             lr *= 0.95
131 |             epoch += 1
132 | 
133 | 
134 |     def predict(self, x):
135 |         layer_input = x
136 |         
137 |         for i in xrange(self.n_layers):
138 |             sigmoid_layer = self.sigmoid_layers[i]
139 |             layer_input = sigmoid_layer.output(input=layer_input)
140 | 
141 |         out = self.log_layer.predict(layer_input)
142 |         return out
143 | 
144 | 
145 | 
146 | def test_dbn(pretrain_lr=0.1, pretraining_epochs=1000, k=1, \
147 |              finetune_lr=0.1, finetune_epochs=200):
148 | 
149 |     x = numpy.array([[1,1,1,0,0,0],
150 |                      [1,0,1,0,0,0],
151 |                      [1,1,1,0,0,0],
152 |                      [0,0,1,1,1,0],
153 |                      [0,0,1,1,0,0],
154 |                      [0,0,1,1,1,0]])
155 |     y = numpy.array([[1, 0],
156 |                      [1, 0],
157 |                      [1, 0],
158 |                      [0, 1],
159 |                      [0, 1],
160 |                      [0, 1]])
161 | 
162 |     
163 |     rng = numpy.random.RandomState(123)
164 | 
165 |     # construct DBN
166 |     dbn = DBN(input=x, label=y, n_ins=6, hidden_layer_sizes=[3, 3], n_outs=2, numpy_rng=rng)
167 | 
168 |     # pre-training (TrainUnsupervisedDBN)
169 |     dbn.pretrain(lr=pretrain_lr, k=1, epochs=pretraining_epochs)
170 |     
171 |     # fine-tuning (DBNSupervisedFineTuning)
172 |     dbn.finetune(lr=finetune_lr, epochs=finetune_epochs)
173 | 
174 | 
175 |     # test
176 |     x = numpy.array([[1, 1, 0, 0, 0, 0],
177 |                      [0, 0, 0, 1, 1, 0],
178 |                      [1, 1, 1, 1, 1, 0]])
179 |     
180 |     print dbn.predict(x)
181 | 
182 | 
183 | 
184 | if __name__ == "__main__":
185 |     test_dbn()
186 | 


--------------------------------------------------------------------------------
/scala/RBM.scala:
--------------------------------------------------------------------------------
  1 | // $ scalac RBM.scala
  2 | // $ scala RBM
  3 | 
  4 | import scala.util.Random
  5 | import scala.math
  6 | 
  7 | class RBM(val N: Int, val n_visible: Int, val n_hidden: Int,
  8 |           _W: Array[Array[Double]]=null, _hbias: Array[Double]=null, _vbias: Array[Double]=null,
  9 |           var rng: Random=null) {
 10 |   
 11 |   var W: Array[Array[Double]] = Array.ofDim[Double](n_hidden, n_visible)
 12 |   var hbias: Array[Double] = new Array[Double](n_hidden)
 13 |   var vbias: Array[Double] = new Array[Double](n_visible)
 14 | 
 15 | 
 16 |   if(rng == null) rng = new Random(1234)
 17 | 
 18 |   if(_W == null) {
 19 |     var i: Int = 0
 20 |     var j: Int = 0
 21 | 
 22 |     val a: Double = 1 / n_visible
 23 |     for(i <- 0 until n_hidden)
 24 |       for(j <- 0 until n_visible)
 25 |         W(i)(j) = uniform(-a, a)
 26 | 
 27 |   } else {
 28 |     W = _W
 29 |   }
 30 | 
 31 |   if(_hbias == null) {
 32 |     var i: Int = 0
 33 |     for(i <- 0 until n_hidden) hbias(i) = 0
 34 |   } else {
 35 |     hbias = _hbias
 36 |   }
 37 | 
 38 |   if(_vbias == null) {
 39 |     var i: Int = 0
 40 |     for(i <- 0 until n_visible) vbias(i) = 0
 41 |   } else {
 42 |     vbias = _vbias
 43 |   }
 44 | 
 45 | 
 46 |   def uniform(min: Double, max: Double): Double = rng.nextDouble() * (max - min) + min
 47 |   def binomial(n: Int, p: Double): Int = {
 48 |     if(p < 0 || p > 1) return 0
 49 |     
 50 |     var c: Int = 0
 51 |     var r: Double = 0
 52 | 
 53 |     var i: Int = 0
 54 |     for(i <- 0 until n) {
 55 |       r = rng.nextDouble()
 56 |       if(r < p) c += 1
 57 |     }
 58 | 
 59 |     c
 60 |   }
 61 | 
 62 |   def sigmoid(x: Double): Double = 1.0 / (1.0 + math.pow(math.E, -x))
 63 | 
 64 |   
 65 |   def contrastive_divergence(input: Array[Int], lr: Double, k: Int) {
 66 |     val ph_mean: Array[Double] = new Array[Double](n_hidden)
 67 |     val ph_sample: Array[Int] = new Array[Int](n_hidden)
 68 |     val nv_means: Array[Double] = new Array[Double](n_visible)
 69 |     val nv_samples: Array[Int] = new Array[Int](n_visible)
 70 |     val nh_means: Array[Double] = new Array[Double](n_hidden)
 71 |     val nh_samples: Array[Int] = new Array[Int](n_hidden)
 72 | 
 73 |     /* CD-k */
 74 |     sample_h_given_v(input, ph_mean, ph_sample)
 75 | 
 76 |     var step: Int = 0
 77 |     for(step <- 0 until k) {
 78 |       if(step == 0) {
 79 |         gibbs_hvh(ph_sample, nv_means, nv_samples, nh_means, nh_samples)
 80 |       } else {
 81 |         gibbs_hvh(nh_samples, nv_means, nv_samples, nh_means, nh_samples)
 82 |       }
 83 |     }
 84 | 
 85 |     var i: Int = 0
 86 |     var j: Int = 0
 87 |     for(i <- 0 until n_hidden) {
 88 |       for(j <- 0 until n_visible) {
 89 |         // W(i)(j) += lr * (ph_sample(i) * input(j) - nh_means(i) * nv_samples(j)) / N
 90 |         W(i)(j) += lr * (ph_mean(i) * input(j) - nh_means(i) * nv_samples(j)) / N
 91 |       }
 92 |       hbias(i) += lr * (ph_sample(i) - nh_means(i)) / N
 93 |     }
 94 | 
 95 |     for(i <- 0 until n_visible) {
 96 |       vbias(i) += lr * (input(i) - nv_samples(i)) / N
 97 |     }
 98 |   }
 99 | 
100 | 
101 |   def sample_h_given_v(v0_sample: Array[Int], mean: Array[Double], sample: Array[Int]) {
102 |     var i: Int = 0
103 |     for(i <- 0 until n_hidden) {
104 |       mean(i) = propup(v0_sample, W(i), hbias(i))
105 |       sample(i) = binomial(1, mean(i))
106 |     }
107 |   }
108 | 
109 |   def sample_v_given_h(h0_sample: Array[Int], mean: Array[Double], sample: Array[Int]) {
110 |     var i: Int = 0
111 |     for(i <- 0 until n_visible) {
112 |       mean(i) = propdown(h0_sample, i, vbias(i))
113 |       sample(i) = binomial(1, mean(i))
114 |     }
115 |   }
116 | 
117 |   def propup(v: Array[Int], w: Array[Double], b: Double): Double = {
118 |     var pre_sigmoid_activation: Double = 0
119 |     var j: Int = 0
120 |     for(j <- 0 until n_visible) {
121 |       pre_sigmoid_activation += w(j) * v(j)
122 |     }
123 |     pre_sigmoid_activation += b
124 |     sigmoid(pre_sigmoid_activation)
125 |   }
126 | 
127 |   def propdown(h: Array[Int], i: Int, b: Double): Double = {
128 |     var pre_sigmoid_activation: Double = 0
129 |     var j: Int = 0
130 |     for(j <- 0 until n_hidden) {
131 |       pre_sigmoid_activation += W(j)(i) * h(j)
132 |     }
133 |     pre_sigmoid_activation += b
134 |     sigmoid(pre_sigmoid_activation)
135 |   }
136 | 
137 |   def gibbs_hvh(h0_sample: Array[Int], nv_means: Array[Double], nv_samples: Array[Int], nh_means: Array[Double], nh_samples: Array[Int]) {
138 |     sample_v_given_h(h0_sample, nv_means, nv_samples)
139 |     sample_h_given_v(nv_samples, nh_means, nh_samples)
140 |   }
141 | 
142 | 
143 |   def reconstruct(v: Array[Int], reconstructed_v: Array[Double]) {
144 |     val h: Array[Double] = new Array[Double](n_hidden)
145 |     var pre_sigmoid_activation: Double = 0
146 |     
147 |     var i: Int = 0
148 |     var j: Int = 0
149 |     
150 |     for(i <- 0 until n_hidden) {
151 |       h(i) = propup(v, W(i), hbias(i))
152 |     }
153 | 
154 |     for(i <- 0 until n_visible) {
155 |       pre_sigmoid_activation = 0
156 |       for(j <- 0 until n_hidden) {
157 |         pre_sigmoid_activation += W(j)(i) * h(j)
158 |       }
159 |       pre_sigmoid_activation += vbias(i)
160 |       reconstructed_v(i) = sigmoid(pre_sigmoid_activation)
161 |     }
162 |   }
163 | }
164 | 
165 | 
166 | object RBM {
167 |   def test_rbm() {
168 |     val rng: Random = new Random(123)
169 | 
170 |     var learning_rate: Double = 0.1
171 |     val training_epochs: Int = 1000
172 |     val k: Int = 1
173 | 
174 |     val train_N: Int = 6;
175 |     val test_N: Int = 2
176 |     val n_visible: Int = 6
177 |     val n_hidden: Int = 3
178 | 
179 |     val train_X: Array[Array[Int]] = Array(
180 |       Array(1, 1, 1, 0, 0, 0),
181 |       Array(1, 0, 1, 0, 0, 0),
182 |       Array(1, 1, 1, 0, 0, 0),
183 |       Array(0, 0, 1, 1, 1, 0),
184 |       Array(0, 0, 1, 0, 1, 0),
185 |       Array(0, 0, 1, 1, 1, 0)
186 |     )
187 | 
188 | 
189 |     val rbm: RBM = new RBM(train_N, n_visible, n_hidden, rng=rng)
190 | 
191 |     var i: Int = 0
192 |     var j: Int = 0
193 | 
194 |     // train
195 |     var epoch: Int = 0
196 |     for(epoch <- 0 until training_epochs) {
197 |       for(i <- 0 until train_N) {
198 |         rbm.contrastive_divergence(train_X(i), learning_rate, k)
199 |       }
200 |     }
201 | 
202 |     // test data
203 |     val test_X: Array[Array[Int]] = Array(
204 |       Array(1, 1, 0, 0, 0, 0),
205 |       Array(0, 0, 0, 1, 1, 0)
206 |     )
207 | 
208 |     val reconstructed_X: Array[Array[Double]] = Array.ofDim[Double](test_N, n_visible)
209 |     for(i <- 0 until test_N) {
210 |       rbm.reconstruct(test_X(i), reconstructed_X(i))
211 |       for(j <- 0 until n_visible) {
212 |         printf("%.5f ", reconstructed_X(i)(j))
213 |       }
214 |       println()
215 |     }
216 | 
217 |   }
218 | 
219 |   def main(args: Array[String]) {
220 |     test_rbm()
221 |   }
222 | 
223 | }
224 | 


--------------------------------------------------------------------------------
/java/DBN/src/DBN.java:
--------------------------------------------------------------------------------
  1 | import java.util.Random;
  2 | 
  3 | public class DBN {
  4 | 	public int N;
  5 | 	public int n_ins;
  6 | 	public int[] hidden_layer_sizes;
  7 | 	public int n_outs;
  8 | 	public int n_layers;
  9 | 	public HiddenLayer[] sigmoid_layers;
 10 | 	public RBM[] rbm_layers;
 11 | 	public LogisticRegression log_layer;
 12 | 	public Random rng;
 13 | 
 14 | 	public static double sigmoid(double x) {
 15 | 		return 1.0 / (1.0 + Math.pow(Math.E, -x));
 16 | 	}
 17 | 	
 18 | 	
 19 | 	public DBN(int N, int n_ins, int[] hidden_layer_sizes, int n_outs, int n_layers, Random rng) {
 20 | 		int input_size;
 21 | 		
 22 | 		this.N = N;
 23 | 		this.n_ins = n_ins;
 24 | 		this.hidden_layer_sizes = hidden_layer_sizes;
 25 | 		this.n_outs = n_outs;
 26 | 		this.n_layers = n_layers;
 27 | 		
 28 | 		this.sigmoid_layers = new HiddenLayer[n_layers];
 29 | 		this.rbm_layers = new RBM[n_layers];
 30 | 
 31 | 		if(rng == null)	this.rng = new Random(1234);
 32 | 		else this.rng = rng;		
 33 | 		
 34 | 		// construct multi-layer
 35 | 		for(int i=0; i<this.n_layers; i++) {
 36 | 			if(i == 0) {
 37 | 				input_size = this.n_ins;
 38 | 			} else {
 39 | 				input_size = this.hidden_layer_sizes[i-1];
 40 | 			}
 41 | 			
 42 | 			// construct sigmoid_layer
 43 | 			this.sigmoid_layers[i] = new HiddenLayer(this.N, input_size, this.hidden_layer_sizes[i], null, null, rng);
 44 | 			
 45 | 			// construct rbm_layer
 46 | 			this.rbm_layers[i] = new RBM(this.N, input_size, this.hidden_layer_sizes[i], this.sigmoid_layers[i].W, this.sigmoid_layers[i].b, null, rng);
 47 | 		}
 48 | 		
 49 | 		// layer for output using LogisticRegression
 50 | 		this.log_layer = new LogisticRegression(this.N, this.hidden_layer_sizes[this.n_layers-1], this.n_outs);
 51 | 	}
 52 | 	
 53 | 	public void pretrain(int[][] train_X, double lr, int k, int epochs) {
 54 | 		int[] layer_input = new int[0];
 55 | 		int prev_layer_input_size;
 56 | 		int[] prev_layer_input;
 57 | 				
 58 | 		for(int i=0; i<n_layers; i++) {  // layer-wise			
 59 | 			for(int epoch=0; epoch<epochs; epoch++) {  // training epochs
 60 | 				for(int n=0; n<N; n++) {  // input x1...xN
 61 | 					// layer input
 62 | 					for(int l=0; l<=i; l++) {
 63 | 						
 64 | 						if(l == 0) {
 65 | 							layer_input = new int[n_ins];
 66 | 							for(int j=0; j<n_ins; j++) layer_input[j] = train_X[n][j];
 67 | 						} else {
 68 | 							if(l == 1) prev_layer_input_size = n_ins;
 69 | 							else prev_layer_input_size = hidden_layer_sizes[l-2];
 70 | 							
 71 | 							prev_layer_input = new int[prev_layer_input_size];
 72 | 							for(int j=0; j<prev_layer_input_size; j++) prev_layer_input[j] = layer_input[j];
 73 | 							
 74 | 							layer_input = new int[hidden_layer_sizes[l-1]];
 75 | 							
 76 | 							sigmoid_layers[l-1].sample_h_given_v(prev_layer_input, layer_input);
 77 | 						}
 78 | 					}
 79 | 					
 80 | 					rbm_layers[i].contrastive_divergence(layer_input, lr, k);
 81 | 				}
 82 | 			}
 83 | 		}
 84 | 	}
 85 | 	
 86 | 	public void finetune(int[][] train_X, int[][] train_Y, double lr, int epochs) {
 87 | 		int[] layer_input = new int[0];
 88 | 		// int prev_layer_input_size;
 89 | 		int[] prev_layer_input = new int[0];
 90 | 		
 91 | 		for(int epoch=0; epoch<epochs; epoch++) {
 92 | 			for(int n=0; n<N; n++) {
 93 | 				
 94 | 				// layer input
 95 | 				for(int i=0; i<n_layers; i++) {
 96 | 					if(i == 0) {
 97 | 						prev_layer_input = new int[n_ins];
 98 | 						for(int j=0; j<n_ins; j++) prev_layer_input[j] = train_X[n][j];
 99 | 					} else {
100 | 						prev_layer_input = new int[hidden_layer_sizes[i-1]];
101 | 						for(int j=0; j<hidden_layer_sizes[i-1]; j++) prev_layer_input[j] = layer_input[j];
102 | 					}
103 | 					
104 | 					layer_input = new int[hidden_layer_sizes[i]];
105 | 					sigmoid_layers[i].sample_h_given_v(prev_layer_input, layer_input);
106 | 				}
107 | 				
108 | 				log_layer.train(layer_input, train_Y[n], lr);
109 | 			}
110 | 			// lr *= 0.95;
111 | 		}
112 | 	}
113 | 	
114 | 	public void predict(int[] x, double[] y) {
115 | 		double[] layer_input = new double[0];
116 | 		// int prev_layer_input_size;
117 | 		double[] prev_layer_input = new double[n_ins];
118 | 		for(int j=0; j<n_ins; j++) prev_layer_input[j] = x[j];
119 | 	
120 | 		double linear_output;
121 | 		
122 | 		
123 | 		// layer activation
124 | 		for(int i=0; i<n_layers; i++) {
125 | 			layer_input = new double[sigmoid_layers[i].n_out];
126 | 			
127 | 			for(int k=0; k<sigmoid_layers[i].n_out; k++) {
128 | 				linear_output = 0.0;
129 | 				
130 | 				for(int j=0; j<sigmoid_layers[i].n_in; j++) {
131 | 					linear_output += sigmoid_layers[i].W[k][j] * prev_layer_input[j];
132 | 				}
133 | 				linear_output += sigmoid_layers[i].b[k];
134 | 				layer_input[k] = sigmoid(linear_output);
135 | 			}
136 | 			
137 | 			if(i < n_layers-1) {
138 | 				prev_layer_input = new double[sigmoid_layers[i].n_out];
139 | 				for(int j=0; j<sigmoid_layers[i].n_out; j++) prev_layer_input[j] = layer_input[j];
140 | 			}
141 | 		}
142 | 		
143 | 		for(int i=0; i<log_layer.n_out; i++) {
144 | 			y[i] = 0;
145 | 			for(int j=0; j<log_layer.n_in; j++) {
146 | 				y[i] += log_layer.W[i][j] * layer_input[j];
147 | 			}
148 | 			y[i] += log_layer.b[i];
149 | 		}
150 | 		
151 | 		log_layer.softmax(y);
152 | 	}
153 | 	
154 | 	private static void test_dbn() {
155 | 		Random rng = new Random(123);
156 | 		
157 | 		double pretrain_lr = 0.1;
158 | 		int pretraining_epochs = 1000;
159 | 		int k = 1;
160 | 		double finetune_lr = 0.1;
161 | 		int finetune_epochs = 500;
162 | 		
163 | 		int train_N = 6;
164 | 		int test_N = 4;
165 | 		int n_ins = 6;
166 | 		int n_outs = 2;
167 | 		int[] hidden_layer_sizes = {3, 3};
168 | 		int n_layers = hidden_layer_sizes.length;
169 | 		
170 | 		// training data
171 | 		int[][] train_X = {
172 | 			{1, 1, 1, 0, 0, 0},
173 | 			{1, 0, 1, 0, 0, 0},
174 | 			{1, 1, 1, 0, 0, 0},
175 | 			{0, 0, 1, 1, 1, 0},
176 | 			{0, 0, 1, 1, 0, 0},
177 | 			{0, 0, 1, 1, 1, 0}
178 | 		};
179 | 		
180 | 		int[][] train_Y = {
181 | 			{1, 0},
182 | 			{1, 0},
183 | 			{1, 0},
184 | 			{0, 1},
185 | 			{0, 1},
186 | 			{0, 1},
187 | 		};
188 | 		
189 | 		
190 | 		// construct DBN
191 | 		DBN dbn = new DBN(train_N, n_ins, hidden_layer_sizes, n_outs, n_layers, rng);
192 | 		
193 | 		// pretrain
194 | 		dbn.pretrain(train_X, pretrain_lr, k, pretraining_epochs);
195 | 		
196 | 		// finetune
197 | 		dbn.finetune(train_X, train_Y, finetune_lr, finetune_epochs);
198 | 		
199 | 		
200 | 		// test data
201 | 		int[][] test_X = {
202 | 			{1, 1, 0, 0, 0, 0},
203 | 			{1, 1, 1, 1, 0, 0},
204 | 			{0, 0, 0, 1, 1, 0},
205 | 			{0, 0, 1, 1, 1, 0},
206 | 		};
207 | 		
208 | 		double[][] test_Y = new double[test_N][n_outs];
209 | 		
210 | 		// test
211 | 		for(int i=0; i<test_N; i++) {
212 | 			dbn.predict(test_X[i], test_Y[i]);
213 | 			for(int j=0; j<n_outs; j++) {
214 | 				System.out.print(test_Y[i][j] + " ");
215 | 			}
216 | 			System.out.println();
217 | 		}
218 | 	}
219 | 	
220 | 	public static void main(String[] args) {
221 | 		test_dbn();
222 | 	}
223 | }
224 | 


--------------------------------------------------------------------------------
/c/RBM.c:
--------------------------------------------------------------------------------
  1 | #include <stdio.h>
  2 | #include <stdlib.h>
  3 | #include <math.h>
  4 | #include "RBM.h"
  5 | #include "utils.h"
  6 | 
  7 | 
  8 | void test_rbm(void);
  9 | 
 10 | 
 11 | double uniform(double min, double max) {
 12 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;  
 13 | }
 14 | 
 15 | int binomial(int n, double p) {
 16 |   if(p < 0 || p > 1) return 0;
 17 | 
 18 |   int i;
 19 |   int c = 0;
 20 |   double r;
 21 | 
 22 |   for(i=0; i<n; i++) {
 23 |     r = rand() / (RAND_MAX + 1.0);
 24 |     if (r < p) c++;
 25 |   }
 26 | 
 27 |   return c;
 28 | }
 29 | 
 30 | double sigmoid(double x) {
 31 |   return 1.0 / (1.0 + exp(-x));
 32 | }
 33 | 
 34 | 
 35 | void RBM__construct(RBM* this, int N, int n_visible, int n_hidden, \
 36 |                     double **W, double *hbias, double *vbias) {
 37 |   int i, j;
 38 |   double a = 1.0 / n_visible;
 39 | 
 40 |   this->N = N;
 41 |   this->n_visible = n_visible;
 42 |   this->n_hidden = n_hidden;
 43 | 
 44 |   if(W == NULL) {
 45 |     this->W = (double **)malloc(sizeof(double*) * n_hidden);
 46 |     this->W[0] = (double *)malloc(sizeof(double) * n_visible * n_hidden);
 47 |     for(i=0; i<n_hidden; i++) this->W[i] = this->W[0] + i * n_visible;
 48 | 
 49 |     for(i=0; i<n_hidden; i++) {
 50 |       for(j=0; j<n_visible; j++) {
 51 |         this->W[i][j] = uniform(-a, a);
 52 |       }
 53 |     }
 54 |   } else {
 55 |     this->W = W;
 56 |   }
 57 | 
 58 |   if(hbias == NULL) {
 59 |     this->hbias = (double *)malloc(sizeof(double) * n_hidden);
 60 |     for(i=0; i<n_hidden; i++) this->hbias[i] = 0;
 61 |   } else {
 62 |     this->hbias = hbias;
 63 |   }
 64 | 
 65 |   if(vbias == NULL) {
 66 |     this->vbias = (double *)malloc(sizeof(double) * n_visible);
 67 |     for(i=0; i<n_visible; i++) this->vbias[i] = 0;
 68 |   } else {
 69 |     this->vbias = vbias;
 70 |   }
 71 | }
 72 | 
 73 | void RBM__destruct(RBM* this) {
 74 |   free(this->W[0]);
 75 |   free(this->W);
 76 |   free(this->hbias);
 77 |   free(this->vbias);
 78 | }
 79 | 
 80 | void RBM_contrastive_divergence(RBM* this, int *input, double lr, int k) {
 81 |   int i, j, step;
 82 |   
 83 |   double *ph_mean = (double *)malloc(sizeof(double) * this->n_hidden);
 84 |   int *ph_sample = (int *)malloc(sizeof(int) * this->n_hidden);
 85 |   double *nv_means = (double *)malloc(sizeof(double) * this->n_visible);
 86 |   int *nv_samples = (int *)malloc(sizeof(int) * this->n_visible);
 87 |   double *nh_means = (double *)malloc(sizeof(double) * this->n_hidden);
 88 |   int *nh_samples = (int *)malloc(sizeof(int) * this->n_hidden);
 89 | 
 90 |   /* CD-k */
 91 |   RBM_sample_h_given_v(this, input, ph_mean, ph_sample);
 92 | 
 93 |   for(step=0; step<k; step++) {
 94 |     if(step == 0) {
 95 |       RBM_gibbs_hvh(this, ph_sample, nv_means, nv_samples, nh_means, nh_samples);
 96 |     } else {
 97 |       RBM_gibbs_hvh(this, nh_samples, nv_means, nv_samples, nh_means, nh_samples);
 98 |     }
 99 |   }
100 | 
101 |   for(i=0; i<this->n_hidden; i++) {
102 |     for(j=0; j<this->n_visible; j++) {
103 |       // this->W[i][j] += lr * (ph_sample[i] * input[j] - nh_means[i] * nv_samples[j]) / this->N;
104 |       this->W[i][j] += lr * (ph_mean[i] * input[j] - nh_means[i] * nv_samples[j]) / this->N;
105 |     }
106 |     this->hbias[i] += lr * (ph_sample[i] - nh_means[i]) / this->N;
107 |   }
108 | 
109 |   for(i=0; i<this->n_visible; i++) {
110 |     this->vbias[i] += lr * (input[i] - nv_samples[i]) / this->N;
111 |   }
112 |   
113 | 
114 |   free(ph_mean);
115 |   free(ph_sample);
116 |   free(nv_means);
117 |   free(nv_samples);
118 |   free(nh_means);
119 |   free(nh_samples);
120 | }
121 | 
122 | 
123 | void RBM_sample_h_given_v(RBM* this, int *v0_sample, double *mean, int *sample) {
124 |   int i;
125 |   for(i=0; i<this->n_hidden; i++) {
126 |     mean[i] = RBM_propup(this, v0_sample, this->W[i], this->hbias[i]);
127 |     sample[i] = binomial(1, mean[i]);
128 |   }
129 | }
130 | 
131 | void RBM_sample_v_given_h(RBM* this, int *h0_sample, double *mean, int *sample) {
132 |   int i;
133 |   for(i=0; i<this->n_visible; i++) {
134 |     mean[i] = RBM_propdown(this, h0_sample, i, this->vbias[i]);
135 |     sample[i] = binomial(1, mean[i]);
136 |   }
137 | }
138 | 
139 | double RBM_propup(RBM* this, int *v, double *w, double b) {
140 |   int j;
141 |   double pre_sigmoid_activation = 0.0;
142 |   for(j=0; j<this->n_visible; j++) {
143 |     pre_sigmoid_activation += w[j] * v[j];
144 |   }
145 |   pre_sigmoid_activation += b;
146 |   return sigmoid(pre_sigmoid_activation);
147 | }
148 | 
149 | double RBM_propdown(RBM* this, int *h, int i, double b) {
150 |   int j;
151 |   double pre_sigmoid_activation = 0.0;
152 | 
153 |   for(j=0; j<this->n_hidden; j++) {
154 |     pre_sigmoid_activation += this->W[j][i] * h[j];
155 |   }
156 |   pre_sigmoid_activation += b;
157 |   return sigmoid(pre_sigmoid_activation);
158 | }
159 | 
160 | void RBM_gibbs_hvh(RBM* this, int *h0_sample, double *nv_means, int *nv_samples, \
161 |                    double *nh_means, int *nh_samples) {
162 |   RBM_sample_v_given_h(this, h0_sample, nv_means, nv_samples);
163 |   RBM_sample_h_given_v(this, nv_samples, nh_means, nh_samples);
164 | }
165 | 
166 | void RBM_reconstruct(RBM* this, int *v, double *reconstructed_v) {
167 |   int i, j;
168 |   double *h = (double *)malloc(sizeof(double) * this->n_hidden);
169 |   double pre_sigmoid_activation;
170 | 
171 |   for(i=0; i<this->n_hidden; i++) {
172 |     h[i] = RBM_propup(this, v, this->W[i], this->hbias[i]);
173 |   }
174 | 
175 |   for(i=0; i<this->n_visible; i++) {
176 |     pre_sigmoid_activation = 0.0;
177 |     for(j=0; j<this->n_hidden; j++) {
178 |       pre_sigmoid_activation += this->W[j][i] * h[j];
179 |     }
180 |     pre_sigmoid_activation += this->vbias[i];
181 | 
182 |     reconstructed_v[i] = sigmoid(pre_sigmoid_activation);
183 |   }
184 | 
185 |   free(h);
186 | }
187 | 
188 | 
189 | 
190 | 
191 | void test_rbm(void) {
192 |   srand(0);
193 | 
194 |   int i, j, epoch;
195 | 
196 |   double learning_rate = 0.1;
197 |   int training_epochs = 1000;
198 |   int k = 1;
199 |   
200 |   int train_N = 6;
201 |   int test_N = 2;
202 |   int n_visible = 6;
203 |   int n_hidden = 3;
204 | 
205 |   // training data
206 |   int train_X[6][6] = {
207 |     {1, 1, 1, 0, 0, 0},
208 |     {1, 0, 1, 0, 0, 0},
209 |     {1, 1, 1, 0, 0, 0},
210 |     {0, 0, 1, 1, 1, 0},
211 |     {0, 0, 1, 0, 1, 0},
212 |     {0, 0, 1, 1, 1, 0}
213 |   };
214 | 
215 |   // construct RBM
216 |   RBM rbm;
217 |   RBM__construct(&rbm, train_N, n_visible, n_hidden, NULL, NULL, NULL);
218 | 
219 |   // train
220 |   for(epoch=0; epoch<training_epochs; epoch++) {
221 |     for(i=0; i<train_N; i++) {
222 |       RBM_contrastive_divergence(&rbm, train_X[i], learning_rate, k);
223 |     }
224 |   }
225 | 
226 | 
227 |   // test data
228 |   int test_X[2][6] = {
229 |     {1, 1, 0, 0, 0, 0},
230 |     {0, 0, 0, 1, 1, 0}
231 |   };
232 |   double reconstructed_X[2][6];
233 | 
234 |   // test
235 |   for(i=0; i<test_N; i++) {
236 |     RBM_reconstruct(&rbm, test_X[i], reconstructed_X[i]);
237 |     for(j=0; j<n_visible; j++) {
238 |       printf("%.5f ", reconstructed_X[i][j]);
239 |     }
240 |     printf("\n");
241 |   }
242 | 
243 | 
244 |   // destruct RBM
245 |   RBM__destruct(&rbm);
246 | }
247 | 
248 | 
249 | 
250 | int main(void) {
251 |   test_rbm();
252 |   
253 |   return 0;
254 | }
255 | 


--------------------------------------------------------------------------------
/scala/DBN.scala:
--------------------------------------------------------------------------------
  1 | import scala.util.Random
  2 | import scala.math
  3 | 
  4 | class DBN(val N: Int, val n_ins: Int, hidden_layer_sizes: Array[Int], val n_outs: Int, val n_layers: Int, var rng: Random=null) {
  5 | 
  6 |   def sigmoid(x: Double): Double = {
  7 |     return 1.0 / (1.0 + math.pow(math.E, -x))
  8 |   }
  9 | 
 10 | 
 11 |   var input_size: Int = 0
 12 |   
 13 |   val sigmoid_layers: Array[HiddenLayer] = new Array[HiddenLayer](n_layers)
 14 |   val rbm_layers: Array[RBM] = new Array[RBM](n_layers)
 15 | 
 16 |   if(rng == null) rng = new Random(1234)
 17 | 
 18 |   var i: Int = 0
 19 |   // construct multi-layer
 20 |   for(i <- 0 until n_layers) {
 21 |     if(i == 0) {
 22 |       input_size = n_ins
 23 |     } else {
 24 |       input_size = hidden_layer_sizes(i-1)
 25 |     }
 26 | 
 27 |     // construct sigmoid_layer
 28 |     sigmoid_layers(i) = new HiddenLayer(N, input_size, hidden_layer_sizes(i), null, null, rng)
 29 | 
 30 |     // construct rbm_layer
 31 |     rbm_layers(i) = new RBM(N, input_size, hidden_layer_sizes(i), sigmoid_layers(i).W, sigmoid_layers(i).b, null, rng)
 32 | 
 33 |   }
 34 | 
 35 |   // layer for output using LogisticRegression
 36 |   val log_layer: LogisticRegression = new LogisticRegression(N, hidden_layer_sizes(n_layers-1), n_outs)
 37 | 
 38 | 
 39 |   def pretrain(train_X: Array[Array[Int]], lr: Double, k: Int, epochs: Int) {
 40 |     var layer_input: Array[Int] = new Array[Int](0)
 41 |     var prev_layer_input_size: Int = 0
 42 |     var prev_layer_input: Array[Int] = new Array[Int](0)
 43 |     
 44 |     var i: Int = 0
 45 |     var j: Int = 0
 46 |     var epoch: Int = 0
 47 |     var n: Int = 0
 48 |     var l: Int = 0
 49 | 
 50 |     for(i <- 0 until n_layers) {  // layer-wise
 51 |       for(epoch <- 0 until epochs) {  // training epochs
 52 |         for(n <- 0 until N) {  // input x1...xN
 53 |           // layer input
 54 |           for(l <- 0 to i) {
 55 |             if(l == 0) {
 56 |               layer_input = new Array[Int](n_ins)
 57 |               for(j <- 0 until n_ins) layer_input(j) = train_X(n)(j)
 58 | 
 59 |             } else {
 60 |               if(l == 1) prev_layer_input_size = n_ins
 61 |               else prev_layer_input_size = hidden_layer_sizes(l-2)
 62 | 
 63 |               prev_layer_input = new Array[Int](prev_layer_input_size)
 64 |               for(j <- 0 until prev_layer_input_size) prev_layer_input(j) = layer_input(j)
 65 | 
 66 |               layer_input = new Array[Int](hidden_layer_sizes(l-1))
 67 |               sigmoid_layers(l-1).sample_h_given_v(prev_layer_input, layer_input)
 68 |             }
 69 |           }
 70 | 
 71 |           rbm_layers(i).contrastive_divergence(layer_input, lr, k)
 72 |         }
 73 |       }
 74 |     }
 75 |   }
 76 | 
 77 | 
 78 |   def finetune(train_X: Array[Array[Int]], train_Y: Array[Array[Int]], lr: Double, epochs: Int) {
 79 |     var layer_input: Array[Int] = new Array[Int](0)
 80 |     var prev_layer_input: Array[Int] = new Array[Int](0)
 81 | 
 82 |     var epoch: Int = 0
 83 |     var n: Int = 0
 84 |     var i: Int = 0
 85 |     var j: Int = 0
 86 | 
 87 |     for(epoch <- 0 until epochs) {
 88 |       for(n <- 0 until N) {
 89 |         
 90 |         // layer input
 91 |         for(i <- 0 until n_layers) {
 92 |           if(i == 0) {
 93 |             prev_layer_input = new Array[Int](n_ins)
 94 |             for(j <- 0 until n_ins) prev_layer_input(j) = train_X(n)(j)
 95 |           } else {
 96 |             prev_layer_input = new Array[Int](hidden_layer_sizes(i-1))
 97 |             for(j <- 0 until hidden_layer_sizes(i-1)) prev_layer_input(j) = layer_input(j)
 98 |           }
 99 | 
100 |           layer_input = new Array[Int](hidden_layer_sizes(i))
101 |           sigmoid_layers(i).sample_h_given_v(prev_layer_input, layer_input)
102 |         }
103 | 
104 |         log_layer.train(layer_input, train_Y(n), lr)
105 |       }
106 |       // lr *= 0.95
107 |     }
108 |   }
109 | 
110 |   def predict(x: Array[Int], y: Array[Double]) {
111 |     var layer_input: Array[Double] = new Array[Double](0)
112 |     var prev_layer_input: Array[Double] = new Array[Double](n_ins)
113 | 
114 |     var i: Int = 0
115 |     var j: Int = 0
116 |     var k: Int = 0
117 | 
118 |     for(j <- 0 until n_ins) prev_layer_input(j) = x(j)
119 |     
120 |     var linear_outoput: Double = 0
121 | 
122 |     // layer activation
123 |     for(i <- 0 until n_layers) {
124 |       layer_input = new Array[Double](sigmoid_layers(i).n_out)
125 | 
126 |       for(k <- 0 until sigmoid_layers(i).n_out) {
127 |         linear_outoput = 0.0
128 | 
129 |         for(j <- 0 until sigmoid_layers(i).n_in) {
130 |           linear_outoput += sigmoid_layers(i).W(k)(j) * prev_layer_input(j)
131 |         }
132 |         linear_outoput += sigmoid_layers(i).b(k)
133 |         layer_input(k) = sigmoid(linear_outoput)
134 |       }
135 |       
136 |       if(i < n_layers-1) {
137 |         prev_layer_input = new Array[Double](sigmoid_layers(i).n_out)
138 |         for(j <- 0 until sigmoid_layers(i).n_out) prev_layer_input(j) = layer_input(j)
139 |       }
140 |     }
141 | 
142 |     for(i <- 0 until log_layer.n_out) {
143 |       y(i) = 0
144 |       for(j <- 0 until log_layer.n_in) {
145 |         y(i) += log_layer.W(i)(j) * layer_input(j)
146 |       }
147 |       y(i) += log_layer.b(i)
148 |     }
149 | 
150 |     log_layer.softmax(y)
151 |   }
152 | 
153 | }
154 | 
155 | 
156 | object DBN {
157 |   def test_dbn() {
158 |     val rng: Random = new Random(123)
159 | 
160 |     val pretrain_lr: Double = 0.1
161 |     val pretraining_epochs: Int = 1000
162 |     val k: Int = 1
163 |     val finetune_lr: Double = 0.1
164 |     val finetune_epochs: Int = 500
165 |     
166 |     val train_N: Int = 6
167 |     val test_N: Int = 4
168 |     val n_ins: Int = 6
169 |     val n_outs: Int = 2
170 |     val hidden_layer_sizes: Array[Int] = Array(3, 3)
171 |     val n_layers = hidden_layer_sizes.length
172 | 
173 | 
174 |     // training data
175 |     val train_X: Array[Array[Int]] = Array(
176 | 			Array(1, 1, 1, 0, 0, 0),
177 | 			Array(1, 0, 1, 0, 0, 0),
178 | 		  Array(1, 1, 1, 0, 0, 0),
179 | 			Array(0, 0, 1, 1, 1, 0),
180 | 			Array(0, 0, 1, 1, 0, 0),
181 | 			Array(0, 0, 1, 1, 1, 0)
182 |     )
183 | 
184 |     val train_Y: Array[Array[Int]] = Array(
185 | 			Array(1, 0),
186 | 			Array(1, 0),
187 | 			Array(1, 0),
188 | 			Array(0, 1),
189 | 			Array(0, 1),
190 | 			Array(0, 1)
191 |     )
192 | 
193 |     // construct DBN
194 |     val dbn: DBN = new DBN(train_N, n_ins, hidden_layer_sizes, n_outs, n_layers, rng)
195 | 
196 | 		// pretrain
197 | 		dbn.pretrain(train_X, pretrain_lr, k, pretraining_epochs);
198 | 		
199 | 		// finetune
200 | 		dbn.finetune(train_X, train_Y, finetune_lr, finetune_epochs);
201 | 		
202 | 		
203 | 		// test data
204 | 		val test_X: Array[Array[Int]] = Array(
205 | 			Array(1, 1, 0, 0, 0, 0),
206 | 			Array(1, 1, 1, 1, 0, 0),
207 | 			Array(0, 0, 0, 1, 1, 0),
208 | 			Array(0, 0, 1, 1, 1, 0)
209 | 		)
210 | 
211 |     val test_Y: Array[Array[Double]] = Array.ofDim[Double](test_N, n_outs)
212 | 
213 |     var i: Int = 0
214 |     var j: Int = 0
215 | 
216 |     // test
217 |     for(i <- 0 until test_N) {
218 |       dbn.predict(test_X(i), test_Y(i))
219 |       for(j <- 0 until n_outs) {
220 |         print(test_Y(i)(j) + " ")
221 |       }
222 |       println()
223 |     }
224 |     
225 |   }
226 |   
227 | 
228 |   def main(args: Array[String]) {
229 |     test_dbn()
230 |   }
231 | }
232 | 


--------------------------------------------------------------------------------
/java/SdA/src/SdA.java:
--------------------------------------------------------------------------------
  1 | import java.util.Random;
  2 | 
  3 | public class SdA {
  4 | 	public int N;
  5 | 	public int n_ins;
  6 | 	public int[] hidden_layer_sizes;
  7 | 	public int n_outs;
  8 | 	public int n_layers;
  9 | 	public HiddenLayer[] sigmoid_layers;
 10 | 	public dA[] dA_layers;
 11 | 	public LogisticRegression log_layer;
 12 | 	public Random rng;
 13 | 
 14 | 	public static double sigmoid(double x) {
 15 | 		return 1.0 / (1.0 + Math.pow(Math.E, -x));
 16 | 	}
 17 | 	
 18 | 	public SdA(int N, int n_ins, int[] hidden_layer_sizes, int n_outs, int n_layers, Random rng) {
 19 | 		int input_size;
 20 | 		
 21 | 		this.N = N;
 22 | 		this.n_ins = n_ins;
 23 | 		this.hidden_layer_sizes = hidden_layer_sizes;
 24 | 		this.n_outs = n_outs;
 25 | 		this.n_layers = n_layers;
 26 | 		
 27 | 		this.sigmoid_layers = new HiddenLayer[n_layers];
 28 | 		this.dA_layers = new dA[n_layers];
 29 | 
 30 | 		if(rng == null)	this.rng = new Random(1234);
 31 | 		else this.rng = rng;		
 32 | 		
 33 | 		// construct multi-layer
 34 | 		for(int i=0; i<this.n_layers; i++) {
 35 | 			if(i == 0) {
 36 | 				input_size = this.n_ins;
 37 | 			} else {
 38 | 				input_size = this.hidden_layer_sizes[i-1];
 39 | 			}
 40 | 			
 41 | 			// construct sigmoid_layer
 42 | 			this.sigmoid_layers[i] = new HiddenLayer(this.N, input_size, this.hidden_layer_sizes[i], null, null, rng);
 43 | 			
 44 | 			// construct dA_layer
 45 | 			this.dA_layers[i] = new dA(this.N, input_size, this.hidden_layer_sizes[i], this.sigmoid_layers[i].W, this.sigmoid_layers[i].b, null, rng);
 46 | 		}
 47 | 		
 48 | 		// layer for output using LogisticRegression
 49 | 		this.log_layer = new LogisticRegression(this.N, this.hidden_layer_sizes[this.n_layers-1], this.n_outs);
 50 | 	}
 51 | 	
 52 | 	public void pretrain(int[][] train_X, double lr, double corruption_level, int epochs) {
 53 | 		int[] layer_input = new int[0];
 54 | 		int prev_layer_input_size;
 55 | 		int[] prev_layer_input;
 56 | 				
 57 | 		for(int i=0; i<n_layers; i++) {  // layer-wise			
 58 | 			for(int epoch=0; epoch<epochs; epoch++) {  // training epochs
 59 | 				for(int n=0; n<N; n++) {  // input x1...xN
 60 | 					// layer input
 61 | 					for(int l=0; l<=i; l++) {
 62 | 						
 63 | 						if(l == 0) {
 64 | 							layer_input = new int[n_ins];
 65 | 							for(int j=0; j<n_ins; j++) layer_input[j] = train_X[n][j];
 66 | 						} else {
 67 | 							if(l == 1) prev_layer_input_size = n_ins;
 68 | 							else prev_layer_input_size = hidden_layer_sizes[l-2];
 69 | 							
 70 | 							prev_layer_input = new int[prev_layer_input_size];
 71 | 							for(int j=0; j<prev_layer_input_size; j++) prev_layer_input[j] = layer_input[j];
 72 | 							
 73 | 							layer_input = new int[hidden_layer_sizes[l-1]];
 74 | 							
 75 | 							sigmoid_layers[l-1].sample_h_given_v(prev_layer_input, layer_input);
 76 | 						}
 77 | 					}
 78 | 					
 79 | 					dA_layers[i].train(layer_input, lr, corruption_level);
 80 | 				}
 81 | 			}
 82 | 		}
 83 | 	}
 84 | 		
 85 | 	public void finetune(int[][] train_X, int[][] train_Y, double lr, int epochs) {
 86 | 		int[] layer_input = new int[0];
 87 | 		// int prev_layer_input_size;
 88 | 		int[] prev_layer_input = new int[0];
 89 | 		
 90 | 		for(int epoch=0; epoch<epochs; epoch++) {
 91 | 			for(int n=0; n<N; n++) {
 92 | 				
 93 | 				// layer input
 94 | 				for(int i=0; i<n_layers; i++) {
 95 | 					if(i == 0) {
 96 | 						prev_layer_input = new int[n_ins];
 97 | 						for(int j=0; j<n_ins; j++) prev_layer_input[j] = train_X[n][j];
 98 | 					} else {
 99 | 						prev_layer_input = new int[hidden_layer_sizes[i-1]];
100 | 						for(int j=0; j<hidden_layer_sizes[i-1]; j++) prev_layer_input[j] = layer_input[j];
101 | 					}
102 | 					
103 | 					layer_input = new int[hidden_layer_sizes[i]];
104 | 					sigmoid_layers[i].sample_h_given_v(prev_layer_input, layer_input);
105 | 				}
106 | 				
107 | 				log_layer.train(layer_input, train_Y[n], lr);
108 | 			}
109 | 			// lr *= 0.95;
110 | 		}
111 | 	}
112 | 	
113 | 	public void predict(int[] x, double[] y) {
114 | 		double[] layer_input = new double[0];
115 | 		// int prev_layer_input_size;
116 | 		double[] prev_layer_input = new double[n_ins];
117 | 		for(int j=0; j<n_ins; j++) prev_layer_input[j] = x[j];
118 | 	
119 | 		double linear_output;
120 | 		
121 | 		
122 | 		// layer activation
123 | 		for(int i=0; i<n_layers; i++) {
124 | 			layer_input = new double[sigmoid_layers[i].n_out];
125 | 			
126 | 			for(int k=0; k<sigmoid_layers[i].n_out; k++) {
127 | 				linear_output = 0.0;
128 | 				
129 | 				for(int j=0; j<sigmoid_layers[i].n_in; j++) {
130 | 					linear_output += sigmoid_layers[i].W[k][j] * prev_layer_input[j];
131 | 				}
132 | 				linear_output += sigmoid_layers[i].b[k];
133 | 				layer_input[k] = sigmoid(linear_output);
134 | 			}
135 | 			
136 | 			if(i < n_layers-1) {
137 | 				prev_layer_input = new double[sigmoid_layers[i].n_out];
138 | 				for(int j=0; j<sigmoid_layers[i].n_out; j++) prev_layer_input[j] = layer_input[j];
139 | 			}
140 | 		}
141 | 		
142 | 		for(int i=0; i<log_layer.n_out; i++) {
143 | 			y[i] = 0;
144 | 			for(int j=0; j<log_layer.n_in; j++) {
145 | 				y[i] += log_layer.W[i][j] * layer_input[j];
146 | 			}
147 | 			y[i] += log_layer.b[i];
148 | 		}
149 | 		
150 | 		log_layer.softmax(y);
151 | 	}
152 | 	
153 | 
154 | 	private static void test_sda() {
155 | 		Random rng = new Random(123);
156 | 		
157 | 		double pretrain_lr = 0.1;
158 | 		double corruption_level = 0.3;
159 | 		int pretraining_epochs = 1000;
160 | 		double finetune_lr = 0.1;
161 | 		int finetune_epochs = 500;
162 | 
163 | 		int train_N = 10;
164 | 		int test_N = 4;
165 | 		int n_ins = 28;
166 | 		int n_outs = 2;
167 | 		int[] hidden_layer_sizes = {15, 15};
168 | 		int n_layers = hidden_layer_sizes.length;
169 | 		
170 | 		// training data
171 | 		int[][] train_X = {
172 | 			{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
173 | 			{0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
174 | 			{1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
175 | 			{0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
176 | 			{1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
177 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
178 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1},
179 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1},
180 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1},
181 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1}
182 | 		};
183 | 
184 | 		int[][] train_Y = {
185 | 			{1, 0},
186 | 			{1, 0},
187 | 			{1, 0},
188 | 			{1, 0},
189 | 			{1, 0},
190 | 			{0, 1},
191 | 			{0, 1},
192 | 			{0, 1},
193 | 			{0, 1},
194 | 			{0, 1}
195 | 		};
196 | 		
197 | 		// construct SdA
198 | 		SdA sda = new SdA(train_N, n_ins, hidden_layer_sizes, n_outs, n_layers, rng);
199 | 		
200 | 		// pretrain
201 | 		sda.pretrain(train_X, pretrain_lr, corruption_level, pretraining_epochs);
202 | 		
203 | 		// finetune
204 | 		sda.finetune(train_X, train_Y, finetune_lr, finetune_epochs);
205 | 		
206 | 
207 | 		// test data
208 | 		int[][] test_X = {
209 | 			{1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
210 | 			{1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
211 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1},
212 | 			{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1}
213 | 		};
214 | 		
215 | 		double[][] test_Y = new double[test_N][n_outs];
216 | 		
217 | 		// test
218 | 		for(int i=0; i<test_N; i++) {
219 | 			sda.predict(test_X[i], test_Y[i]);
220 | 			for(int j=0; j<n_outs; j++) {
221 | 				System.out.print(test_Y[i][j] + " ");
222 | 			}
223 | 			System.out.println();
224 | 		}
225 | 	}
226 | 	
227 | 	public static void main(String[] args) {
228 | 		test_sda();
229 | 	}
230 | }
231 | 


--------------------------------------------------------------------------------
/scala/SdA.scala:
--------------------------------------------------------------------------------
  1 | import scala.util.Random
  2 | import scala.math
  3 | 
  4 | class SdA(val N: Int, val n_ins: Int, hidden_layer_sizes: Array[Int], val n_outs: Int, val n_layers:Int, var rng: Random=null) {
  5 | 
  6 |   def sigmoid(x: Double): Double = {
  7 |     return 1.0 / (1.0 + math.pow(math.E, -x))
  8 |   }
  9 | 
 10 |   var input_size: Int = 0
 11 | 
 12 |   // var hidden_layer_sizes: Array[Int] = new Array[Int](n_layers)
 13 |   var sigmoid_layers: Array[HiddenLayer] = new Array[HiddenLayer](n_layers)
 14 |   var dA_layers: Array[dA] = new Array[dA](n_layers)
 15 | 
 16 |   if(rng == null) rng = new Random(1234)
 17 | 
 18 | 
 19 |   var i: Int = 0
 20 | 
 21 |   // construct multi-layer
 22 |   for(i <- 0 until n_layers) {
 23 |     if(i == 0) {
 24 |       input_size = n_ins
 25 |     } else {
 26 |       input_size = hidden_layer_sizes(i-1)
 27 |     }
 28 |     
 29 |     // construct sigmoid_layer
 30 |     sigmoid_layers(i) = new HiddenLayer(N, input_size, hidden_layer_sizes(i), null, null, rng)
 31 | 
 32 |     // construct dA_layer
 33 |     dA_layers(i) = new dA(N, input_size, hidden_layer_sizes(i), sigmoid_layers(i).W, sigmoid_layers(i).b, null, rng)
 34 |   }
 35 | 
 36 |   // layer for output using LogisticRegression
 37 |   val log_layer = new LogisticRegression(N, hidden_layer_sizes(n_layers-1), n_outs)
 38 | 
 39 | 
 40 |   def pretrain(train_X: Array[Array[Int]], lr: Double, corruption_level: Double, epochs: Int) {
 41 |     var layer_input: Array[Int] = new Array[Int](0)
 42 |     var prev_layer_input_size: Int = 0
 43 |     var prev_layer_input: Array[Int] = new Array[Int](0)
 44 | 
 45 |     var i: Int = 0
 46 |     var j: Int = 0
 47 |     var epoch: Int = 0
 48 |     var n: Int = 0
 49 |     var l: Int = 0
 50 | 
 51 |     for(i <- 0 until n_layers) {  // layer-wise
 52 |       for(epoch <- 0 until epochs) {  // training epochs
 53 |         for(n <- 0 until N) {  // input x1...xN
 54 |           // layer input
 55 |           for(l <- 0 to i) {
 56 |             if(l == 0) {
 57 |               layer_input = new Array[Int](n_ins)
 58 |               for(j <- 0 until n_ins) layer_input(j) = train_X(n)(j)
 59 |             } else {
 60 |               if(l == 1) prev_layer_input_size = n_ins
 61 |               else prev_layer_input_size = hidden_layer_sizes(l-2)
 62 | 
 63 |               prev_layer_input = new Array[Int](prev_layer_input_size)
 64 |               for(j <- 0 until prev_layer_input_size) prev_layer_input(j) = layer_input(j)
 65 | 
 66 |               layer_input = new Array[Int](hidden_layer_sizes(l-1))
 67 |               
 68 |               sigmoid_layers(l-1).sample_h_given_v(prev_layer_input, layer_input)
 69 |             }
 70 |           }
 71 | 
 72 |           dA_layers(i).train(layer_input, lr, corruption_level)
 73 |         }
 74 |       }
 75 |     }
 76 |     
 77 |   }
 78 | 
 79 | 
 80 |   def finetune(train_X: Array[Array[Int]], train_Y: Array[Array[Int]], lr: Double, epochs: Int) {
 81 |     var layer_input: Array[Int] = new Array[Int](0)
 82 |     var prev_layer_input: Array[Int] = new Array[Int](0)
 83 | 
 84 |     var epoch: Int = 0
 85 |     var n: Int = 0
 86 |     
 87 |     
 88 |     for(epoch <- 0 until epochs) {
 89 |       for(n <- 0 until N) {
 90 |         
 91 |         // layer input
 92 |         for(i <- 0 until n_layers) {
 93 |           if(i == 0) {
 94 |             prev_layer_input = new Array[Int](n_ins)
 95 |             for(j <- 0 until n_ins) prev_layer_input(j) = train_X(n)(j)
 96 |           } else {
 97 |             prev_layer_input = new Array[Int](hidden_layer_sizes(i-1))
 98 |             for(j <- 0 until hidden_layer_sizes(i-1)) prev_layer_input(j) = layer_input(j)
 99 |           }
100 | 
101 |           layer_input = new Array[Int](hidden_layer_sizes(i))
102 |           sigmoid_layers(i).sample_h_given_v(prev_layer_input, layer_input)
103 |         }
104 | 
105 |         log_layer.train(layer_input, train_Y(n), lr)
106 |       }
107 |       // lr *= 0.95
108 |     }
109 |   }
110 | 
111 |   def predict(x: Array[Int], y: Array[Double]) {
112 |     var layer_input: Array[Double] = new Array[Double](0)
113 |     var prev_layer_input: Array[Double] = new Array[Double](n_ins)
114 |     
115 |     var j: Int = 0
116 |     for(j <- 0 until n_ins) prev_layer_input(j) = x(j)
117 | 
118 |     var linear_output: Double = 0.0
119 | 
120 |     // layer activation
121 |     var i: Int = 0
122 |     var k: Int = 0
123 | 
124 |     for(i <- 0 until n_layers) {
125 |       layer_input = new Array[Double](sigmoid_layers(i).n_out)
126 | 
127 |       for(k <- 0 until sigmoid_layers(i).n_out) {
128 |         linear_output = 0.0
129 | 
130 |         for(j <- 0 until sigmoid_layers(i).n_in) {
131 |           linear_output += sigmoid_layers(i).W(k)(j) * prev_layer_input(j)
132 |         }
133 |         linear_output += sigmoid_layers(i).b(k)
134 |         layer_input(k) = sigmoid(linear_output)
135 |       }
136 | 
137 |       if(i < n_layers-1) {
138 |         prev_layer_input = new Array[Double](sigmoid_layers(i).n_out)
139 |         for(j <- 0 until sigmoid_layers(i).n_out) prev_layer_input(j) = layer_input(j)
140 |       }
141 |     }
142 | 
143 |     for(i <- 0 until log_layer.n_out) {
144 |       y(i) = 0
145 |       for(j <- 0 until log_layer.n_in) {
146 |         y(i) += log_layer.W(i)(j) * layer_input(j)
147 |       }
148 |       y(i) += log_layer.b(i)
149 |     }
150 | 
151 |     log_layer.softmax(y)
152 |   }
153 | 
154 | }
155 | 
156 | 
157 | object SdA {
158 |   def test_sda() {
159 |     val rng: Random = new Random(123)
160 |     
161 |     val pretrain_lr: Double = 0.1
162 |     val corruption_level: Double = 0.3
163 |     val pretraining_epochs: Int = 1000
164 |     val finetune_lr: Double = 0.1
165 |     val finetune_epochs: Int = 500
166 | 
167 |     val train_N: Int = 10
168 |     val test_N: Int = 4
169 |     val n_ins: Int = 28
170 |     val n_outs: Int = 2
171 |     val hidden_layer_sizes: Array[Int] = Array(15, 15)
172 |     val n_layers: Int = hidden_layer_sizes.length
173 | 
174 |     // training data
175 |     val train_X: Array[Array[Int]] = Array(
176 | 			Array(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
177 | 			Array(0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
178 | 			Array(1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
179 | 			Array(0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
180 | 			Array(1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
181 | 			Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
182 | 			Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1),
183 | 			Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1),
184 | 			Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1),
185 | 			Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1)
186 |     )
187 | 
188 |     val train_Y: Array[Array[Int]] = Array(
189 | 			Array(1, 0),
190 | 			Array(1, 0),
191 | 			Array(1, 0),
192 | 			Array(1, 0),
193 | 			Array(1, 0),
194 | 			Array(0, 1),
195 | 			Array(0, 1),
196 | 			Array(0, 1),
197 | 			Array(0, 1),
198 | 			Array(0, 1)
199 |     )
200 |     
201 |     // construct SdA
202 |     val sda:SdA = new SdA(train_N, n_ins, hidden_layer_sizes, n_outs, n_layers, rng)
203 | 
204 |     // pretrain
205 |     sda.pretrain(train_X, pretrain_lr, corruption_level, pretraining_epochs)
206 | 
207 |     // finetune
208 |     sda.finetune(train_X, train_Y, finetune_lr, finetune_epochs)
209 |       
210 |     // test data
211 |     val test_X: Array[Array[Int]] = Array(
212 | 			Array(1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
213 | 			Array(1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
214 | 			Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1),
215 | 			Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1)
216 |     )
217 | 
218 |     val test_Y: Array[Array[Double]] = Array.ofDim[Double](test_N, n_outs)
219 | 
220 |     // test
221 |     var i: Int = 0
222 |     var j: Int = 0
223 | 
224 |     for(i <- 0 until test_N) {
225 |       sda.predict(test_X(i), test_Y(i))
226 |       for(j <- 0 until n_outs) {
227 |         print(test_Y(i)(j) + " ")
228 |       }
229 |       println()
230 |     }
231 |   }
232 | 
233 |   def main(args: Array[String]) {
234 |     test_sda()
235 |   }
236 | }
237 | 


--------------------------------------------------------------------------------
/cpp/DBN.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <math.h>
  3 | #include "HiddenLayer.h"
  4 | #include "RBM.h"
  5 | #include "LogisticRegression.h"
  6 | #include "DBN.h"
  7 | using namespace std;
  8 | 
  9 | 
 10 | double uniform(double min, double max) {
 11 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;
 12 | }
 13 | 
 14 | int binomial(int n, double p) {
 15 |   if(p < 0 || p > 1) return 0;
 16 |   
 17 |   int c = 0;
 18 |   double r;
 19 |   
 20 |   for(int i=0; i<n; i++) {
 21 |     r = rand() / (RAND_MAX + 1.0);
 22 |     if (r < p) c++;
 23 |   }
 24 | 
 25 |   return c;
 26 | }
 27 | 
 28 | double sigmoid(double x) {
 29 |   return 1.0 / (1.0 + exp(-x));
 30 | }
 31 | 
 32 | 
 33 | // DBN
 34 | DBN::DBN(int size, int n_i, int *hls, int n_o, int n_l) {
 35 |   int input_size;
 36 |   
 37 |   N = size;
 38 |   n_ins = n_i;
 39 |   hidden_layer_sizes = hls;
 40 |   n_outs = n_o;
 41 |   n_layers = n_l;
 42 | 
 43 |   sigmoid_layers = new HiddenLayer*[n_layers];
 44 |   rbm_layers = new RBM*[n_layers];
 45 | 
 46 |   // construct multi-layer
 47 |   for(int i=0; i<n_layers; i++) {
 48 |     if(i == 0) {
 49 |       input_size = n_ins;
 50 |     } else {
 51 |       input_size = hidden_layer_sizes[i-1];
 52 |     }
 53 | 
 54 |     // construct sigmoid_layer
 55 |     sigmoid_layers[i] = new HiddenLayer(N, input_size, hidden_layer_sizes[i], NULL, NULL);
 56 | 
 57 |     // construct rbm_layer
 58 |     rbm_layers[i] = new RBM(N, input_size, hidden_layer_sizes[i],\
 59 |                             sigmoid_layers[i]->W, sigmoid_layers[i]->b, NULL);
 60 |   }
 61 | 
 62 |   // layer for output using LogisticRegression
 63 |   log_layer = new LogisticRegression(N, hidden_layer_sizes[n_layers-1], n_outs);
 64 | }
 65 | 
 66 | DBN::~DBN() {
 67 |   delete log_layer;
 68 | 
 69 |   for(int i=0; i<n_layers; i++) {
 70 |     delete sigmoid_layers[i];
 71 |     delete rbm_layers[i];
 72 |   }
 73 |   delete[] sigmoid_layers;
 74 |   delete[] rbm_layers;
 75 | }
 76 | 
 77 | 
 78 | void DBN::pretrain(int *input, double lr, int k, int epochs) {
 79 |   int *layer_input;
 80 |   int prev_layer_input_size;
 81 |   int *prev_layer_input;
 82 | 
 83 |   int *train_X = new int[n_ins];
 84 | 
 85 |   for(int i=0; i<n_layers; i++) {  // layer-wise
 86 | 
 87 |     for(int epoch=0; epoch<epochs; epoch++) {  // training epochs
 88 | 
 89 |       for(int n=0; n<N; n++) { // input x1...xN
 90 |         // initial input
 91 |         for(int m=0; m<n_ins; m++) train_X[m] = input[n * n_ins + m];
 92 | 
 93 |         // layer input
 94 |         for(int l=0; l<=i; l++) {
 95 | 
 96 |           if(l == 0) {
 97 |             layer_input = new int[n_ins];
 98 |             for(int j=0; j<n_ins; j++) layer_input[j] = train_X[j];
 99 |           } else {
100 |             if(l == 1) prev_layer_input_size = n_ins;
101 |             else prev_layer_input_size = hidden_layer_sizes[l-2];
102 | 
103 |             prev_layer_input = new int[prev_layer_input_size];
104 |             for(int j=0; j<prev_layer_input_size; j++) prev_layer_input[j] = layer_input[j];
105 |             delete[] layer_input;
106 | 
107 |             layer_input = new int[hidden_layer_sizes[l-1]];
108 | 
109 |             sigmoid_layers[l-1]->sample_h_given_v(prev_layer_input, layer_input);
110 |             delete[] prev_layer_input;
111 |           }
112 |         }
113 | 
114 |         rbm_layers[i]->contrastive_divergence(layer_input, lr, k);
115 |       }
116 | 
117 |     }
118 |   }
119 | 
120 |   delete[] train_X;
121 |   delete[] layer_input;
122 | }
123 | 
124 | void DBN::finetune(int *input, int *label, double lr, int epochs) {
125 |   int *layer_input;
126 |   // int prev_layer_input_size;
127 |   int *prev_layer_input;
128 | 
129 |   int *train_X = new int[n_ins];
130 |   int *train_Y = new int[n_outs];
131 | 
132 |   for(int epoch=0; epoch<epochs; epoch++) {
133 |     for(int n=0; n<N; n++) { // input x1...xN
134 |       // initial input
135 |       for(int m=0; m<n_ins; m++)  train_X[m] = input[n * n_ins + m];
136 |       for(int m=0; m<n_outs; m++) train_Y[m] = label[n * n_outs + m];
137 | 
138 |       // layer input
139 |       for(int i=0; i<n_layers; i++) {
140 |         if(i == 0) {
141 |           prev_layer_input = new int[n_ins];
142 |           for(int j=0; j<n_ins; j++) prev_layer_input[j] = train_X[j];
143 |         } else {
144 |           prev_layer_input = new int[hidden_layer_sizes[i-1]];
145 |           for(int j=0; j<hidden_layer_sizes[i-1]; j++) prev_layer_input[j] = layer_input[j];
146 |           delete[] layer_input;
147 |         }
148 | 
149 | 
150 |         layer_input = new int[hidden_layer_sizes[i]];
151 |         sigmoid_layers[i]->sample_h_given_v(prev_layer_input, layer_input);
152 |         delete[] prev_layer_input;
153 |       }
154 | 
155 |       log_layer->train(layer_input, train_Y, lr);
156 |     }
157 |     // lr *= 0.95;
158 |   }
159 | 
160 |   delete[] layer_input;
161 |   delete[] train_X;
162 |   delete[] train_Y;
163 | }
164 | 
165 | void DBN::predict(int *x, double *y) {
166 |   double *layer_input;
167 |   // int prev_layer_input_size;
168 |   double *prev_layer_input;
169 | 
170 |   double linear_output;
171 | 
172 |   prev_layer_input = new double[n_ins];
173 |   for(int j=0; j<n_ins; j++) prev_layer_input[j] = x[j];
174 | 
175 |   // layer activation
176 |   for(int i=0; i<n_layers; i++) {
177 |     layer_input = new double[sigmoid_layers[i]->n_out];
178 | 
179 |     for(int k=0; k<sigmoid_layers[i]->n_out; k++) {
180 |       linear_output = 0.0;
181 | 
182 |       for(int j=0; j<sigmoid_layers[i]->n_in; j++) {
183 |         linear_output += sigmoid_layers[i]->W[k][j] * prev_layer_input[j];
184 |       }
185 |       linear_output += sigmoid_layers[i]->b[k];
186 |       layer_input[k] = sigmoid(linear_output);
187 |     }
188 |     delete[] prev_layer_input;
189 | 
190 |     if(i < n_layers-1) {
191 |       prev_layer_input = new double[sigmoid_layers[i]->n_out];
192 |       for(int j=0; j<sigmoid_layers[i]->n_out; j++) prev_layer_input[j] = layer_input[j];
193 |       delete[] layer_input;
194 |     }
195 |   }
196 |   
197 |   for(int i=0; i<log_layer->n_out; i++) {
198 |     y[i] = 0;
199 |     for(int j=0; j<log_layer->n_in; j++) {
200 |       y[i] += log_layer->W[i][j] * layer_input[j];
201 |     }
202 |     y[i] += log_layer->b[i];
203 |   }
204 |   
205 |   log_layer->softmax(y);
206 | 
207 | 
208 |   delete[] layer_input;
209 | }
210 | 
211 | 
212 | // HiddenLayer
213 | HiddenLayer::HiddenLayer(int size, int in, int out, double **w, double *bp) {
214 |   N = size;
215 |   n_in = in;
216 |   n_out = out;
217 | 
218 |   if(w == NULL) {
219 |     W = new double*[n_out];
220 |     for(int i=0; i<n_out; i++) W[i] = new double[n_in];
221 |     double a = 1.0 / n_in;
222 | 
223 |     for(int i=0; i<n_out; i++) {
224 |       for(int j=0; j<n_in; j++) {
225 |         W[i][j] = uniform(-a, a);
226 |       }
227 |     }
228 |   } else {
229 |     W = w;
230 |   }
231 | 
232 |   if(bp == NULL) {
233 |     b = new double[n_out];
234 |   } else {
235 |     b = bp;
236 |   }
237 | }
238 | 
239 | HiddenLayer::~HiddenLayer() {
240 |   for(int i=0; i<n_out; i++) delete W[i];
241 |   delete[] W;
242 |   delete[] b;
243 | }
244 | 
245 | double HiddenLayer::output(int *input, double *w, double b) {
246 |   double linear_output = 0.0;
247 |   for(int j=0; j<n_in; j++) {
248 |     linear_output += w[j] * input[j];
249 |   }
250 |   linear_output += b;
251 |   return sigmoid(linear_output);
252 | }
253 | 
254 | void HiddenLayer::sample_h_given_v(int *input, int *sample) {
255 |   for(int i=0; i<n_out; i++) {
256 |     sample[i] = binomial(1, output(input, W[i], b[i]));
257 |   }
258 | }
259 | 
260 | 
261 | // RBM 
262 | RBM::RBM(int size, int n_v, int n_h, double **w, double *hb, double *vb) {
263 |   N = size;
264 |   n_visible = n_v;
265 |   n_hidden = n_h;
266 | 
267 |   if(w == NULL) {
268 |     W = new double*[n_hidden];
269 |     for(int i=0; i<n_hidden; i++) W[i] = new double[n_visible];
270 |     double a = 1.0 / n_visible;
271 | 
272 |     for(int i=0; i<n_hidden; i++) {
273 |       for(int j=0; j<n_visible; j++) {
274 |         W[i][j] = uniform(-a, a);
275 |       }
276 |     }
277 |   } else {
278 |     W = w;
279 |   }
280 | 
281 |   if(hb == NULL) {
282 |     hbias = new double[n_hidden];
283 |     for(int i=0; i<n_hidden; i++) hbias[i] = 0;
284 |   } else {
285 |     hbias = hb;
286 |   }
287 | 
288 |   if(vb == NULL) {
289 |     vbias = new double[n_visible];
290 |     for(int i=0; i<n_visible; i++) vbias[i] = 0;
291 |   } else {
292 |     vbias = vb;
293 |   }
294 | }
295 | 
296 | RBM::~RBM() {
297 |   // for(int i=0; i<n_hidden; i++) delete[] W[i];
298 |   // delete[] W;
299 |   // delete[] hbias;
300 |   delete[] vbias;
301 | }
302 | 
303 | 
304 | void RBM::contrastive_divergence(int *input, double lr, int k) {
305 |   double *ph_mean = new double[n_hidden];
306 |   int *ph_sample = new int[n_hidden];
307 |   double *nv_means = new double[n_visible];
308 |   int *nv_samples = new int[n_visible];
309 |   double *nh_means = new double[n_hidden];
310 |   int *nh_samples = new int[n_hidden];
311 | 
312 |   /* CD-k */
313 |   sample_h_given_v(input, ph_mean, ph_sample);
314 | 
315 |   for(int step=0; step<k; step++) {
316 |     if(step == 0) {
317 |       gibbs_hvh(ph_sample, nv_means, nv_samples, nh_means, nh_samples);
318 |     } else {
319 |       gibbs_hvh(nh_samples, nv_means, nv_samples, nh_means, nh_samples);
320 |     }
321 |   }
322 | 
323 |   for(int i=0; i<n_hidden; i++) {
324 |     for(int j=0; j<n_visible; j++) {
325 |       // W[i][j] += lr * (ph_sample[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
326 |       W[i][j] += lr * (ph_mean[i] * input[j] - nh_means[i] * nv_samples[j]) / N;
327 |     }
328 |     hbias[i] += lr * (ph_sample[i] - nh_means[i]) / N;
329 |   }
330 | 
331 |   for(int i=0; i<n_visible; i++) {
332 |     vbias[i] += lr * (input[i] - nv_samples[i]) / N;
333 |   }
334 | 
335 |   delete[] ph_mean;
336 |   delete[] ph_sample;
337 |   delete[] nv_means;
338 |   delete[] nv_samples;
339 |   delete[] nh_means;
340 |   delete[] nh_samples;
341 | }
342 | 
343 | void RBM::sample_h_given_v(int *v0_sample, double *mean, int *sample) {
344 |   for(int i=0; i<n_hidden; i++) {
345 |     mean[i] = propup(v0_sample, W[i], hbias[i]);
346 |     sample[i] = binomial(1, mean[i]);
347 |   }
348 | }
349 | 
350 | void RBM::sample_v_given_h(int *h0_sample, double *mean, int *sample) {
351 |   for(int i=0; i<n_visible; i++) {
352 |     mean[i] = propdown(h0_sample, i, vbias[i]);
353 |     sample[i] = binomial(1, mean[i]);
354 |   }
355 | }
356 | 
357 | double RBM::propup(int *v, double *w, double b) {
358 |   double pre_sigmoid_activation = 0.0;
359 |   for(int j=0; j<n_visible; j++) {
360 |     pre_sigmoid_activation += w[j] * v[j];
361 |   }
362 |   pre_sigmoid_activation += b;
363 |   return sigmoid(pre_sigmoid_activation);
364 | }
365 | 
366 | double RBM::propdown(int *h, int i, double b) {
367 |   double pre_sigmoid_activation = 0.0;
368 |   for(int j=0; j<n_hidden; j++) {
369 |     pre_sigmoid_activation += W[j][i] * h[j];
370 |   }
371 |   pre_sigmoid_activation += b;
372 |   return sigmoid(pre_sigmoid_activation);
373 | }
374 | 
375 | void RBM::gibbs_hvh(int *h0_sample, double *nv_means, int *nv_samples, \
376 |                     double *nh_means, int *nh_samples) {
377 |   sample_v_given_h(h0_sample, nv_means, nv_samples);
378 |   sample_h_given_v(nv_samples, nh_means, nh_samples);
379 | }
380 | 
381 | void RBM::reconstruct(int *v, double *reconstructed_v) {
382 |   double *h = new double[n_hidden];
383 |   double pre_sigmoid_activation;
384 | 
385 |   for(int i=0; i<n_hidden; i++) {
386 |     h[i] = propup(v, W[i], hbias[i]);
387 |   }
388 | 
389 |   for(int i=0; i<n_visible; i++) {
390 |     pre_sigmoid_activation = 0.0;
391 |     for(int j=0; j<n_hidden; j++) {
392 |       pre_sigmoid_activation += W[j][i] * h[j];
393 |     }
394 |     pre_sigmoid_activation += vbias[i];
395 | 
396 |     reconstructed_v[i] = sigmoid(pre_sigmoid_activation);
397 |   }
398 | 
399 |   delete[] h;
400 | }
401 | 
402 | 
403 | // LogisticRegression
404 | LogisticRegression::LogisticRegression(int size, int in, int out) {
405 |   N = size;
406 |   n_in = in;
407 |   n_out = out;
408 | 
409 |   W = new double*[n_out];
410 |   for(int i=0; i<n_out; i++) W[i] = new double[n_in];
411 |   b = new double[n_out];
412 | 
413 |   for(int i=0; i<n_out; i++) {
414 |     for(int j=0; j<n_in; j++) {
415 |       W[i][j] = 0;
416 |     }
417 |     b[i] = 0;
418 |   }
419 | }
420 | 
421 | LogisticRegression::~LogisticRegression() {
422 |   for(int i=0; i<n_out; i++) delete[] W[i];
423 |   delete[] W;
424 |   delete[] b;
425 | }
426 | 
427 | 
428 | void LogisticRegression::train(int *x, int *y, double lr) {
429 |   double *p_y_given_x = new double[n_out];
430 |   double *dy = new double[n_out];
431 | 
432 |   for(int i=0; i<n_out; i++) {
433 |     p_y_given_x[i] = 0;
434 |     for(int j=0; j<n_in; j++) {
435 |       p_y_given_x[i] += W[i][j] * x[j];
436 |     }
437 |     p_y_given_x[i] += b[i];
438 |   }
439 |   softmax(p_y_given_x);
440 | 
441 |   for(int i=0; i<n_out; i++) {
442 |     dy[i] = y[i] - p_y_given_x[i];
443 | 
444 |     for(int j=0; j<n_in; j++) {
445 |       W[i][j] += lr * dy[i] * x[j] / N;
446 |     }
447 | 
448 |     b[i] += lr * dy[i] / N;
449 |   }
450 |   
451 |   delete[] p_y_given_x;
452 |   delete[] dy;
453 | }
454 | 
455 | void LogisticRegression::softmax(double *x) {
456 |   double max = 0.0;
457 |   double sum = 0.0;
458 |   
459 |   for(int i=0; i<n_out; i++) if(max < x[i]) max = x[i];
460 |   for(int i=0; i<n_out; i++) {
461 |     x[i] = exp(x[i] - max);
462 |     sum += x[i];
463 |   } 
464 | 
465 |   for(int i=0; i<n_out; i++) x[i] /= sum;
466 | }
467 | 
468 | void LogisticRegression::predict(int *x, double *y) {
469 |   for(int i=0; i<n_out; i++) {
470 |     y[i] = 0;
471 |     for(int j=0; j<n_in; j++) {
472 |       y[i] += W[i][j] * x[j];
473 |     }
474 |     y[i] += b[i];
475 |   }
476 | 
477 |   softmax(y);
478 | }
479 | 
480 | 
481 | 
482 | 
483 | 
484 | void test_dbn() {
485 |   srand(0);
486 | 
487 |   double pretrain_lr = 0.1;
488 |   int pretraining_epochs = 1000;
489 |   int k = 1;
490 |   double finetune_lr = 0.1;
491 |   int finetune_epochs = 500;
492 | 
493 |   int train_N = 6;
494 |   int test_N = 3;
495 |   int n_ins = 6;
496 |   int n_outs = 2;
497 |   int hidden_layer_sizes[] = {3, 3};
498 |   int n_layers = sizeof(hidden_layer_sizes) / sizeof(hidden_layer_sizes[0]);
499 | 
500 |   // training data
501 |   int train_X[6][6] = {
502 |     {1, 1, 1, 0, 0, 0},
503 |     {1, 0, 1, 0, 0, 0},
504 |     {1, 1, 1, 0, 0, 0},
505 |     {0, 0, 1, 1, 1, 0},
506 |     {0, 0, 1, 1, 0, 0},
507 |     {0, 0, 1, 1, 1, 0}
508 |   };
509 | 
510 |   int train_Y[6][2] = {
511 |     {1, 0},
512 |     {1, 0},
513 |     {1, 0},
514 |     {0, 1},
515 |     {0, 1},
516 |     {0, 1}
517 |   };
518 | 
519 | 
520 |   
521 |   // construct DBN
522 |   DBN dbn(train_N, n_ins, hidden_layer_sizes, n_outs, n_layers);
523 | 
524 |   // pretrain
525 |   dbn.pretrain(*train_X, pretrain_lr, k, pretraining_epochs);
526 | 
527 |   // finetune
528 |   dbn.finetune(*train_X, *train_Y, finetune_lr, finetune_epochs);
529 |   
530 | 
531 |   // test data
532 |   int test_X[3][6] = {
533 |     {1, 1, 0, 0, 0, 0},
534 |     {0, 0, 0, 1, 1, 0},
535 |     {1, 1, 1, 1, 1, 0}
536 |   };
537 | 
538 |   double test_Y[3][2];
539 | 
540 | 
541 |   // test
542 |   for(int i=0; i<test_N; i++) {
543 |     dbn.predict(test_X[i], test_Y[i]);
544 |     for(int j=0; j<n_outs; j++) {
545 |       cout << test_Y[i][j] << " ";
546 |     }
547 |     cout << endl;
548 |   }
549 | 
550 | }
551 | 
552 | 
553 | 
554 | 
555 | 
556 | int main() {
557 |   test_dbn();
558 |   return 0;
559 | }
560 | 


--------------------------------------------------------------------------------
/cpp/SdA.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <math.h>
  3 | #include "HiddenLayer.h"
  4 | #include "dA.h"
  5 | #include "LogisticRegression.h"
  6 | #include "SdA.h"
  7 | using namespace std;
  8 | 
  9 | double uniform(double min, double max) {
 10 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;
 11 | }
 12 | 
 13 | int binomial(int n, double p) {
 14 |   if(p < 0 || p > 1) return 0;
 15 |   
 16 |   int c = 0;
 17 |   double r;
 18 |   
 19 |   for(int i=0; i<n; i++) {
 20 |     r = rand() / (RAND_MAX + 1.0);
 21 |     if (r < p) c++;
 22 |   }
 23 | 
 24 |   return c;
 25 | }
 26 | 
 27 | double sigmoid(double x) {
 28 |   return 1.0 / (1.0 + exp(-x));
 29 | }
 30 | 
 31 | 
 32 | // SdA
 33 | SdA::SdA(int size, int n_i, int *hls, int n_o, int n_l) {
 34 |   int input_size;
 35 | 
 36 |   N = size;
 37 |   n_ins = n_i;
 38 |   hidden_layer_sizes = hls;
 39 |   n_outs = n_o;
 40 |   n_layers = n_l;
 41 | 
 42 |   sigmoid_layers = new HiddenLayer*[n_layers];
 43 |   dA_layers = new dA*[n_layers];
 44 | 
 45 |   // construct multi-layer
 46 |   for(int i=0; i<n_layers; i++) {
 47 |     if(i == 0) {
 48 |       input_size = n_ins;
 49 |     } else {
 50 |       input_size = hidden_layer_sizes[i-1];
 51 |     }
 52 | 
 53 |     // construct sigmoid_layer
 54 |     sigmoid_layers[i] = new HiddenLayer(N, input_size, hidden_layer_sizes[i], NULL, NULL);
 55 | 
 56 |     // construct dA_layer
 57 |     dA_layers[i] = new dA(N, input_size, hidden_layer_sizes[i],\
 58 |                           sigmoid_layers[i]->W, sigmoid_layers[i]->b, NULL);
 59 |   }
 60 | 
 61 |   // layer for output using LogisticRegression
 62 |   log_layer = new LogisticRegression(N, hidden_layer_sizes[n_layers-1], n_outs);
 63 | }
 64 | 
 65 | SdA::~SdA() {
 66 |   delete log_layer;
 67 | 
 68 |   for(int i=0; i<n_layers; i++) {
 69 |     delete sigmoid_layers[i];
 70 |     delete dA_layers[i];
 71 |   }
 72 |   delete[] sigmoid_layers;
 73 |   delete[] dA_layers;
 74 | }
 75 | 
 76 | void SdA::pretrain(int *input, double lr, double corruption_level, int epochs) {
 77 |   int *layer_input;
 78 |   int prev_layer_input_size;
 79 |   int *prev_layer_input;
 80 | 
 81 |   int *train_X = new int[n_ins];
 82 | 
 83 |   for(int i=0; i<n_layers; i++) {  // layer-wise
 84 | 
 85 |     for(int epoch=0; epoch<epochs; epoch++) {  // training epochs
 86 | 
 87 |       for(int n=0; n<N; n++) { // input x1...xN
 88 |         // initial input
 89 |         for(int m=0; m<n_ins; m++) train_X[m] = input[n * n_ins + m];
 90 | 
 91 |         // layer input
 92 |         for(int l=0; l<=i; l++) {
 93 | 
 94 |           if(l == 0) {
 95 |             layer_input = new int[n_ins];
 96 |             for(int j=0; j<n_ins; j++) layer_input[j] = train_X[j];
 97 |           } else {
 98 |             if(l == 1) prev_layer_input_size = n_ins;
 99 |             else prev_layer_input_size = hidden_layer_sizes[l-2];
100 | 
101 |             prev_layer_input = new int[prev_layer_input_size];
102 |             for(int j=0; j<prev_layer_input_size; j++) prev_layer_input[j] = layer_input[j];
103 |             delete[] layer_input;
104 | 
105 |             layer_input = new int[hidden_layer_sizes[l-1]];
106 | 
107 |             sigmoid_layers[l-1]->sample_h_given_v(prev_layer_input, layer_input);
108 |             delete[] prev_layer_input;
109 |           }
110 |         }
111 | 
112 |         dA_layers[i]->train(layer_input, lr, corruption_level);
113 | 
114 |       }
115 |     }
116 |   }
117 | 
118 |   delete[] train_X;
119 |   delete[] layer_input;
120 | }
121 | 
122 | void SdA::finetune(int *input, int *label, double lr, int epochs) {
123 |   int *layer_input;
124 |   int prev_layer_input_size;
125 |   int *prev_layer_input;
126 | 
127 |   int *train_X = new int[n_ins];
128 |   int *train_Y = new int[n_outs];
129 | 
130 |   for(int epoch=0; epoch<epochs; epoch++) {
131 |     for(int n=0; n<N; n++) { // input x1...xN
132 |       // initial input
133 |       for(int m=0; m<n_ins; m++)  train_X[m] = input[n * n_ins + m];
134 |       for(int m=0; m<n_outs; m++) train_Y[m] = label[n * n_outs + m];
135 | 
136 |       // layer input
137 |       for(int i=0; i<n_layers; i++) {
138 |         if(i == 0) {
139 |           prev_layer_input = new int[n_ins];
140 |           for(int j=0; j<n_ins; j++) prev_layer_input[j] = train_X[j];
141 |         } else {
142 |           prev_layer_input = new int[hidden_layer_sizes[i-1]];
143 |           for(int j=0; j<hidden_layer_sizes[i-1]; j++) prev_layer_input[j] = layer_input[j];
144 |           delete[] layer_input;
145 |         }
146 | 
147 | 
148 |         layer_input = new int[hidden_layer_sizes[i]];
149 |         sigmoid_layers[i]->sample_h_given_v(prev_layer_input, layer_input);
150 |         delete[] prev_layer_input;
151 |       }
152 | 
153 |       log_layer->train(layer_input, train_Y, lr);
154 |     }
155 |     // lr *= 0.95;
156 |   }
157 | 
158 |   delete[] layer_input;
159 |   delete[] train_X;
160 |   delete[] train_Y;
161 | }
162 | 
163 | void SdA::predict(int *x, double *y) {
164 |   double *layer_input;
165 |   int prev_layer_input_size;
166 |   double *prev_layer_input;
167 | 
168 |   double linear_output;
169 | 
170 |   prev_layer_input = new double[n_ins];
171 |   for(int j=0; j<n_ins; j++) prev_layer_input[j] = x[j];
172 | 
173 |   // layer activation
174 |   for(int i=0; i<n_layers; i++) {
175 |     layer_input = new double[sigmoid_layers[i]->n_out];
176 | 
177 |     for(int k=0; k<sigmoid_layers[i]->n_out; k++) {
178 |       linear_output = 0.0;
179 | 
180 |       for(int j=0; j<sigmoid_layers[i]->n_in; j++) {
181 |         linear_output += sigmoid_layers[i]->W[k][j] * prev_layer_input[j];
182 |       }
183 |       linear_output += sigmoid_layers[i]->b[k];
184 |       layer_input[k] = sigmoid(linear_output);
185 |     }
186 |     delete[] prev_layer_input;
187 | 
188 |     if(i < n_layers-1) {
189 |       prev_layer_input = new double[sigmoid_layers[i]->n_out];
190 |       for(int j=0; j<sigmoid_layers[i]->n_out; j++) prev_layer_input[j] = layer_input[j];
191 |       delete[] layer_input;
192 |     }
193 |   }
194 |   
195 |   for(int i=0; i<log_layer->n_out; i++) {
196 |     y[i] = 0;
197 |     for(int j=0; j<log_layer->n_in; j++) {
198 |       y[i] += log_layer->W[i][j] * layer_input[j];
199 |     }
200 |     y[i] += log_layer->b[i];
201 |   }
202 |   
203 |   log_layer->softmax(y);
204 | 
205 | 
206 |   delete[] layer_input;
207 | }
208 | 
209 | 
210 | // HiddenLayer
211 | HiddenLayer::HiddenLayer(int size, int in, int out, double **w, double *bp) {
212 |   N = size;
213 |   n_in = in;
214 |   n_out = out;
215 | 
216 |   if(w == NULL) {
217 |     W = new double*[n_out];
218 |     for(int i=0; i<n_out; i++) W[i] = new double[n_in];
219 |     double a = 1.0 / n_in;
220 | 
221 |     for(int i=0; i<n_out; i++) {
222 |       for(int j=0; j<n_in; j++) {
223 |         W[i][j] = uniform(-a, a);
224 |       }
225 |     }
226 |   } else {
227 |     W = w;
228 |   }
229 | 
230 |   if(bp == NULL) {
231 |     b = new double[n_out];
232 |   } else {
233 |     b = bp;
234 |   }
235 | }
236 | 
237 | HiddenLayer::~HiddenLayer() {
238 |   for(int i=0; i<n_out; i++) delete W[i];
239 |   delete[] W;
240 |   delete[] b;
241 | }
242 | 
243 | double HiddenLayer::output(int *input, double *w, double b) {
244 |   double linear_output = 0.0;
245 |   for(int j=0; j<n_in; j++) {
246 |     linear_output += w[j] * input[j];
247 |   }
248 |   linear_output += b;
249 |   return sigmoid(linear_output);
250 | }
251 | 
252 | void HiddenLayer::sample_h_given_v(int *input, int *sample) {
253 |   for(int i=0; i<n_out; i++) {
254 |     sample[i] = binomial(1, output(input, W[i], b[i]));
255 |   }
256 | }
257 | 
258 | 
259 | // dA
260 | dA::dA(int size, int n_v, int n_h, double **w, double *hb, double *vb) {
261 |   N = size;
262 |   n_visible = n_v;
263 |   n_hidden = n_h;
264 | 
265 |   if(w == NULL) {
266 |     W = new double*[n_hidden];
267 |     for(int i=0; i<n_hidden; i++) W[i] = new double[n_visible];
268 |     double a = 1.0 / n_visible;
269 | 
270 |     for(int i=0; i<n_hidden; i++) {
271 |       for(int j=0; j<n_visible; j++) {
272 |         W[i][j] = uniform(-a, a);
273 |       }
274 |     }
275 |   } else {
276 |     W = w;
277 |   }
278 | 
279 |   if(hb == NULL) {
280 |     hbias = new double[n_hidden];
281 |     for(int i=0; i<n_hidden; i++) hbias[i] = 0;
282 |   } else {
283 |     hbias = hb;
284 |   }
285 | 
286 |   if(vb == NULL) {
287 |     vbias = new double[n_visible];
288 |     for(int i=0; i<n_visible; i++) vbias[i] = 0;
289 |   } else {
290 |     vbias = vb;
291 |   }
292 | }
293 | 
294 | dA::~dA() {
295 |   // for(int i=0; i<n_hidden; i++) delete[] W[i];
296 |   // delete[] W;
297 |   // delete[] hbias;
298 |   delete[] vbias;
299 | }
300 | 
301 | void dA::get_corrupted_input(int *x, int *tilde_x, double p) {
302 |   for(int i=0; i<n_visible; i++) {
303 |     if(x[i] == 0) {
304 |       tilde_x[i] = 0;
305 |     } else {
306 |       tilde_x[i] = binomial(1, p);
307 |     }
308 |   }
309 | }
310 | 
311 | // Encode
312 | void dA::get_hidden_values(int *x, double *y) {
313 |   for(int i=0; i<n_hidden; i++) {
314 |     y[i] = 0;
315 |     for(int j=0; j<n_visible; j++) {
316 |       y[i] += W[i][j] * x[j];
317 |     }
318 |     y[i] += hbias[i];
319 |     y[i] = sigmoid(y[i]);
320 |   }
321 | }
322 | 
323 | // Decode
324 | void dA::get_reconstructed_input(double *y, double *z) {
325 |   for(int i=0; i<n_visible; i++) {
326 |     z[i] = 0;
327 |     for(int j=0; j<n_hidden; j++) {
328 |       z[i] += W[j][i] * y[j];
329 |     }
330 |     z[i] += vbias[i];
331 |     z[i] = sigmoid(z[i]);
332 |   }
333 | }
334 | 
335 | void dA::train(int *x, double lr, double corruption_level) {
336 |   int *tilde_x = new int[n_visible];
337 |   double *y = new double[n_hidden];
338 |   double *z = new double[n_visible];
339 | 
340 |   double *L_vbias = new double[n_visible];
341 |   double *L_hbias = new double[n_hidden];
342 | 
343 |   double p = 1 - corruption_level;
344 | 
345 |   get_corrupted_input(x, tilde_x, p);
346 |   get_hidden_values(tilde_x, y);
347 |   get_reconstructed_input(y, z);
348 |   
349 |   // vbias
350 |   for(int i=0; i<n_visible; i++) {
351 |     L_vbias[i] = x[i] - z[i];
352 |     vbias[i] += lr * L_vbias[i] / N;
353 |   }
354 | 
355 |   // hbias
356 |   for(int i=0; i<n_hidden; i++) {
357 |     L_hbias[i] = 0;
358 |     for(int j=0; j<n_visible; j++) {
359 |       L_hbias[i] += W[i][j] * L_vbias[j];
360 |     }
361 |     L_hbias[i] *= y[i] * (1 - y[i]);
362 | 
363 |     hbias[i] += lr * L_hbias[i] / N;
364 |   }
365 |   
366 |   // W
367 |   for(int i=0; i<n_hidden; i++) {
368 |     for(int j=0; j<n_visible; j++) {
369 |       W[i][j] += lr * (L_hbias[i] * tilde_x[j] + L_vbias[j] * y[i]) / N;
370 |     }
371 |   }
372 | 
373 |   delete[] L_hbias;
374 |   delete[] L_vbias;
375 |   delete[] z;
376 |   delete[] y;
377 |   delete[] tilde_x;
378 | }
379 | 
380 | void dA::reconstruct(int *x, double *z) {
381 |   double *y = new double[n_hidden];
382 | 
383 |   get_hidden_values(x, y);
384 |   get_reconstructed_input(y, z);
385 | 
386 |   delete[] y;
387 | }
388 | 
389 | 
390 | // LogisticRegression
391 | LogisticRegression::LogisticRegression(int size, int in, int out) {
392 |   N = size;
393 |   n_in = in;
394 |   n_out = out;
395 | 
396 |   W = new double*[n_out];
397 |   for(int i=0; i<n_out; i++) W[i] = new double[n_in];
398 |   b = new double[n_out];
399 | 
400 |   for(int i=0; i<n_out; i++) {
401 |     for(int j=0; j<n_in; j++) {
402 |       W[i][j] = 0;
403 |     }
404 |     b[i] = 0;
405 |   }
406 | }
407 | 
408 | LogisticRegression::~LogisticRegression() {
409 |   for(int i=0; i<n_out; i++) delete[] W[i];
410 |   delete[] W;
411 |   delete[] b;
412 | }
413 | 
414 | 
415 | void LogisticRegression::train(int *x, int *y, double lr) {
416 |   double *p_y_given_x = new double[n_out];
417 |   double *dy = new double[n_out];
418 | 
419 |   for(int i=0; i<n_out; i++) {
420 |     p_y_given_x[i] = 0;
421 |     for(int j=0; j<n_in; j++) {
422 |       p_y_given_x[i] += W[i][j] * x[j];
423 |     }
424 |     p_y_given_x[i] += b[i];
425 |   }
426 |   softmax(p_y_given_x);
427 | 
428 |   for(int i=0; i<n_out; i++) {
429 |     dy[i] = y[i] - p_y_given_x[i];
430 | 
431 |     for(int j=0; j<n_in; j++) {
432 |       W[i][j] += lr * dy[i] * x[j] / N;
433 |     }
434 | 
435 |     b[i] += lr * dy[i] / N;
436 |   }
437 |   
438 |   delete[] p_y_given_x;
439 |   delete[] dy;
440 | }
441 | 
442 | void LogisticRegression::softmax(double *x) {
443 |   double max = 0.0;
444 |   double sum = 0.0;
445 |   
446 |   for(int i=0; i<n_out; i++) if(max < x[i]) max = x[i];
447 |   for(int i=0; i<n_out; i++) {
448 |     x[i] = exp(x[i] - max);
449 |     sum += x[i];
450 |   } 
451 | 
452 |   for(int i=0; i<n_out; i++) x[i] /= sum;
453 | }
454 | 
455 | void LogisticRegression::predict(int *x, double *y) {
456 |   for(int i=0; i<n_out; i++) {
457 |     y[i] = 0;
458 |     for(int j=0; j<n_in; j++) {
459 |       y[i] += W[i][j] * x[j];
460 |     }
461 |     y[i] += b[i];
462 |   }
463 | 
464 |   softmax(y);
465 | }
466 | 
467 | 
468 | void test_sda() {
469 |   srand(0);
470 | 
471 |   double pretrain_lr = 0.1;
472 |   double corruption_level = 0.3;
473 |   int pretraining_epochs = 1000;
474 |   double finetune_lr = 0.1;
475 |   int finetune_epochs = 500;
476 | 
477 |   int train_N = 10;
478 |   int test_N = 4;
479 |   int n_ins = 28;
480 |   int n_outs = 2;
481 |   int hidden_layer_sizes[] = {15, 15};
482 |   int n_layers = sizeof(hidden_layer_sizes) / sizeof(hidden_layer_sizes[0]);
483 | 
484 |   // training data
485 |   int train_X[10][28] = {
486 |     {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
487 |     {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
488 |     {1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
489 |     {0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
490 |     {1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
491 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
492 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1},
493 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1},
494 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1},
495 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1}
496 |   };
497 | 
498 |   int train_Y[10][2] = {
499 |     {1, 0},
500 |     {1, 0},
501 |     {1, 0},
502 |     {1, 0},
503 |     {1, 0},
504 |     {0, 1},
505 |     {0, 1},
506 |     {0, 1},
507 |     {0, 1},
508 |     {0, 1}
509 |   };
510 | 
511 |   // construct SdA
512 |   SdA sda(train_N, n_ins, hidden_layer_sizes, n_outs, n_layers);
513 | 
514 |   // pretrain
515 |   sda.pretrain(*train_X, pretrain_lr, corruption_level, pretraining_epochs);
516 | 
517 |   // finetune
518 |   sda.finetune(*train_X, *train_Y, finetune_lr, finetune_epochs);
519 | 
520 | 
521 |   // test data
522 |   int test_X[4][28] = {
523 |     {1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
524 |     {1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
525 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1},
526 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1}
527 |   };
528 | 
529 |   double test_Y[4][28];
530 | 
531 |   // test
532 |   for(int i=0; i<test_N; i++) {
533 |     sda.predict(test_X[i], test_Y[i]);
534 |     for(int j=0; j<n_outs; j++) {
535 |       printf("%.5f ", test_Y[i][j]);
536 |     }
537 |     cout << endl;
538 |   }
539 |   
540 | }
541 | 
542 | 
543 | int main() {
544 |   test_sda();
545 |   return 0;
546 | }
547 | 


--------------------------------------------------------------------------------
/c/DBN.c:
--------------------------------------------------------------------------------
  1 | #include <stdio.h>
  2 | #include <stdlib.h>
  3 | #include <math.h>
  4 | #include "HiddenLayer.h"
  5 | #include "RBM.h"
  6 | #include "LogisticRegression.h"
  7 | #include "DBN.h"
  8 | #include "utils.h"
  9 | 
 10 | void test_dbn(void);
 11 | 
 12 | 
 13 | double uniform(double min, double max) {
 14 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;  
 15 | }
 16 | 
 17 | int binomial(int n, double p) {
 18 |   if(p < 0 || p > 1) return 0;
 19 | 
 20 |   int i;
 21 |   int c = 0;
 22 |   double r;
 23 | 
 24 |   for(i=0; i<n; i++) {
 25 |     r = rand() / (RAND_MAX + 1.0);
 26 |     if (r < p) c++;
 27 |   }
 28 | 
 29 |   return c;
 30 | }
 31 | 
 32 | double sigmoid(double x) {
 33 |   return 1.0 / (1.0 + exp(-x));
 34 | }
 35 | 
 36 | 
 37 | 
 38 | // DBN
 39 | void DBN__construct(DBN* this, int N, \
 40 |                     int n_ins, int *hidden_layer_sizes, int n_outs, int n_layers) {
 41 |   int i, input_size;
 42 | 
 43 |   this->N = N;
 44 |   this->n_ins = n_ins;
 45 |   this->hidden_layer_sizes = hidden_layer_sizes;
 46 |   this->n_outs = n_outs;
 47 |   this->n_layers = n_layers;
 48 | 
 49 |   this->sigmoid_layers = (HiddenLayer *)malloc(sizeof(HiddenLayer) * n_layers);
 50 |   this->rbm_layers = (RBM *)malloc(sizeof(RBM) * n_layers);
 51 | 
 52 |   // construct multi-layer
 53 |   for(i=0; i<n_layers; i++) {
 54 |     if(i == 0) {
 55 |       input_size = n_ins;
 56 |     } else {
 57 |       input_size = hidden_layer_sizes[i-1];
 58 |     }
 59 | 
 60 |     // construct sigmoid_layer
 61 |     HiddenLayer__construct(&(this->sigmoid_layers[i]), \
 62 |                            N, input_size, hidden_layer_sizes[i], NULL, NULL);
 63 | 
 64 |     // construct rbm_layer
 65 |     RBM__construct(&(this->rbm_layers[i]), N, input_size, hidden_layer_sizes[i], \
 66 |                    this->sigmoid_layers[i].W, this->sigmoid_layers[i].b, NULL);
 67 |     
 68 |   }
 69 | 
 70 |   // layer for output using LogisticRegression
 71 |   LogisticRegression__construct(&(this->log_layer), \
 72 |                                 N, hidden_layer_sizes[n_layers-1], n_outs);
 73 |   
 74 | }
 75 | 
 76 | void DBN__destruct(DBN* this) {
 77 |   int i;
 78 |   for(i=0; i<this->n_layers; i++) {
 79 |     HiddenLayer__destruct(&(this->sigmoid_layers[i]));
 80 |     RBM__destruct(&(this->rbm_layers[i]));
 81 |   }
 82 |   free(this->sigmoid_layers);
 83 |   free(this->rbm_layers);
 84 | }
 85 | 
 86 | void DBN_pretrain(DBN* this, int *input, double lr, int k, int epochs) {
 87 |   int i, j, l, m, n, epoch;
 88 |   
 89 |   int *layer_input;
 90 |   int prev_layer_input_size;
 91 |   int *prev_layer_input;
 92 | 
 93 |   int *train_X = (int *)malloc(sizeof(int) * this->n_ins);
 94 | 
 95 |   for(i=0; i<this->n_layers; i++) { // layer-wise
 96 | 
 97 |     for(epoch=0; epoch<epochs; epoch++) { // training epochs
 98 | 
 99 |       for(n=0; n<this->N; n++) { // input x1...xN
100 |         // initial input
101 |         for(m=0; m<this->n_ins; m++) train_X[m] = input[n * this->n_ins + m];
102 | 
103 |         // layer input
104 |         for(l=0; l<=i; l++) {
105 |           if(l == 0) {
106 |             layer_input = (int *)malloc(sizeof(int) * this->n_ins);
107 |             for(j=0; j<this->n_ins; j++) layer_input[j] = train_X[j];
108 |           } else {
109 |             if(l == 1) prev_layer_input_size = this->n_ins;
110 |             else prev_layer_input_size = this->hidden_layer_sizes[l-2];
111 | 
112 |             prev_layer_input = (int *)malloc(sizeof(int) * prev_layer_input_size);
113 |             for(j=0; j<prev_layer_input_size; j++) prev_layer_input[j] = layer_input[j];
114 |             free(layer_input);
115 | 
116 |             layer_input = (int *)malloc(sizeof(int) * this->hidden_layer_sizes[l-1]);
117 | 
118 |             HiddenLayer_sample_h_given_v(&(this->sigmoid_layers[l-1]), \
119 |                                          prev_layer_input, layer_input);
120 |             free(prev_layer_input);
121 |           }
122 |         }
123 | 
124 |         RBM_contrastive_divergence(&(this->rbm_layers[i]), layer_input, lr, k);
125 |       }
126 |       
127 |     }
128 |   }
129 |   
130 |   free(train_X);
131 |   free(layer_input);
132 | }
133 | 
134 | void DBN_finetune(DBN* this, int *input, int *label, double lr, int epochs) {
135 |   int i, j, m, n, epoch;
136 |   
137 |   int *layer_input;
138 |   // int prev_layer_input_size;
139 |   int *prev_layer_input;
140 | 
141 |   int *train_X = (int *)malloc(sizeof(int) * this->n_ins);
142 |   int *train_Y = (int *)malloc(sizeof(int) * this->n_outs);
143 | 
144 |   for(epoch=0; epoch<epochs; epoch++) {
145 |     for(n=0; n<this->N; n++) { // input x1...xN
146 |       // initial input
147 |       for(m=0; m<this->n_ins; m++)  train_X[m] = input[n * this->n_ins + m];
148 |       for(m=0; m<this->n_outs; m++) train_Y[m] = label[n * this->n_outs + m];
149 | 
150 |       // layer input
151 |       for(i=0; i<this->n_layers; i++) {
152 |         if(i == 0) {
153 |           prev_layer_input = (int *)malloc(sizeof(int) * this->n_ins);
154 |           for(j=0; j<this->n_ins; j++) prev_layer_input[j] = train_X[j];
155 |         } else {
156 |           prev_layer_input = (int *)malloc(sizeof(int) * this->hidden_layer_sizes[i-1]);
157 |           for(j=0; j<this->hidden_layer_sizes[i-1]; j++) prev_layer_input[j] = layer_input[j];
158 |           free(layer_input);
159 |         }
160 | 
161 | 
162 |         layer_input = (int *)malloc(sizeof(int) * this->hidden_layer_sizes[i]);
163 |         HiddenLayer_sample_h_given_v(&(this->sigmoid_layers[i]), \
164 |                                      prev_layer_input, layer_input);
165 |         free(prev_layer_input);
166 |       }
167 | 
168 |       LogisticRegression_train(&(this->log_layer), layer_input, train_Y, lr);
169 |     }
170 |     // lr *= 0.95;
171 |   }
172 | 
173 |   free(layer_input);
174 |   free(train_X);
175 |   free(train_Y);
176 | }
177 | 
178 | void DBN_predict(DBN* this, int *x, double *y) {
179 |   int i, j, k;
180 |   double *layer_input;
181 |   // int prev_layer_input_size;
182 |   double *prev_layer_input;
183 | 
184 |   double linear_output;
185 | 
186 |   prev_layer_input = (double *)malloc(sizeof(double) * this->n_ins);
187 |   for(j=0; j<this->n_ins; j++) prev_layer_input[j] = x[j];
188 | 
189 |   // layer activation
190 |   for(i=0; i<this->n_layers; i++) {
191 |     layer_input = (double *)malloc(sizeof(double) * this->sigmoid_layers[i].n_out);
192 | 
193 |     for(k=0; k<this->sigmoid_layers[i].n_out; k++) {
194 |       linear_output = 0.0;
195 | 
196 |       for(j=0; j<this->sigmoid_layers[i].n_in; j++) {
197 |         linear_output += this->sigmoid_layers[i].W[k][j] * prev_layer_input[j];
198 |       }
199 |       linear_output += this->sigmoid_layers[i].b[k];
200 |       layer_input[k] = sigmoid(linear_output);
201 |     }
202 |     free(prev_layer_input);
203 | 
204 |     if(i < this->n_layers-1) {
205 |       prev_layer_input = (double *)malloc(sizeof(double) * this->sigmoid_layers[i].n_out);
206 |       for(j=0; j<this->sigmoid_layers[i].n_out; j++) prev_layer_input[j] = layer_input[j];
207 |       free(layer_input);
208 |     }
209 |   }
210 | 
211 |   for(i=0; i<this->log_layer.n_out; i++) {
212 |     y[i] = 0;
213 |     for(j=0; j<this->log_layer.n_in; j++) {
214 |       y[i] += this->log_layer.W[i][j] * layer_input[j];
215 |     }
216 |     y[i] += this->log_layer.b[i];
217 |   }
218 | 
219 |   LogisticRegression_softmax(&(this->log_layer), y);
220 | 
221 |   free(layer_input);
222 | }
223 | 
224 | 
225 | 
226 | // HiddenLayer
227 | void HiddenLayer__construct(HiddenLayer* this, int N, int n_in, int n_out, \
228 |                             double **W, double *b) {
229 |   int i, j;
230 |   double a = 1.0 / n_in;
231 | 
232 |   this->N = N;
233 |   this->n_in = n_in;
234 |   this->n_out = n_out;
235 |   
236 |   if(W == NULL) {
237 |     this->W = (double **)malloc(sizeof(double*) * n_out);
238 |     this->W[0] = (double *)malloc(sizeof(double) * n_in * n_out);
239 |     for(i=0; i<n_out; i++) this->W[i] = this->W[0] + i * n_in;
240 | 
241 |     for(i=0; i<n_out; i++) {
242 |       for(j=0; j<n_in; j++) {
243 |         this->W[i][j] = uniform(-a, a);
244 |       }
245 |     }
246 |   } else {
247 |     this->W = W;
248 |   }
249 | 
250 |   if(b == NULL) {
251 |     this->b = (double *)malloc(sizeof(double) * n_out);
252 |   } else {
253 |     this->b = b;
254 |   }
255 | }
256 | 
257 | void HiddenLayer__destruct(HiddenLayer* this) {
258 |   free(this->W[0]);
259 |   free(this->W);
260 |   free(this->b);
261 | }
262 | 
263 | double HiddenLayer_output(HiddenLayer* this, int *input, double *w, double b) {
264 |   int j;
265 |   double linear_output = 0.0;
266 |   for(j=0; j<this->n_in; j++) {
267 |     linear_output += w[j] * input[j];
268 |   }
269 |   linear_output += b;
270 |   return sigmoid(linear_output);
271 | }
272 | 
273 | void HiddenLayer_sample_h_given_v(HiddenLayer* this, int *input, int *sample) {
274 |   int i;
275 |   for(i=0; i<this->n_out; i++) {
276 |     sample[i] = binomial(1, HiddenLayer_output(this, input, this->W[i], this->b[i]));
277 |   }
278 | }
279 | 
280 | 
281 | // RBM
282 | void RBM__construct(RBM* this, int N, int n_visible, int n_hidden, \
283 |                     double **W, double *hbias, double *vbias) {
284 |   int i, j;
285 |   double a = 1.0 / n_visible;
286 | 
287 |   this->N = N;
288 |   this->n_visible = n_visible;
289 |   this->n_hidden = n_hidden;
290 | 
291 |   if(W == NULL) {
292 |     this->W = (double **)malloc(sizeof(double*) * n_hidden);
293 |     this->W[0] = (double *)malloc(sizeof(double) * n_visible * n_hidden);
294 |     for(i=0; i<n_hidden; i++) this->W[i] = this->W[0] + i * n_visible;
295 | 
296 |     for(i=0; i<n_hidden; i++) {
297 |       for(j=0; j<n_visible; j++) {
298 |         this->W[i][j] = uniform(-a, a);
299 |       }
300 |     }
301 |   } else {
302 |     this->W = W;
303 |   }
304 | 
305 |   if(hbias == NULL) {
306 |     this->hbias = (double *)malloc(sizeof(double) * n_hidden);
307 |     for(i=0; i<n_hidden; i++) this->hbias[i] = 0;
308 |   } else {
309 |     this->hbias = hbias;
310 |   }
311 | 
312 |   if(vbias == NULL) {
313 |     this->vbias = (double *)malloc(sizeof(double) * n_visible);
314 |     for(i=0; i<n_visible; i++) this->vbias[i] = 0;
315 |   } else {
316 |     this->vbias = vbias;
317 |   }
318 | }
319 | 
320 | void RBM__destruct(RBM* this) {
321 |   // free(this->W[0]);
322 |   // free(this->W);
323 |   // free(this->hbias);
324 |   free(this->vbias);
325 | }
326 | 
327 | void RBM_contrastive_divergence(RBM* this, int *input, double lr, int k) {
328 |   int i, j, step;
329 |   
330 |   double *ph_mean = (double *)malloc(sizeof(double) * this->n_hidden);
331 |   int *ph_sample = (int *)malloc(sizeof(int) * this->n_hidden);
332 |   double *nv_means = (double *)malloc(sizeof(double) * this->n_visible);
333 |   int *nv_samples = (int *)malloc(sizeof(int) * this->n_visible);
334 |   double *nh_means = (double *)malloc(sizeof(double) * this->n_hidden);
335 |   int *nh_samples = (int *)malloc(sizeof(int) * this->n_hidden);
336 | 
337 |   /* CD-k */
338 |   RBM_sample_h_given_v(this, input, ph_mean, ph_sample);
339 | 
340 |   for(step=0; step<k; step++) {
341 |     if(step == 0) {
342 |       RBM_gibbs_hvh(this, ph_sample, nv_means, nv_samples, nh_means, nh_samples);
343 |     } else {
344 |       RBM_gibbs_hvh(this, nh_samples, nv_means, nv_samples, nh_means, nh_samples);
345 |     }
346 |   }
347 | 
348 |   for(i=0; i<this->n_hidden; i++) {
349 |     for(j=0; j<this->n_visible; j++) {
350 |       // this->W[i][j] += lr * (ph_sample[i] * input[j] - nh_means[i] * nv_samples[j]) / this->N;
351 |       this->W[i][j] += lr * (ph_mean[i] * input[j] - nh_means[i] * nv_samples[j]) / this->N;
352 |     }
353 |     this->hbias[i] += lr * (ph_sample[i] - nh_means[i]) / this->N;
354 |   }
355 | 
356 |   for(i=0; i<this->n_visible; i++) {
357 |     this->vbias[i] += lr * (input[i] - nv_samples[i]) / this->N;
358 |   }
359 |   
360 | 
361 |   free(ph_mean);
362 |   free(ph_sample);
363 |   free(nv_means);
364 |   free(nv_samples);
365 |   free(nh_means);
366 |   free(nh_samples);
367 | }
368 | 
369 | 
370 | void RBM_sample_h_given_v(RBM* this, int *v0_sample, double *mean, int *sample) {
371 |   int i;
372 |   for(i=0; i<this->n_hidden; i++) {
373 |     mean[i] = RBM_propup(this, v0_sample, this->W[i], this->hbias[i]);
374 |     sample[i] = binomial(1, mean[i]);
375 |   }
376 | }
377 | 
378 | void RBM_sample_v_given_h(RBM* this, int *h0_sample, double *mean, int *sample) {
379 |   int i;
380 |   for(i=0; i<this->n_visible; i++) {
381 |     mean[i] = RBM_propdown(this, h0_sample, i, this->vbias[i]);
382 |     sample[i] = binomial(1, mean[i]);
383 |   }
384 | }
385 | 
386 | double RBM_propup(RBM* this, int *v, double *w, double b) {
387 |   int j;
388 |   double pre_sigmoid_activation = 0.0;
389 |   for(j=0; j<this->n_visible; j++) {
390 |     pre_sigmoid_activation += w[j] * v[j];
391 |   }
392 |   pre_sigmoid_activation += b;
393 |   return sigmoid(pre_sigmoid_activation);
394 | }
395 | 
396 | double RBM_propdown(RBM* this, int *h, int i, double b) {
397 |   int j;
398 |   double pre_sigmoid_activation = 0.0;
399 | 
400 |   for(j=0; j<this->n_hidden; j++) {
401 |     pre_sigmoid_activation += this->W[j][i] * h[j];
402 |   }
403 |   pre_sigmoid_activation += b;
404 |   return sigmoid(pre_sigmoid_activation);
405 | }
406 | 
407 | void RBM_gibbs_hvh(RBM* this, int *h0_sample, double *nv_means, int *nv_samples, \
408 |                    double *nh_means, int *nh_samples) {
409 |   RBM_sample_v_given_h(this, h0_sample, nv_means, nv_samples);
410 |   RBM_sample_h_given_v(this, nv_samples, nh_means, nh_samples);
411 | }
412 | 
413 | void RBM_reconstruct(RBM* this, int *v, double *reconstructed_v) {
414 |   int i, j;
415 |   double *h = (double *)malloc(sizeof(double) * this->n_hidden);
416 |   double pre_sigmoid_activation;
417 | 
418 |   for(i=0; i<this->n_hidden; i++) {
419 |     h[i] = RBM_propup(this, v, this->W[i], this->hbias[i]);
420 |   }
421 | 
422 |   for(i=0; i<this->n_visible; i++) {
423 |     pre_sigmoid_activation = 0.0;
424 |     for(j=0; j<this->n_hidden; j++) {
425 |       pre_sigmoid_activation += this->W[j][i] * h[j];
426 |     }
427 |     pre_sigmoid_activation += this->vbias[i];
428 | 
429 |     reconstructed_v[i] = sigmoid(pre_sigmoid_activation);
430 |   }
431 | 
432 |   free(h);
433 | }
434 | 
435 | 
436 | // LogisticRegression
437 | void LogisticRegression__construct(LogisticRegression *this, int N, int n_in, int n_out) {
438 |   int i, j;
439 |   this->N = N;
440 |   this->n_in = n_in;
441 |   this->n_out = n_out;
442 | 
443 |   this->W = (double **)malloc(sizeof(double*) * n_out);
444 |   this->W[0] = (double *)malloc(sizeof(double) * n_in * n_out);
445 |   for(i=0; i<n_out; i++) this->W[i] = this->W[0] + i * n_in;
446 |   this->b = (double *)malloc(sizeof(double) * n_out);
447 | 
448 |   for(i=0; i<n_out; i++) {
449 |     for(j=0; j<n_in; j++) {
450 |       this->W[i][j] = 0;
451 |     }
452 |     this->b[i] = 0;
453 |   }
454 | }
455 | 
456 | void LogisticRegression__destruct(LogisticRegression *this) {
457 |   free(this->W[0]);
458 |   free(this->W);
459 |   free(this->b);
460 | }
461 | 
462 | void LogisticRegression_train(LogisticRegression *this, int *x, int *y, double lr) {
463 |   int i,j;
464 |   double *p_y_given_x = (double *)malloc(sizeof(double) * this->n_out);
465 |   double *dy = (double *)malloc(sizeof(double) * this->n_out);
466 | 
467 |   for(i=0; i<this->n_out; i++) {
468 |     p_y_given_x[i] = 0;
469 |     for(j=0; j<this->n_in; j++) {
470 |       p_y_given_x[i] += this->W[i][j] * x[j];
471 |     }
472 |     p_y_given_x[i] += this->b[i];
473 |   }
474 |   LogisticRegression_softmax(this, p_y_given_x);
475 | 
476 |   for(i=0; i<this->n_out; i++) {
477 |     dy[i] = y[i] - p_y_given_x[i];
478 | 
479 |     for(j=0; j<this->n_in; j++) {
480 |       this->W[i][j] += lr * dy[i] * x[j] / this->N;
481 |     }
482 | 
483 |     this->b[i] += lr * dy[i] / this->N;
484 |   }
485 | 
486 |   free(p_y_given_x);
487 |   free(dy);
488 | }
489 | 
490 | void LogisticRegression_softmax(LogisticRegression *this, double *x) {
491 |   int i;
492 |   double max = 0.0;
493 |   double sum = 0.0;
494 | 
495 |   for(i=0; i<this->n_out; i++) if(max < x[i]) max = x[i];
496 |   for(i=0; i<this->n_out; i++) {
497 |     x[i] = exp(x[i] - max);
498 |     sum += x[i];
499 |   }
500 | 
501 |   for(i=0; i<this->n_out; i++) x[i] /= sum;
502 | }
503 | 
504 | void LogisticRegression_predict(LogisticRegression *this, int *x, double *y) {
505 |   int i,j;
506 | 
507 |   for(i=0; i<this->n_out; i++) {
508 |     y[i] = 0;
509 |     for(j=0; j<this->n_in; j++) {
510 |       y[i] += this->W[i][j] * x[j];
511 |     }
512 |     y[i] += this->b[i];
513 |   }
514 | 
515 |   LogisticRegression_softmax(this, y);
516 | }
517 | 
518 | 
519 | void test_dbn(void) {
520 |   srand(0);
521 | 
522 |   int i, j;
523 |   
524 |   double pretrain_lr = 0.1;
525 |   int pretraining_epochs = 1000;
526 |   int k = 1;
527 |   double finetune_lr = 0.1;
528 |   int finetune_epochs = 500;
529 | 
530 |   int train_N = 6;
531 |   int test_N = 4;
532 |   int n_ins = 6;
533 |   int n_outs = 2;
534 |   int hidden_layer_sizes[] = {3, 3};
535 |   int n_layers = sizeof(hidden_layer_sizes) / sizeof(hidden_layer_sizes[0]);
536 | 
537 | 
538 |   // training data
539 |   int train_X[6][6] = {
540 |     {1, 1, 1, 0, 0, 0},
541 |     {1, 0, 1, 0, 0, 0},
542 |     {1, 1, 1, 0, 0, 0},
543 |     {0, 0, 1, 1, 1, 0},
544 |     {0, 0, 1, 1, 0, 0},
545 |     {0, 0, 1, 1, 1, 0}
546 |   };
547 | 
548 |   int train_Y[6][2] = {
549 |     {1, 0},
550 |     {1, 0},
551 |     {1, 0},
552 |     {0, 1},
553 |     {0, 1},
554 |     {0, 1}
555 |   };
556 | 
557 |   // construct DBN
558 |   DBN dbn;
559 |   DBN__construct(&dbn, train_N, n_ins, hidden_layer_sizes, n_outs, n_layers);
560 | 
561 |   // pretrain
562 |   DBN_pretrain(&dbn, *train_X, pretrain_lr, k, pretraining_epochs);
563 | 
564 |   // finetune
565 |   DBN_finetune(&dbn, *train_X, *train_Y, finetune_lr, finetune_epochs);
566 | 
567 |   // test data
568 |   int test_X[4][6] = {
569 |     {1, 1, 0, 0, 0, 0},
570 |     {1, 1, 1, 1, 0, 0},
571 |     {0, 0, 0, 1, 1, 0},
572 |     {0, 0, 1, 1, 1, 0}
573 |   };
574 | 
575 |   double test_Y[4][2];
576 | 
577 |   // test
578 |   for(i=0; i<test_N; i++) {
579 |     DBN_predict(&dbn, test_X[i], test_Y[i]);
580 |     for(j=0; j<n_outs; j++) {
581 |       printf("%.5f ", test_Y[i][j]);
582 |     }
583 |     printf("\n");
584 |   }
585 | 
586 |   // destruct DBN
587 |   DBN__destruct(&dbn);
588 |   
589 | }
590 | 
591 | 
592 | 
593 | int main(void) {
594 |   test_dbn();
595 |   return 0;
596 | }
597 | 


--------------------------------------------------------------------------------
/c/SdA.c:
--------------------------------------------------------------------------------
  1 | #include <stdio.h>
  2 | #include <stdlib.h>
  3 | #include <math.h>
  4 | #include "HiddenLayer.h"
  5 | #include "dA.h"
  6 | #include "LogisticRegression.h"
  7 | #include "SdA.h"
  8 | #include "utils.h"
  9 | 
 10 | void test_sda(void);
 11 | 
 12 | 
 13 | double uniform(double min, double max) {
 14 |   return rand() / (RAND_MAX + 1.0) * (max - min) + min;  
 15 | }
 16 | 
 17 | int binomial(int n, double p) {
 18 |   if(p < 0 || p > 1) return 0;
 19 | 
 20 |   int i;
 21 |   int c = 0;
 22 |   double r;
 23 | 
 24 |   for(i=0; i<n; i++) {
 25 |     r = rand() / (RAND_MAX + 1.0);
 26 |     if (r < p) c++;
 27 |   }
 28 | 
 29 |   return c;
 30 | }
 31 | 
 32 | double sigmoid(double x) {
 33 |   return 1.0 / (1.0 + exp(-x));
 34 | }
 35 | 
 36 | 
 37 | // SdA
 38 | void SdA__construct(SdA* this, int N, \
 39 |                     int n_ins, int *hidden_layer_sizes, int n_outs, int n_layers) {
 40 |   int i, input_size;
 41 | 
 42 |   this->N = N;
 43 |   this->n_ins = n_ins;
 44 |   this->hidden_layer_sizes = hidden_layer_sizes;
 45 |   this->n_outs = n_outs;
 46 |   this->n_layers = n_layers;
 47 | 
 48 |   this->sigmoid_layers = (HiddenLayer *)malloc(sizeof(HiddenLayer) * n_layers);
 49 |   this->dA_layers = (dA *)malloc(sizeof(dA) * n_layers);
 50 | 
 51 |   // construct multi-layer
 52 |   for(i=0; i<n_layers; i++) {
 53 |     if(i == 0) {
 54 |       input_size = n_ins;
 55 |     } else {
 56 |       input_size = hidden_layer_sizes[i-1];
 57 |     }
 58 | 
 59 |     // construct sigmoid_layer
 60 |     HiddenLayer__construct(&(this->sigmoid_layers[i]), \
 61 |                            N, input_size, hidden_layer_sizes[i], NULL, NULL);
 62 | 
 63 |     // construct dA_layer
 64 |     dA__construct(&(this->dA_layers[i]), N, input_size, hidden_layer_sizes[i], \
 65 |                    this->sigmoid_layers[i].W, this->sigmoid_layers[i].b, NULL);
 66 |     
 67 |   }
 68 | 
 69 |   // layer for output using LogisticRegression
 70 |   LogisticRegression__construct(&(this->log_layer), \
 71 |                                 N, hidden_layer_sizes[n_layers-1], n_outs);
 72 | }
 73 | 
 74 | void SdA__destruct(SdA* this) {
 75 |   int i;
 76 |   for(i=0; i<this->n_layers; i++) {
 77 |     HiddenLayer__destruct(&(this->sigmoid_layers[i]));
 78 |     dA__destruct(&(this->dA_layers[i]));
 79 |   }
 80 |   free(this->sigmoid_layers);
 81 |   free(this->dA_layers);
 82 | }
 83 | 
 84 | void SdA_pretrain(SdA* this, int *input, double lr, double corruption_level, int epochs) {
 85 |   int i, j, l, m, n, epoch;
 86 |   
 87 |   int *layer_input;
 88 |   int prev_layer_input_size;
 89 |   int *prev_layer_input;
 90 | 
 91 |   int *train_X = (int *)malloc(sizeof(int) * this->n_ins);
 92 | 
 93 |   for(i=0; i<this->n_layers; i++) { // layer-wise
 94 | 
 95 |     for(epoch=0; epoch<epochs; epoch++) { // training epochs
 96 | 
 97 |       for(n=0; n<this->N; n++) { // input x1...xN
 98 |         // initial input
 99 |         for(m=0; m<this->n_ins; m++) train_X[m] = input[n * this->n_ins + m];
100 | 
101 |         // layer input
102 |         for(l=0; l<=i; l++) {
103 |           if(l == 0) {
104 |             layer_input = (int *)malloc(sizeof(int) * this->n_ins);
105 |             for(j=0; j<this->n_ins; j++) layer_input[j] = train_X[j];
106 |           } else {
107 |             if(l == 1) prev_layer_input_size = this->n_ins;
108 |             else prev_layer_input_size = this->hidden_layer_sizes[l-2];
109 | 
110 |             prev_layer_input = (int *)malloc(sizeof(int) * prev_layer_input_size);
111 |             for(j=0; j<prev_layer_input_size; j++) prev_layer_input[j] = layer_input[j];
112 |             free(layer_input);
113 | 
114 |             layer_input = (int *)malloc(sizeof(int) * this->hidden_layer_sizes[l-1]);
115 | 
116 |             HiddenLayer_sample_h_given_v(&(this->sigmoid_layers[l-1]), \
117 |                                          prev_layer_input, layer_input);
118 |             free(prev_layer_input);
119 |           }
120 |         }
121 | 
122 |         dA_train(&(this->dA_layers[i]), layer_input, lr, corruption_level);
123 |       }
124 |       
125 |     }
126 |   }
127 |   
128 |   free(train_X);
129 |   free(layer_input);
130 | }
131 | 
132 | void SdA_finetune(SdA* this, int *input, int *label, double lr, int epochs) {
133 |   int i, j, m, n, epoch;
134 |   
135 |   int *layer_input;
136 |   int prev_layer_input_size;
137 |   int *prev_layer_input;
138 | 
139 |   int *train_X = (int *)malloc(sizeof(int) * this->n_ins);
140 |   int *train_Y = (int *)malloc(sizeof(int) * this->n_outs);
141 | 
142 |   for(epoch=0; epoch<epochs; epoch++) {
143 |     for(n=0; n<this->N; n++) { // input x1...xN
144 |       // initial input
145 |       for(m=0; m<this->n_ins; m++)  train_X[m] = input[n * this->n_ins + m];
146 |       for(m=0; m<this->n_outs; m++) train_Y[m] = label[n * this->n_outs + m];
147 | 
148 |       // layer input
149 |       for(i=0; i<this->n_layers; i++) {
150 |         if(i == 0) {
151 |           prev_layer_input = (int *)malloc(sizeof(int) * this->n_ins);
152 |           for(j=0; j<this->n_ins; j++) prev_layer_input[j] = train_X[j];
153 |         } else {
154 |           prev_layer_input = (int *)malloc(sizeof(int) * this->hidden_layer_sizes[i-1]);
155 |           for(j=0; j<this->hidden_layer_sizes[i-1]; j++) prev_layer_input[j] = layer_input[j];
156 |           free(layer_input);
157 |         }
158 | 
159 | 
160 |         layer_input = (int *)malloc(sizeof(int) * this->hidden_layer_sizes[i]);
161 |         HiddenLayer_sample_h_given_v(&(this->sigmoid_layers[i]), \
162 |                                      prev_layer_input, layer_input);
163 |         free(prev_layer_input);
164 |       }
165 | 
166 |       LogisticRegression_train(&(this->log_layer), layer_input, train_Y, lr);
167 |     }
168 |     // lr *= 0.95;
169 |   }
170 | 
171 |   free(layer_input);
172 |   free(train_X);
173 |   free(train_Y);
174 | }
175 | 
176 | void SdA_predict(SdA* this, int *x, double *y) {
177 |   int i, j, k;
178 |   double *layer_input;
179 |   int prev_layer_input_size;
180 |   double *prev_layer_input;
181 | 
182 |   double linear_output;
183 | 
184 |   prev_layer_input = (double *)malloc(sizeof(double) * this->n_ins);
185 |   for(j=0; j<this->n_ins; j++) prev_layer_input[j] = x[j];
186 | 
187 |   // layer activation
188 |   for(i=0; i<this->n_layers; i++) {
189 |     layer_input = (double *)malloc(sizeof(double) * this->sigmoid_layers[i].n_out);
190 | 
191 |     for(k=0; k<this->sigmoid_layers[i].n_out; k++) {
192 |       linear_output = 0.0;
193 | 
194 |       for(j=0; j<this->sigmoid_layers[i].n_in; j++) {
195 |         linear_output += this->sigmoid_layers[i].W[k][j] * prev_layer_input[j];
196 |       }
197 |       linear_output += this->sigmoid_layers[i].b[k];
198 |       layer_input[k] = sigmoid(linear_output);
199 |     }
200 |     free(prev_layer_input);
201 | 
202 |     if(i < this->n_layers-1) {
203 |       prev_layer_input = (double *)malloc(sizeof(double) * this->sigmoid_layers[i].n_out);
204 |       for(j=0; j<this->sigmoid_layers[i].n_out; j++) prev_layer_input[j] = layer_input[j];
205 |       free(layer_input);
206 |     }
207 |   }
208 | 
209 |   for(i=0; i<this->log_layer.n_out; i++) {
210 |     y[i] = 0;
211 |     for(j=0; j<this->log_layer.n_in; j++) {
212 |       y[i] += this->log_layer.W[i][j] * layer_input[j];
213 |     }
214 |     y[i] += this->log_layer.b[i];
215 |   }
216 | 
217 |   LogisticRegression_softmax(&(this->log_layer), y);
218 | 
219 |   free(layer_input);
220 | }
221 | 
222 | 
223 | // HiddenLayer
224 | void HiddenLayer__construct(HiddenLayer* this, int N, int n_in, int n_out, \
225 |                             double **W, double *b) {
226 |   int i, j;
227 |   double a = 1.0 / n_in;
228 | 
229 |   this->N = N;
230 |   this->n_in = n_in;
231 |   this->n_out = n_out;
232 |   
233 |   if(W == NULL) {
234 |     this->W = (double **)malloc(sizeof(double*) * n_out);
235 |     this->W[0] = (double *)malloc(sizeof(double) * n_in * n_out);
236 |     for(i=0; i<n_out; i++) this->W[i] = this->W[0] + i * n_in;
237 | 
238 |     for(i=0; i<n_out; i++) {
239 |       for(j=0; j<n_in; j++) {
240 |         this->W[i][j] = uniform(-a, a);
241 |       }
242 |     }
243 |   } else {
244 |     this->W = W;
245 |   }
246 | 
247 |   if(b == NULL) {
248 |     this->b = (double *)malloc(sizeof(double) * n_out);
249 |   } else {
250 |     this->b = b;
251 |   }
252 | }
253 | 
254 | void HiddenLayer__destruct(HiddenLayer* this) {
255 |   free(this->W[0]);
256 |   free(this->W);
257 |   free(this->b);
258 | }
259 | 
260 | double HiddenLayer_output(HiddenLayer* this, int *input, double *w, double b) {
261 |   int j;
262 |   double linear_output = 0.0;
263 |   for(j=0; j<this->n_in; j++) {
264 |     linear_output += w[j] * input[j];
265 |   }
266 |   linear_output += b;
267 |   return sigmoid(linear_output);
268 | }
269 | 
270 | void HiddenLayer_sample_h_given_v(HiddenLayer* this, int *input, int *sample) {
271 |   int i;
272 |   for(i=0; i<this->n_out; i++) {
273 |     sample[i] = binomial(1, HiddenLayer_output(this, input, this->W[i], this->b[i]));
274 |   }
275 | }
276 | 
277 | 
278 | // dA
279 | void dA__construct(dA* this, int N, int n_visible, int n_hidden, \
280 |                    double **W, double *hbias, double *vbias) {
281 |   int i, j;
282 |   double a = 1.0 / n_visible;
283 |   
284 |   this->N = N;
285 |   this->n_visible = n_visible;
286 |   this->n_hidden = n_hidden;
287 | 
288 |   if(W == NULL) {
289 |     this->W = (double **)malloc(sizeof(double*) * n_hidden);
290 |     this->W[0] = (double *)malloc(sizeof(double) * n_visible * n_hidden);
291 |     for(i=0; i<n_hidden; i++) this->W[i] = this->W[0] + i * n_visible;
292 | 
293 |     for(i=0; i<n_hidden; i++) {
294 |       for(j=0; j<n_visible; j++) {
295 |         this->W[i][j] = uniform(-a, a);
296 |       }
297 |     }
298 |   } else {
299 |     this->W = W;
300 |   }
301 | 
302 |   if(hbias == NULL) {
303 |     this->hbias = (double *)malloc(sizeof(double) * n_hidden);
304 |     for(i=0; i<n_hidden; i++) this->hbias[i] = 0;
305 |   } else {
306 |     this->hbias = hbias;
307 |   }
308 | 
309 |   if(vbias == NULL) {
310 |     this->vbias = (double *)malloc(sizeof(double) * n_visible);
311 |     for(i=0; i<n_visible; i++) this->vbias[i] = 0;
312 |   } else {
313 |     this->vbias = vbias;
314 |   }
315 | }
316 | 
317 | void dA__destruct(dA* this) {
318 |   // free(this->W[0]);
319 |   // free(this->W);
320 |   // free(this->hbias);
321 |   free(this->vbias);
322 | }
323 | 
324 | void dA_get_corrupted_input(dA* this, int *x, int *tilde_x, double p) {
325 |   int i;
326 |   for(i=0; i<this->n_visible; i++) {
327 |     if(x[i] == 0) {
328 |       tilde_x[i] = 0;
329 |     } else {
330 |       tilde_x[i] = binomial(1, p);
331 |     }
332 |   }
333 | }
334 | 
335 | // Encode
336 | void dA_get_hidden_values(dA* this, int *x, double *y) {
337 |   int i,j;
338 |   for(i=0; i<this->n_hidden; i++) {
339 |     y[i] = 0;
340 |     for(j=0; j<this->n_visible; j++) {
341 |       y[i] += this->W[i][j] * x[j];
342 |     }
343 |     y[i] += this->hbias[i];
344 |     y[i] = sigmoid(y[i]);
345 |   }
346 | }
347 | 
348 | // Decode
349 | void dA_get_reconstructed_input(dA* this, double *y, double *z) {
350 |   int i, j;
351 |   for(i=0; i<this->n_visible; i++) {
352 |     z[i] = 0;
353 |     for(j=0; j<this->n_hidden; j++) {
354 |       z[i] += this->W[j][i] * y[j];
355 |     }
356 |     z[i] += this->vbias[i];
357 |     z[i] = sigmoid(z[i]);
358 |   }
359 | }
360 | 
361 | 
362 | void dA_train(dA* this, int *x, double lr, double corruption_level) {
363 |   int i, j;
364 |   
365 |   int *tilde_x = (int *)malloc(sizeof(int) * this->n_visible);
366 |   double *y = (double *)malloc(sizeof(double) * this->n_hidden);
367 |   double *z = (double *)malloc(sizeof(double) * this->n_visible);
368 | 
369 |   double *L_vbias = (double *)malloc(sizeof(double) * this->n_visible);
370 |   double *L_hbias = (double *)malloc(sizeof(double) * this->n_hidden);
371 | 
372 |   double p = 1 - corruption_level;
373 | 
374 |   dA_get_corrupted_input(this, x, tilde_x, p);
375 |   dA_get_hidden_values(this, tilde_x, y);
376 |   dA_get_reconstructed_input(this, y, z);
377 | 
378 |   // vbias
379 |   for(i=0; i<this->n_visible; i++) {
380 |     L_vbias[i] = x[i] - z[i];
381 |     this->vbias[i] += lr * L_vbias[i] / this->N;
382 |   }
383 | 
384 |   // hbias
385 |   for(i=0; i<this->n_hidden; i++) {
386 |     L_hbias[i] = 0;
387 |     for(j=0; j<this->n_visible; j++) {
388 |       L_hbias[i] += this->W[i][j] * L_vbias[j];
389 |     }
390 |     L_hbias[i] *= y[i] * (1 - y[i]);
391 | 
392 |     this->hbias[i] += lr * L_hbias[i] / this->N;
393 |   }
394 | 
395 |   // W
396 |   for(i=0; i<this->n_hidden; i++) {
397 |     for(j=0; j<this->n_visible; j++) {
398 |       this->W[i][j] += lr * (L_hbias[i] * tilde_x[j] + L_vbias[j] * y[i]) / this->N;
399 |     }
400 |   }
401 | 
402 |   free(L_hbias);
403 |   free(L_vbias);
404 |   free(z);
405 |   free(y);
406 |   free(tilde_x);
407 | }
408 | 
409 | void dA_reconstruct(dA* this, int *x, double *z) {
410 |   int i;
411 |   double *y = (double *)malloc(sizeof(double) * this->n_hidden);
412 | 
413 |   dA_get_hidden_values(this, x, y);
414 |   dA_get_reconstructed_input(this, y, z);
415 | 
416 |   free(y);
417 | }
418 | 
419 | 
420 | // LogisticRegression
421 | void LogisticRegression__construct(LogisticRegression *this, int N, int n_in, int n_out) {
422 |   int i, j;
423 |   this->N = N;
424 |   this->n_in = n_in;
425 |   this->n_out = n_out;
426 | 
427 |   this->W = (double **)malloc(sizeof(double*) * n_out);
428 |   this->W[0] = (double *)malloc(sizeof(double) * n_in * n_out);
429 |   for(i=0; i<n_out; i++) this->W[i] = this->W[0] + i * n_in;
430 |   this->b = (double *)malloc(sizeof(double) * n_out);
431 | 
432 |   for(i=0; i<n_out; i++) {
433 |     for(j=0; j<n_in; j++) {
434 |       this->W[i][j] = 0;
435 |     }
436 |     this->b[i] = 0;
437 |   }
438 | }
439 | 
440 | void LogisticRegression__destruct(LogisticRegression *this) {
441 |   free(this->W[0]);
442 |   free(this->W);
443 |   free(this->b);
444 | }
445 | 
446 | void LogisticRegression_train(LogisticRegression *this, int *x, int *y, double lr) {
447 |   int i,j;
448 |   double *p_y_given_x = (double *)malloc(sizeof(double) * this->n_out);
449 |   double *dy = (double *)malloc(sizeof(double) * this->n_out);
450 | 
451 |   for(i=0; i<this->n_out; i++) {
452 |     p_y_given_x[i] = 0;
453 |     for(j=0; j<this->n_in; j++) {
454 |       p_y_given_x[i] += this->W[i][j] * x[j];
455 |     }
456 |     p_y_given_x[i] += this->b[i];
457 |   }
458 |   LogisticRegression_softmax(this, p_y_given_x);
459 | 
460 |   for(i=0; i<this->n_out; i++) {
461 |     dy[i] = y[i] - p_y_given_x[i];
462 | 
463 |     for(j=0; j<this->n_in; j++) {
464 |       this->W[i][j] += lr * dy[i] * x[j] / this->N;
465 |     }
466 | 
467 |     this->b[i] += lr * dy[i] / this->N;
468 |   }
469 | 
470 |   free(p_y_given_x);
471 |   free(dy);
472 | }
473 | 
474 | void LogisticRegression_softmax(LogisticRegression *this, double *x) {
475 |   int i;
476 |   double max = 0.0;
477 |   double sum = 0.0;
478 | 
479 |   for(i=0; i<this->n_out; i++) if(max < x[i]) max = x[i];
480 |   for(i=0; i<this->n_out; i++) {
481 |     x[i] = exp(x[i] - max);
482 |     sum += x[i];
483 |   }
484 | 
485 |   for(i=0; i<this->n_out; i++) x[i] /= sum;
486 | }
487 | 
488 | void LogisticRegression_predict(LogisticRegression *this, int *x, double *y) {
489 |   int i,j;
490 | 
491 |   for(i=0; i<this->n_out; i++) {
492 |     y[i] = 0;
493 |     for(j=0; j<this->n_in; j++) {
494 |       y[i] += this->W[i][j] * x[j];
495 |     }
496 |     y[i] += this->b[i];
497 |   }
498 | 
499 |   LogisticRegression_softmax(this, y);
500 | }
501 | 
502 | 
503 | void test_sda(void) {
504 |   srand(0);
505 | 
506 |   int i, j;
507 | 
508 |   double pretrain_lr = 0.1;
509 |   double corruption_level = 0.3;
510 |   int pretraining_epochs = 1000;
511 |   double finetune_lr = 0.1;
512 |   int finetune_epochs = 500;
513 | 
514 |   int train_N = 10;
515 |   int test_N = 4;
516 |   int n_ins = 28;
517 |   int n_outs = 2;
518 |   int hidden_layer_sizes[] = {15, 15};
519 |   int n_layers = sizeof(hidden_layer_sizes) / sizeof(hidden_layer_sizes[0]);
520 | 
521 |   // training data
522 |   int train_X[10][28] = {
523 |     {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
524 |     {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
525 |     {1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
526 |     {0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
527 |     {1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
528 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
529 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1},
530 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1},
531 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1},
532 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1}
533 |   };
534 | 
535 |   int train_Y[10][2] = {
536 |     {1, 0},
537 |     {1, 0},
538 |     {1, 0},
539 |     {1, 0},
540 |     {1, 0},
541 |     {0, 1},
542 |     {0, 1},
543 |     {0, 1},
544 |     {0, 1},
545 |     {0, 1}
546 |   };
547 | 
548 |   // construct SdA
549 |   SdA sda;
550 |   SdA__construct(&sda, train_N, n_ins, hidden_layer_sizes, n_outs, n_layers);
551 | 
552 |   // pretrain
553 |   SdA_pretrain(&sda, *train_X, pretrain_lr, corruption_level, pretraining_epochs);
554 | 
555 |   // finetune
556 |   SdA_finetune(&sda, *train_X, *train_Y, finetune_lr, finetune_epochs);
557 | 
558 |   // test data
559 |   int test_X[4][28] = {
560 |     {1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
561 |     {1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
562 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1},
563 |     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1}
564 |   };
565 | 
566 |   double test_Y[4][28];
567 | 
568 | 
569 |   // test
570 |   for(i=0; i<test_N; i++) {
571 |     SdA_predict(&sda, test_X[i], test_Y[i]);
572 |     for(j=0; j<n_outs; j++) {
573 |       printf("%.5f ", test_Y[i][j]);
574 |     }
575 |     printf("\n");
576 |   }
577 | 
578 |   
579 |   // destruct DBN
580 |   SdA__destruct(&sda);
581 | }
582 | 
583 | 
584 | int main(void) {
585 |   test_sda();
586 |   return 0;
587 | }
588 | 


--------------------------------------------------------------------------------