├── .gitignore
├── CMakeLists.txt
├── README.md
└── src
    ├── CGOptimizer.cc
    ├── CGOptimizer.h
    ├── GP.cc
    ├── GP.h
    ├── KernelFunction.cc
    ├── KernelFunction.h
    ├── MeanFunction.cc
    ├── MeanFunction.h
    ├── SPGP.cc
    ├── SPGP.h
    ├── armadillo_backsub.cc
    ├── armadillo_backsub.h
    ├── myTypes.h
    ├── test.cc
    └── test_spgp.cc


/.gitignore:
--------------------------------------------------------------------------------
1 | /build/
2 | /src/GP_preprocessor.cc
3 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 2.8.3)
 2 | project(gp)
 3 | set(CMAKE_BUILD_TYPE Debug)
 4 | 
 5 | find_package(ARMADILLO REQUIRED)
 6 | 
 7 | find_path(LAPACK_HEADERS
 8 |   clapack.h
 9 |   HINTS /System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/Headers/ /usr/include/atlas/)
10 | 
11 | include_directories(
12 |   ${LAPACK_HEADERS}
13 |   ${ARMADILLO_INCLUDE_DIRS})
14 | 
15 | #common commands for building c++ executables and libraries
16 | add_library(${PROJECT_NAME}
17 |   src/GP.cc
18 |   src/KernelFunction.cc
19 |   src/MeanFunction.cc
20 |   src/CGOptimizer.cc
21 |   src/armadillo_backsub.cc)
22 | target_link_libraries(${PROJECT_NAME}
23 |   ${ARMADILLO_LIBRARIES}
24 |   gsl)
25 | if(APPLE)
26 |   set_target_properties(${PROJECT_NAME}
27 |     PROPERTIES LINK_FLAGS "${LINK_FLAGS} -framework Accelerate -framework vecLib")
28 | endif()
29 | 
30 | add_library(SPGP
31 |   src/SPGP.cc
32 |   )
33 | target_link_libraries(SPGP
34 |   ${PROJECT_NAME}
35 |   ${ARMADILLO_LIBRARIES}
36 |   gsl)
37 | if(APPLE)
38 |   set_target_properties(SPGP
39 |     PROPERTIES LINK_FLAGS "${LINK_FLAGS} -framework Accelerate -framework vecLib")
40 | endif()
41 | 
42 | add_executable(test_gp src/test.cc)
43 | target_link_libraries(test_gp ${ARMADILLO_LIBRARIES} ${PROJECT_NAME})
44 | if(APPLE)
45 |   set_target_properties(test_gp
46 |     PROPERTIES LINK_FLAGS "${LINK_FLAGS} -framework Accelerate")
47 | endif()
48 | 
49 | add_executable(test_spgp src/test_spgp.cc)
50 | target_link_libraries(test_spgp ${ARMADILLO_LIBRARIES} ${PROJECT_NAME} SPGP)
51 | if(APPLE)
52 |   set_target_properties(test_spgp
53 |     PROPERTIES LINK_FLAGS "${LINK_FLAGS} -framework Accelerate")
54 | endif()
55 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Gaussian Process Libary
 2 | =======================
 3 | 
 4 | This library provides a C++ implementation of Gaussian process regression as described in ["Gaussian Processes for Machine Learning"](http://www.gaussianprocess.org/gpml/) by Carl Edward Rasmussen and Christopher K. I. Williams.  The design goal of the software is to provide an easy interface with fast performance by using efficient wrappers around low-level LAPACK code.
 5 | 
 6 | Dependencies
 7 | ------------
 8 | 
 9 | * [Armadillo](http://arma.sourceforge.net/) is a C++ linear algebra library with LAPACK integration.
10 | * [Gnu Scientific Library (GSL)](http://www.gnu.org/software/gsl/) provides some non-linear optimization routines for doing hyper-parameter estimation.
11 | 
12 | Building
13 | --------
14 | Standard cmake build..
15 | 
16 |      $ mkdir build && cd build
17 |      $ cmake ..
18 |      $ make
19 | 
20 | Basic Usage
21 | -----------
22 | An example is given in 'src/test.cc'
23 | 
24 |      #include <iostream>
25 |      #include "GP.h"
26 | 
27 |      using namespace std;
28 | 
29 |      int main(int argc, char **argv)
30 |      {
31 |        Col<REAL> kernel_param = "1.0 4.0";
32 |        SqExpKernel kernel(kernel_param);
33 |        ConstantMean mean("10");
34 |        GP gp(0.1, &kernel, &mean);
35 | 
36 |        Mat<REAL> X(1,1000);
37 |        Row<REAL> y;
38 |        y.set_size(X.n_cols);
39 | 
40 |        for(int i=0; i < X.n_cols; ++i) {
41 |          X(0,i) = i;
42 |          y(i) = 10+sin(i/(2*Math<REAL>::pi()*X.n_cols));
43 |        }
44 | 
45 |        Mat<REAL> Xs = X;
46 |        Row<REAL> mu, var;
47 |        REAL mu1, var1;
48 | 
49 |        cout << "Setting training data\n";
50 |        gp.SetTraining(X, y);
51 |        cout << "Making predictions\n";
52 |        gp.Predict(Xs, mu, var);
53 |        // gp.Predict(Xs.col(0), mu1, var1);
54 | 
55 |        Col<REAL> grad;
56 |        gp.GradLikelihoodKernelParams(grad);
57 |        grad.print("Likelihood Gradient (kernel)");
58 | 
59 |        cout << "X:\n" << X << endl;
60 |        cout << "y:\n" << y << endl;
61 |        cout << "Xs:\n" << Xs << endl;
62 |        cout << "mu:\n" << mu << endl;
63 | 
64 |        Row<REAL> error = y - mu;
65 |        cout << "error:\n" << norm(error,2) << endl;
66 | 
67 |        cout << "Log-likelihood: " << gp.ComputeLikelihood() << endl;
68 | 
69 |        return 0;
70 |      }
71 | 
72 | License
73 | -------
74 | Code is licensed under the BSD license.
75 | 
76 | Copyright (c) 2011, Jon Fink
77 | All rights reserved.
78 | 
79 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
80 | 
81 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
82 | 
83 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
84 | 
85 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
86 | 


--------------------------------------------------------------------------------
/src/CGOptimizer.cc:
--------------------------------------------------------------------------------
 1 | #include <gsl/gsl_multimin.h>
 2 | #include "GP.h"
 3 | 
 4 | #include <iostream>
 5 | using namespace std;
 6 | 
 7 | #include "CGOptimizer.h"
 8 | 
 9 | CGOptimizer::CGOptimizer(void *gp)
10 | {
11 |   this->gp = gp;
12 | }
13 | 
14 | void CGOptimizer::Initialize(const Col<REAL> &init, 
15 | 			     double (*f_eval)(const gsl_vector*, void*), 
16 | 			     void (*df_eval)(const gsl_vector*, void*, gsl_vector*),
17 | 			     void (*fdf_eval)(const gsl_vector*, void*, double*, gsl_vector*),
18 | 			     double step, double eps)
19 | {
20 |   my_func.n = init.n_elem;
21 |   my_func.f = f_eval;
22 |   my_func.df = df_eval;
23 |   my_func.fdf = fdf_eval;
24 |   my_func.params = (void*)this->gp;
25 | 
26 |   x = gsl_vector_alloc(init.n_elem);
27 |   for(unsigned int i=0; i < init.n_elem; ++i)
28 |     gsl_vector_set(x, i, init(i));
29 | 
30 |   T = gsl_multimin_fdfminimizer_conjugate_fr;
31 |   s = gsl_multimin_fdfminimizer_alloc (T, init.n_elem);
32 | 
33 |   gsl_multimin_fdfminimizer_set (s, &my_func, x, step, eps);
34 |   //gsl_multimin_fdfminimizer_set (s, &my_func, x, 0.1, 1e-2);
35 | }
36 | 
37 | int CGOptimizer::Optimize(Col<REAL> &soln, int max_iterations)
38 | {
39 |   int iter = 0;
40 |   int status;
41 | 
42 |   do {
43 |     iter++;
44 |     status = gsl_multimin_fdfminimizer_iterate (s);
45 |      
46 |     if (status) {
47 |       printf("Dropping out of optimization _before_ gradient test\n");
48 |       break;
49 |     }
50 |      
51 |     status = gsl_multimin_test_gradient (s->gradient, 1e-3);
52 |      
53 |     if (status == GSL_SUCCESS)
54 |       printf ("Minimum found at:\n");
55 | 
56 |     printf("%5d ", iter);
57 |     for(unsigned int i=0; i < my_func.n; ++i)
58 |       printf("%.2f ", gsl_vector_get(s->x, i));
59 |     printf("%10.5f\n", s->f);
60 |   }
61 |   while (status == GSL_CONTINUE && iter < max_iterations);
62 | 
63 |   soln.set_size(my_func.n);
64 |   for(unsigned int i=0; i < my_func.n; ++i) {
65 |     soln(i) = gsl_vector_get(s->x, i);
66 |   }
67 | 
68 |   printf("status = %d\n", status);
69 |   switch(status) {
70 |   case GSL_SUCCESS:
71 |     printf("GSL_SUCCESS\n");
72 |     break;
73 |   case GSL_CONTINUE:
74 |     printf("GSL_CONTINUE (failed on iterations)\n");
75 |     break;
76 |   };
77 | 
78 |   if((status == GSL_SUCCESS) || (status == GSL_CONTINUE))
79 |     return 0;
80 | 
81 |   return -1;
82 | }
83 | 


--------------------------------------------------------------------------------
/src/CGOptimizer.h:
--------------------------------------------------------------------------------
 1 | #ifndef _COGOPTIMIZER_H_
 2 | #define _COGOPTIMIZER_H_
 3 | 
 4 | #include <gsl/gsl_math.h>
 5 | #include <gsl/gsl_multimin.h>
 6 | 
 7 | class GP;
 8 | 
 9 | class CGOptimizer {
10 |  public:
11 |   CGOptimizer(void *gp);
12 |   
13 |   void Initialize(const Col<REAL> &init, 
14 | 		  double (*f_eval)(const gsl_vector*, void*), 
15 | 		  void (*df_eval)(const gsl_vector*, void*, gsl_vector*),
16 | 		  void (*fdf_eval)(const gsl_vector*, void*, double*, gsl_vector*),
17 | 		  double step=10.0, double eps=0.1);
18 | 
19 |   int Optimize(Col<REAL> &soln, int max_iterations=10);
20 | 
21 |  private:
22 |   void *gp;
23 |   const gsl_multimin_fdfminimizer_type *T;
24 |   gsl_multimin_fdfminimizer *s;
25 |   gsl_multimin_function_fdf my_func;
26 | 
27 |   gsl_vector *x;
28 | };
29 | 
30 | #endif
31 | 


--------------------------------------------------------------------------------
/src/GP.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | using namespace std;
  3 | 
  4 | //#define BENCHMARK
  5 | 
  6 | #include "GP.h"
  7 | #include "CGOptimizer.h"
  8 | #include "armadillo_backsub.h"
  9 | 
 10 | GP::GP(REAL s2_n, KernelFunction *kernel, MeanFunction *mean)
 11 | {
 12 |   this->kernel = kernel;
 13 |   this->mean = mean;
 14 |   this->s2_n = s2_n;
 15 |   this->need_to_compute_alpha = false;
 16 |   this->need_to_compute_chol = false;
 17 | }
 18 | 
 19 | void GP::SetKernelFuncParams(const Col<REAL>& param)
 20 | {
 21 |   this->kernel->SetParams(param);
 22 | 
 23 |   MatrixMap(this->K, this->X, this->X);
 24 |   K += this->s2_n*eye<Mat<REAL> >(X.n_cols, X.n_cols);
 25 | 
 26 |   this->need_to_compute_alpha = true;
 27 |   this->need_to_compute_chol = true;
 28 | }
 29 | 
 30 | 
 31 | void GP::SetMeanFuncParams(const Col<REAL>& param)
 32 | {
 33 |   this->mean->SetParams(param);
 34 | 
 35 |   this->meanvals.set_size(this->X.n_cols);
 36 |   for(unsigned int i=0; i < this->X.n_cols; ++i)
 37 |     this->meanvals(i) = this->mean->Eval(this->X.col(i));
 38 | 
 39 |   this->need_to_compute_alpha = true;
 40 | }
 41 | 
 42 | void GP::SetNoise(const REAL &s2_n)
 43 | {
 44 |   K -= this->s2_n*eye<Mat<REAL> >(X.n_cols, X.n_cols);
 45 |   this->s2_n = s2_n;
 46 |   K += this->s2_n*eye<Mat<REAL> >(X.n_cols, X.n_cols);
 47 | 
 48 |   this->need_to_compute_alpha = true;
 49 |   this->need_to_compute_chol = true;
 50 | }
 51 | 
 52 | void GP::SetTraining(const Mat<REAL>& X, const Row<REAL> &y)
 53 | {
 54 |   this->y.set_size(y.n_cols);
 55 | 
 56 |   MatrixMap(K, X, X);
 57 |   this->X = X;
 58 |   this->y = y;
 59 | 
 60 |   this->meanvals.set_size(X.n_cols);
 61 |   for(unsigned int i=0; i < X.n_cols; ++i)
 62 |     this->meanvals(i) = this->mean->Eval(X.col(i));
 63 | 
 64 |   K += this->s2_n*eye<Mat<REAL> >(X.n_cols, X.n_cols);
 65 | 
 66 |   this->need_to_compute_alpha = true;
 67 |   this->need_to_compute_chol = true;
 68 | }
 69 | 
 70 | void GP::AddTraining(const Mat<REAL>& X, const Row<REAL> &y)
 71 | {
 72 |   int orig_n_cols, orig_n_rows;
 73 |   int new_n_cols, new_n_rows;
 74 |   // Extend y
 75 |   orig_n_cols = this->y.n_cols;
 76 |   new_n_cols = orig_n_cols + y.n_cols;
 77 |   this->y.set_size(new_n_cols);
 78 |   this->y.cols(orig_n_cols, new_n_cols-1) = y;
 79 | 
 80 |   // Extend & Compute meanvals
 81 |   orig_n_cols = this->meanvals.n_cols;
 82 |   new_n_cols = orig_n_cols + X.n_cols;
 83 |   this->meanvals.set_size(new_n_cols);
 84 |   for(unsigned int i=0; i < X.n_cols; ++i)
 85 |     this->meanvals(orig_n_cols + i) = this->mean->Eval(X.col(i));
 86 | 
 87 |   // Extend X
 88 |   orig_n_cols = this->X.n_cols;
 89 |   orig_n_rows = this->X.n_rows;
 90 |   new_n_cols = orig_n_cols + X.n_cols;
 91 |   new_n_rows = orig_n_rows;
 92 |   this->X.set_size(new_n_rows, new_n_cols);
 93 |   this->X.submat(0, orig_n_cols, new_n_rows-1, new_n_cols-1) = X;
 94 | 
 95 |   MatrixMap(this->K, this->X, this->X);
 96 |   this->K += this->s2_n*eye<Mat<REAL> >(this->X.n_cols, this->X.n_cols);
 97 | 
 98 |   this->need_to_compute_alpha = true;
 99 |   this->need_to_compute_chol = true;
100 | }
101 | 
102 | void GP::ComputeChol()
103 | {
104 |   if(this->need_to_compute_chol) {
105 | #ifdef BENCHMARK
106 |     wall_clock timer;
107 |     timer.tic();
108 | #endif
109 |     L = chol(K);
110 |     this->need_to_compute_w = true;
111 |     this->need_to_compute_chol = false;
112 | #ifdef BENCHMARK
113 |     printf("%f seconds to compute cholesky\n", timer.toc());
114 | #endif
115 |   }
116 | }
117 | 
118 | void GP::ComputeAlpha()
119 | {
120 |   if(this->need_to_compute_alpha) {
121 |     // Ensure that L is up to date
122 |     ComputeChol();
123 | #ifdef BENCHMARK
124 |     wall_clock timer;
125 |     timer.tic();
126 | #endif
127 |     this->alpha.set_size(y.n_cols);
128 |     cholbacksub(alpha, this->L, trans(this->y - this->meanvals));
129 |     this->need_to_compute_alpha = false;
130 | #ifdef BENCHMARK
131 |     printf("%f seconds to compute alpha\n", timer.toc());
132 | #endif
133 |   }
134 | }
135 | 
136 | void GP::ComputeW()
137 | {
138 |   if(this->need_to_compute_w) {
139 |     ComputeChol();
140 |     this->W.set_size(this->L.n_rows, this->L.n_cols);
141 |     cholbacksub(W, this->L, eye<Mat<REAL> >(K.n_rows, K.n_cols));
142 |     this->need_to_compute_w = false;
143 |   }
144 | }
145 | 
146 | REAL GP::ComputeLikelihood()
147 | {
148 |   ComputeChol();
149 |   ComputeAlpha();
150 | 
151 |   this->loglikelihood = -0.5*dot( (this->y-this->meanvals), this->alpha);
152 |   this->loglikelihood += -accu(log(L.diag()));
153 |   this->loglikelihood += -0.5*this->y.n_elem*log(2*Math<REAL>::pi());
154 | 
155 |   return this->loglikelihood;
156 | }
157 | 
158 | void GP::GradLikelihoodMeanParams(Col<REAL> &grad)
159 | {
160 |   ComputeAlpha();
161 |   this->mean->Grad(grad, X.col(0));
162 | 
163 |   Col<REAL> tmp;
164 |   Mat<REAL> grad_tmp(grad.n_rows, X.n_cols);
165 |   for(unsigned int i=0; i < X.n_cols; ++i) {
166 |     this->mean->Grad(tmp, X.col(i));
167 |     grad_tmp.col(i) = -tmp;
168 |   }
169 | 
170 |   grad = grad_tmp*this->alpha;
171 | }
172 | 
173 | void GP::HessianLikelihoodMeanParams(Mat<REAL> &hessian)
174 | {
175 |   ComputeAlpha();
176 |   ComputeW();
177 | 
178 |   Col<REAL> grad;
179 |   this->mean->Grad(grad, X.col(0));
180 | 
181 |   // Get Gradient and hessian of mean function at each training point X
182 |   // with current parameters \theta
183 |   Mat<REAL> grad_all(grad.n_elem, X.n_cols);
184 |   std::vector<Col<REAL> > grad_tmp(X.n_cols);
185 |   std::vector<Mat<REAL> > hessian_tmp(X.n_cols);
186 |   for(unsigned int i=0; i < X.n_cols; ++i) {
187 |     grad_tmp[i].set_size(grad.n_rows);
188 |     hessian_tmp[i].set_size(grad.n_rows, grad.n_rows);
189 | 
190 |     this->mean->Grad(grad_tmp[i], X.col(i));
191 |     grad_all.col(i) = grad_tmp[i];
192 |     this->mean->Hessian(hessian_tmp[i], X.col(i));
193 |   }
194 | 
195 |   // grad_tmp[k] is the gradient of m(X_i, theta)
196 |   // hessian_tmp[i] is the hessian of m(X_i, theta)
197 | 
198 |   hessian.set_size(grad.n_rows, grad.n_rows);
199 |   hessian.fill(0);
200 |   for(unsigned int i=0; i < grad.n_rows; ++i) {
201 |     for(unsigned int j=0; j < grad.n_rows; ++j) {
202 | 
203 |       for(unsigned int k=0; k < alpha.n_elem; ++k) {
204 |         hessian(i,j) += hessian_tmp[k](i,j)*alpha(k);
205 |       }
206 | 
207 |       hessian(i,j) -= dot(grad_all.row(i), this->W*trans(grad_all.row(j)));
208 |     }
209 |   }
210 | }
211 | 
212 | void GP::GradLikelihoodKernelParams(Col<REAL> &grad)
213 | {
214 |   ComputeAlpha();
215 |   ComputeW();
216 | 
217 |   // Do this to get the number of params
218 |   this->kernel->Grad(grad, X.col(0), X.col(1));
219 | 
220 |   std::vector<Mat<REAL> > partialK(grad.n_elem);
221 | 
222 |   for(unsigned int k=0; k < grad.n_elem; ++k) {
223 |     partialK[k].set_size(X.n_cols, X.n_cols);
224 |   }
225 | 
226 |   for(unsigned int i=0; i < X.n_cols; ++i) {
227 |     for(unsigned int j=0; j < X.n_cols; ++j) {
228 |       this->kernel->Grad(grad, X.col(i), X.col(j));
229 |       for(unsigned int k=0; k < grad.n_elem; ++k) {
230 |         partialK[k](i,j) = grad(k);
231 |       }
232 |     }
233 |   }
234 | 
235 |   for(unsigned int k=0; k < grad.n_elem; ++k) {
236 |     Mat<REAL> tmp = trans(alpha)*partialK[k]*alpha;
237 |     grad(k) = 0.5*(tmp(0,0)) - 0.5*trace(W*partialK[k]);
238 |   }
239 | }
240 | 
241 | void GP::GradLikelihoodNoise(Col<REAL> &grad)
242 | {
243 |   ComputeW();
244 |   grad.set_size(1);
245 | 
246 |   Mat<REAL> partialK = 2*this->s2_n*eye<Mat<REAL> >(this->X.n_cols, this->X.n_cols);
247 |   Mat<REAL> tmp = trans(alpha)*partialK*alpha;
248 |   grad(0) = 0.5*(tmp(0,0)) - 0.5*trace(W*partialK);
249 | }
250 | 
251 | void GP::Predict(const Mat<REAL> &Xs, Row<REAL> &mu)
252 | {
253 | #ifdef BENCHMARK
254 |   wall_clock timer;
255 |   timer.tic();
256 | #endif
257 | 
258 |   mu.set_size(Xs.n_cols);
259 | 
260 |   REAL mu1;
261 | 
262 |   for(unsigned int i=0; i < Xs.n_cols; ++i) {
263 |     this->Predict(Xs.col(i), mu1);
264 |     if(isinf(mu1)) {
265 |       printf("INF?!?: %f %f\n", Xs(0,i), Xs(1,i));
266 |     }
267 |     mu(i) = mu1;
268 |   }
269 | 
270 | #ifdef BENCHMARK
271 |   printf("%f seconds to predict\n", timer.toc());
272 | #endif
273 | }
274 | 
275 | void GP::Predict(const Mat<REAL> &Xs, Row<REAL> &mu, Row<REAL> &var)
276 | {
277 | 
278 | #ifdef BENCHMARK
279 |   wall_clock timer;
280 |   timer.tic();
281 | #endif
282 | 
283 |   mu.set_size(Xs.n_cols);
284 |   var.set_size(Xs.n_cols);
285 | 
286 |   REAL mu1, var1;
287 | 
288 |   for(unsigned int i=0; i < Xs.n_cols; ++i) {
289 |     this->Predict(Xs.col(i), mu1, var1);
290 |     mu(i) = mu1;
291 |     var(i) = var1;
292 |   }
293 | 
294 | #ifdef BENCHMARK
295 |   printf("%f seconds to predict (with var)\n", timer.toc());
296 | #endif
297 | }
298 | 
299 | void GP::Predict(const Col<REAL> &Xs, REAL &mu)
300 | {
301 |   ComputeChol();
302 |   ComputeAlpha();
303 | 
304 |   Col<REAL> kstar(this->X.n_cols);
305 |   for(unsigned int i=0; i < this->X.n_cols; ++i)
306 |     kstar(i) = this->kernel->Eval(Xs, this->X.col(i));
307 | 
308 |   mu = dot(kstar,alpha) + this->mean->Eval(Xs);
309 | }
310 | 
311 | void GP::Predict(const Col<REAL> &Xs, REAL &mu, REAL &var)
312 | {
313 |   ComputeChol();
314 |   ComputeAlpha();
315 | 
316 |   Col<REAL> kstar(X.n_cols);
317 |   for(unsigned int i=0; i < X.n_cols; ++i)
318 |     kstar(i) = this->kernel->Eval(Xs, X.col(i));
319 | 
320 |   mu = dot(kstar,alpha) + this->mean->Eval(Xs);
321 | 
322 |   // trans(L) is LOWER TRIANGULAR
323 |   Col<REAL> v(kstar.n_elem);
324 |   solve_tri(v, trans(L), kstar, false);
325 |   REAL kxsxs = this->kernel->Eval(Xs, Xs);
326 |   var = kxsxs - dot(v, v) + this->s2_n;
327 | }
328 | 
329 | void GP::PredictGradient(const Col<REAL> &Xs, Col<REAL> &grad)
330 | {
331 |   grad.set_size(Xs.n_elem);
332 | 
333 |   ComputeChol();
334 |   ComputeAlpha();
335 | 
336 |   Mat<REAL> dkstar(grad.n_rows, X.n_cols);
337 |   for(unsigned int i=0; i < X.n_cols; ++i) {
338 |     Col<REAL> tmp;
339 |     this->kernel->GradX(tmp, Xs, X.col(i));
340 |     dkstar.col(i) = tmp;
341 |   }
342 | 
343 |   this->mean->GradX(grad, Xs);
344 | 
345 |   for(unsigned int i=0; i < grad.n_elem; ++i) {
346 |     grad(i) += dot(dkstar.row(i), alpha);
347 |   }
348 | }
349 | 
350 | void GP::PredictGradient(const Col<REAL> &Xs, Col<REAL> &grad, Col<REAL> &vargrad)
351 | {
352 |   grad.set_size(Xs.n_elem);
353 |   vargrad.set_size(Xs.n_elem);
354 | 
355 |   ComputeChol();
356 |   ComputeAlpha();
357 | 
358 | 
359 |   Col<REAL> kstar(X.n_cols);
360 |   Mat<REAL> dkstar(grad.n_rows, X.n_cols);
361 |   for(unsigned int i=0; i < X.n_cols; ++i) {
362 |     Col<REAL> tmp;
363 |     this->kernel->GradX(tmp, Xs, X.col(i));
364 |     dkstar.col(i) = tmp;
365 | 
366 |     kstar(i) = this->kernel->Eval(Xs, X.col(i));
367 |   }
368 | 
369 |   this->mean->GradX(grad, Xs);
370 | 
371 |   for(unsigned int i=0; i < grad.n_elem; ++i) {
372 |     grad(i) += dot(dkstar.row(i), alpha);
373 |   }
374 | 
375 |   Col<REAL> v(kstar.n_elem);
376 |   Col<REAL> tmp(kstar.n_elem);
377 |   Mat<REAL> dv(kstar.n_elem, grad.n_elem);
378 |   solve_tri(v, trans(L), kstar, false);
379 |   
380 |   //printf("About to solve for dv\n");
381 |   solve_tri(tmp, trans(L), trans(dkstar.row(0)), false);
382 |   dv.col(0) = tmp;
383 |   solve_tri(tmp, trans(L), trans(dkstar.row(1)), false);
384 |   dv.col(1) = tmp;
385 |   //dv.print("dv: ");
386 | 
387 |   Col<REAL> dkxsxs;
388 |   this->kernel->GradX(dkxsxs, Xs, Xs);
389 | 
390 |   vargrad = dkxsxs - 2*trans(dv)*v;
391 |   //v.print("v: ");
392 | }
393 | 
394 | void GP::MatrixMap(Mat<REAL> &matrix, const Mat<REAL> &a, const Mat<REAL> &b)
395 | {
396 |   matrix.set_size(a.n_cols, b.n_cols);
397 |   for(unsigned int i=0; i < a.n_cols; ++i) {
398 |     for(unsigned int j=0; j < b.n_cols; ++j) {
399 |       REAL val = this->kernel->Eval(a.col(i), b.col(j));
400 |       matrix(i,j) = val;
401 |     }
402 |   }
403 | }
404 | 
405 | void GP::OptimizeNoiseParam(REAL &noise_param, int max_iterations)
406 | {
407 |   CGOptimizer opt(reinterpret_cast<void*>(this));
408 | 
409 |   Col<REAL> noise_params(1);
410 |   noise_params(0) = noise_param;
411 | 
412 |   opt.Initialize(noise_params, &f_eval_noise, &df_eval_noise, &fdf_eval_noise, 0.5, 0.1);
413 |   opt.Optimize(noise_params, max_iterations);
414 | }
415 | 
416 | void GP::OptimizeMeanParam(Col<REAL> &mean_param, int max_iterations)
417 | {
418 |   CGOptimizer opt(reinterpret_cast<void*>(this));
419 | 
420 |   opt.Initialize(mean_param, &f_eval_mean, &df_eval_mean, &fdf_eval_mean, 1.5, 0.01);
421 |   opt.Optimize(mean_param, max_iterations);
422 | }
423 | 
424 | void GP::OptimizeKernelParam(Col<REAL> &kernel_param, int max_iterations)
425 | {
426 |   CGOptimizer opt(reinterpret_cast<void*>(this));
427 | 
428 |   opt.Initialize(kernel_param, &f_eval_kernel, &df_eval_kernel, &fdf_eval_kernel, 20, 0.1);
429 |   opt.Optimize(kernel_param, max_iterations);
430 | }
431 | 
432 | double f_eval_mean(const gsl_vector *x, void *param)
433 | {
434 |   GP *gp_obj = reinterpret_cast<GP*>(param);
435 | 
436 |   Col<REAL> mean_param(gp_obj->GetMeanFunction()->GetParamDim());
437 |   for(unsigned int i=0; i < mean_param.n_elem; ++i) {
438 |     mean_param(i) = gsl_vector_get(x, i);
439 |   }
440 | 
441 |   gp_obj->SetMeanFuncParams(mean_param);
442 | 
443 |   double ret = -gp_obj->ComputeLikelihood();
444 | 
445 |   return ret;
446 | }
447 | 
448 | void df_eval_mean(const gsl_vector *x, void *param, gsl_vector *g)
449 | {
450 |   GP *gp_obj = reinterpret_cast<GP*>(param);
451 | 
452 |   Col<REAL> mean_param(gp_obj->GetMeanFunction()->GetParamDim());
453 |   for(unsigned int i=0; i < mean_param.n_elem; ++i) {
454 |     mean_param(i) = gsl_vector_get(x, i);
455 |   }
456 |   gp_obj->SetMeanFuncParams(mean_param);
457 | 
458 |   Col<REAL> grad;
459 |   gp_obj->GradLikelihoodMeanParams(grad);
460 | 
461 |   for(unsigned int i=0; i < grad.n_elem; ++i) {
462 |     gsl_vector_set(g, i, -grad(i));
463 |   }
464 | }
465 | 
466 | void fdf_eval_mean(const gsl_vector *x, void *param, double *f, gsl_vector *g)
467 | {
468 |   *f = f_eval_mean(x, param);
469 |   df_eval_mean(x, param, g);
470 | }
471 | 
472 | double f_eval_kernel(const gsl_vector *x, void *param)
473 | {
474 |   GP *gp_obj = reinterpret_cast<GP*>(param);
475 | 
476 |   Col<REAL> kernel_param(gp_obj->GetKernelFunction()->GetParamDim());
477 |   for(unsigned int i=0; i < kernel_param.n_elem; ++i) {
478 |     kernel_param(i) = gsl_vector_get(x, i);
479 |     if(kernel_param(i) < 1e-6) {
480 |       return 1e6;
481 |     }
482 |   }
483 | 
484 |   gp_obj->SetKernelFuncParams(kernel_param);
485 | 
486 |   double ret = -gp_obj->ComputeLikelihood();
487 | 
488 |   return ret;
489 | }
490 | 
491 | void df_eval_kernel(const gsl_vector *x, void *param, gsl_vector *g)
492 | {
493 |   GP *gp_obj = reinterpret_cast<GP*>(param);
494 | 
495 |   Col<REAL> kernel_param(gp_obj->GetKernelFunction()->GetParamDim());
496 |   for(unsigned int i=0; i < kernel_param.n_elem; ++i) {
497 |     kernel_param(i) = gsl_vector_get(x, i);
498 |   }
499 |   gp_obj->SetKernelFuncParams(kernel_param);
500 | 
501 |   Col<REAL> grad;
502 |   gp_obj->GradLikelihoodKernelParams(grad);
503 | 
504 |   for(unsigned int i=0; i < grad.n_elem; ++i) {
505 |     gsl_vector_set(g, i, -grad(i));
506 |   }
507 | }
508 | 
509 | void fdf_eval_kernel(const gsl_vector *x, void *param, double *f, gsl_vector *g)
510 | {
511 |   *f = f_eval_kernel(x, param);
512 |   df_eval_kernel(x, param, g);
513 | }
514 | 
515 | double f_eval_noise(const gsl_vector *x, void *param)
516 | {
517 |   GP *gp_obj = reinterpret_cast<GP*>(param);
518 | 
519 |   REAL noise_param;
520 |   noise_param = gsl_vector_get(x, 0);
521 |   if(noise_param < 1e-6) {
522 |     return 1e6;
523 |   }
524 | 
525 |   gp_obj->SetNoise(noise_param);
526 | 
527 |   double ret = -gp_obj->ComputeLikelihood();
528 | 
529 |   return ret;
530 | }
531 | 
532 | void df_eval_noise(const gsl_vector *x, void *param, gsl_vector *g)
533 | {
534 |   GP *gp_obj = reinterpret_cast<GP*>(param);
535 | 
536 |   REAL noise_param;
537 |   noise_param = gsl_vector_get(x, 0);
538 | 
539 |   gp_obj->SetNoise(noise_param);
540 | 
541 |   Col<REAL> grad;
542 |   gp_obj->GradLikelihoodNoise(grad);
543 | 
544 |   for(unsigned int i=0; i < grad.n_elem; ++i) {
545 |     gsl_vector_set(g, i, -grad(i));
546 |   }
547 | }
548 | 
549 | void fdf_eval_noise(const gsl_vector *x, void *param, double *f, gsl_vector *g)
550 | {
551 |   *f = f_eval_noise(x, param);
552 |   df_eval_noise(x, param, g);
553 | }
554 | 
555 | 


--------------------------------------------------------------------------------
/src/GP.h:
--------------------------------------------------------------------------------
 1 | #ifndef _GP_H_
 2 | #define _GP_H_
 3 | 
 4 | #include "armadillo"
 5 | using namespace arma;
 6 | #include "myTypes.h"
 7 | 
 8 | #include "KernelFunction.h"
 9 | #include "MeanFunction.h"
10 | 
11 | #include <gsl/gsl_math.h>
12 | #include <gsl/gsl_multimin.h>
13 | 
14 | double f_eval_mean(const gsl_vector *x, void *param);
15 | void df_eval_mean(const gsl_vector *x, void *param, gsl_vector *g);
16 | void fdf_eval_mean(const gsl_vector *x, void *param, double *f, gsl_vector *g);
17 | 
18 | double f_eval_kernel(const gsl_vector *x, void *param);
19 | void df_eval_kernel(const gsl_vector *x, void *param, gsl_vector *g);
20 | void fdf_eval_kernel(const gsl_vector *x, void *param, double *f, gsl_vector *g);
21 | 
22 | double f_eval_noise(const gsl_vector *x, void *param);
23 | void df_eval_noise(const gsl_vector *x, void *param, gsl_vector *g);
24 | void fdf_eval_noise(const gsl_vector *x, void *param, double *f, gsl_vector *g);
25 | 
26 | class GP {
27 | public:
28 |   GP(REAL s2_n, KernelFunction *kernel, MeanFunction *mean);
29 |   
30 |   void SetTraining(const Mat<REAL>& X, const Row<REAL> &y);
31 |   void AddTraining(const Mat<REAL>& X, const Row<REAL> &y);
32 |   void Predict(const Mat<REAL> &Xs, Row<REAL> &mu);
33 |   void Predict(const Mat<REAL> &Xs, Row<REAL> &mu, Row<REAL> &var);
34 |   void Predict(const Col<REAL> &Xs, REAL &mu);
35 |   void Predict(const Col<REAL> &Xs, REAL &mu, REAL &var);
36 | 
37 |   void PredictGradient(const Col<REAL> &Xs, Col<REAL> &grad);
38 |   void PredictGradient(const Col<REAL> &Xs, Col<REAL> &grad, Col<REAL> &vargrad);
39 | 
40 |   void ComputeAlpha();
41 |   void ComputeChol();
42 |   void ComputeW();
43 |   REAL ComputeLikelihood();
44 | 
45 |   void SetKernelFuncParams(const Col<REAL>& param);
46 |   void SetMeanFuncParams(const Col<REAL>& param);
47 |   void SetNoise(const REAL &s2_n);
48 | 
49 |   void GradLikelihoodMeanParams(Col<REAL> &grad);
50 |   void GradLikelihoodKernelParams(Col<REAL> &grad);
51 |   void GradLikelihoodNoise(Col<REAL> &grad);
52 | 
53 |   void HessianLikelihoodMeanParams(Mat<REAL> &hessian);
54 | 
55 |   void OptimizeNoiseParam(REAL &noise_param, int max_iterations=10);
56 |   void OptimizeMeanParam(Col<REAL> &mean_param, int max_iterations=10);
57 |   void OptimizeKernelParam(Col<REAL> &kernel_param, int max_iterations=10);
58 | 
59 |   inline Mat<REAL> &GetTrainingData() { return X; }
60 |   inline KernelFunction *GetKernelFunction() { return this->kernel; }
61 |   inline MeanFunction *GetMeanFunction() { return this->mean; }
62 |   
63 | protected:
64 |   KernelFunction *kernel;
65 |   MeanFunction *mean;
66 |   Mat<REAL> K;
67 |   Mat<REAL> W;
68 |   Mat<REAL> X;
69 |   Mat<REAL> L;
70 |   Row<REAL> y;
71 |   Row<REAL> meanvals;
72 |   Col<REAL> alpha;
73 |   REAL s2_n;
74 |   REAL loglikelihood;
75 | 
76 |   bool need_to_compute_alpha;
77 |   bool need_to_compute_chol;
78 |   bool need_to_compute_w;
79 | 
80 |   void MatrixMap(Mat<REAL> &matrix, const Mat<REAL> &a, const Mat<REAL> &b);
81 | };
82 | 
83 | #endif
84 | 


--------------------------------------------------------------------------------
/src/KernelFunction.cc:
--------------------------------------------------------------------------------
 1 | #include "KernelFunction.h"
 2 | 
 3 | SqExpKernel::SqExpKernel()
 4 | {
 5 |   this->SetParams("1 1");
 6 |   this->param_dim = param.n_elem;
 7 | }
 8 | 
 9 | SqExpKernel::SqExpKernel(const Col<REAL>& param)
10 | {
11 |   this->SetParams(param);
12 |   this->param_dim = param.n_elem;
13 | }
14 | 
15 | REAL SqExpKernel::Eval(const Col<REAL>& a, const Col<REAL> &b)
16 | {
17 |   REAL dist = norm(a-b, 2);
18 |   return param(0)*exp(-(dist*dist)/(2*param(1)*param(1)));
19 | }
20 | 
21 | void SqExpKernel::Grad(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b)
22 | {
23 |   REAL dist = norm(a-b, 2);
24 |   grad.set_size(param.n_elem);
25 |   grad(0) = exp(-(dist*dist)/(2*param(1)*param(1)));
26 |   grad(1) = dist*dist*param(0)*(1/pow(param(1),3))*exp(-(dist*dist)/(2*param(1)*param(1)));
27 | }
28 | 
29 | void SqExpKernel::GradX(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b)
30 | {
31 |   REAL dist = norm(a-b, 2);
32 |   grad.set_size(a.n_elem);
33 | 
34 |   if(dist < 1e-6) {
35 |     grad.fill(0);
36 |   }
37 |   else {
38 |     grad(0) = -((a(0) - b(0))*param(0))*exp(-(dist*dist)/(2*param(1)*param(1)))/
39 |       (2*param(1)*param(1)*dist);
40 |     grad(1) = -((a(1) - b(1))*param(0))*exp(-(dist*dist)/(2*param(1)*param(1)))/
41 |       (2*param(1)*param(1)*dist);
42 |   }
43 | }
44 | 
45 | SqExpCircleKernel::SqExpCircleKernel()
46 | {
47 |   this->SetParams("1 1");
48 |   this->param_dim = param.n_elem;
49 | }
50 | 
51 | SqExpCircleKernel::SqExpCircleKernel(const Col<REAL>& param)
52 | {
53 |   this->SetParams(param);
54 |   this->param_dim = param.n_elem;
55 | }
56 | 
57 | REAL SqExpCircleKernel::Eval(const Col<REAL>& a, const Col<REAL> &b)
58 | {
59 |   if(a.n_elem != 1 || b.n_elem != 1) {
60 |     fprintf(stderr, "SqExpCircleKernel only works for one-dimensional angles");
61 |     return 0;
62 |   }
63 | 
64 |   Col<REAL> aPt(2), bPt(2);
65 |   aPt(0) = cos(a(0)); aPt(1) = sin(a(0));
66 |   bPt(0) = cos(b(0)); bPt(1) = sin(b(0));
67 | 
68 |   REAL dist = norm(aPt-bPt, 2);
69 |   return param(0)*exp(-1/(2*param(1)*param(1))*dist);
70 | }
71 | 
72 | void SqExpCircleKernel::Grad(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b)
73 | {
74 |   grad.set_size(param.n_elem);
75 |   grad.fill(0);
76 | }
77 | 
78 | void SqExpCircleKernel::GradX(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b)
79 | {
80 |   grad.set_size(a.n_elem);
81 |   grad.fill(0);
82 | 
83 | }
84 | 


--------------------------------------------------------------------------------
/src/KernelFunction.h:
--------------------------------------------------------------------------------
 1 | #ifndef _KERNEL_FUNCTION_
 2 | #define _KERNEL_FUNCTION_
 3 | 
 4 | #include "armadillo"
 5 | using namespace arma;
 6 | 
 7 | #include "myTypes.h"
 8 | 
 9 | class KernelFunction {
10 |  public:
11 |   virtual ~KernelFunction() {}
12 |   virtual REAL Eval(const Col<REAL>& a, const Col<REAL> &b)=0;
13 |   virtual void Grad(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b)=0;
14 |   virtual void SetParams(const Col<REAL>& param) {
15 |     this->param = param;
16 |   }
17 | 
18 |   virtual Col<REAL> GetParams() {
19 |     return this->param;
20 |   }
21 | 
22 |   virtual void GradX(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b)=0;
23 |   int GetParamDim() { return this->param_dim; }
24 | 
25 |  protected:
26 |   int param_dim;
27 |   Col<REAL> param;
28 | };
29 | 
30 | class SqExpKernel : public KernelFunction {
31 |  public:
32 |   SqExpKernel();
33 |   SqExpKernel(const Col<REAL>& param);
34 |   REAL Eval(const Col<REAL>& a, const Col<REAL> &b);
35 |   void Grad(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b);
36 |   void GradX(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b);
37 | };
38 | 
39 | class SqExpCircleKernel : public KernelFunction {
40 |  public:
41 |   SqExpCircleKernel();
42 |   SqExpCircleKernel(const Col<REAL>& param);
43 |   REAL Eval(const Col<REAL>& a, const Col<REAL> &b);
44 |   void Grad(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b);
45 |   void GradX(Col<REAL> &grad, const Col<REAL>& a, const Col<REAL>& b);
46 | };
47 | 
48 | #endif
49 | 


--------------------------------------------------------------------------------
/src/MeanFunction.cc:
--------------------------------------------------------------------------------
  1 | #include "MeanFunction.h"
  2 | 
  3 | void MeanFunction::SetParams(const Col<REAL>& param)
  4 | {
  5 |   this->param = param;
  6 | }
  7 | 
  8 | Col<REAL> MeanFunction::GetParams()
  9 | {
 10 |   return this->param;
 11 | }
 12 | 
 13 | 
 14 | void MeanFunction::SetGradMult(const Col<REAL>& grad_mult)
 15 | {
 16 |   this->grad_mult = grad_mult;
 17 | }
 18 | 
 19 | int MeanFunction::GetParamDim() { return this->param_dim; }
 20 | 
 21 | ConstantMean::ConstantMean()
 22 | {
 23 |   this->param.set_size(1);
 24 |   param(0) = 0;
 25 |   this->param_dim = param.n_elem;
 26 |   this->grad_mult.set_size(param.n_elem);
 27 |   this->grad_mult.fill(1.0);
 28 | }
 29 | 
 30 | ConstantMean::ConstantMean(const Col<REAL>& param)
 31 | {
 32 |   this->SetParams(param);
 33 |   this->param_dim = param.n_elem;
 34 |   this->grad_mult.set_size(param.n_elem);
 35 |   this->grad_mult.fill(1.0);
 36 | }
 37 | 
 38 | REAL ConstantMean::Eval(const Col<REAL>& x)
 39 | {
 40 |   return param(0);
 41 | }
 42 | 
 43 | void ConstantMean::Grad(Col<REAL> &grad, const Col<REAL>& x)
 44 | {
 45 |   grad.set_size(param.n_elem);
 46 |   grad(0) = 1;
 47 | 
 48 |   grad *= this->grad_mult;
 49 | }
 50 | 
 51 | void ConstantMean::Hessian(Mat<REAL> &hessian, const Col<REAL>& x)
 52 | {
 53 |   hessian.set_size(param.n_elem, param.n_elem);
 54 |   hessian.fill(0);
 55 | }
 56 | 
 57 | void ConstantMean::GradX(Col<REAL> &grad, const Col<REAL>& x)
 58 | {
 59 |   grad.set_size(x.n_elem);
 60 |   grad.fill(0);
 61 | }
 62 | 
 63 | LogMean::LogMean()
 64 | {
 65 |   this->SetParams("49 3 0 0");
 66 |   this->param_dim = param.n_elem;
 67 |   this->grad_mult.set_size(param.n_elem);
 68 |   this->grad_mult.fill(1.0);
 69 | }
 70 | 
 71 | LogMean::LogMean(const Col<REAL>& param)
 72 | {
 73 |   this->SetParams(param);
 74 |   this->param_dim = param.n_elem;
 75 |   this->grad_mult.set_size(param.n_elem);
 76 |   this->grad_mult.fill(1.0);
 77 | }
 78 | 
 79 | void LogMean::SetParams(const Col<REAL>& param)
 80 | {
 81 | 
 82 |   this->param = param;
 83 |   source.set_size(2);
 84 |   source(0) = param(2);
 85 |   source(1) = param(3);
 86 | }
 87 | 
 88 | REAL LogMean::Eval(const Col<REAL>& x)
 89 | {
 90 |   REAL dist = norm(source-x, 2);
 91 |   if(dist < 1e-6)
 92 |     return -param(0);
 93 |   else
 94 |     return -param(0) - 10*param(1)*log10(dist);
 95 | }
 96 | 
 97 | void LogMean::Grad(Col<REAL> &grad, const Col<REAL>& x)
 98 | {
 99 |   grad.set_size(param.n_elem);
100 |   /*
101 |   grad(0) = -1;
102 |   grad(1) = -10*log10(norm(source-x, 2));
103 |   grad(2) = 10*param(1)*(source(0)-x(0))/(log(10)*
104 | 					  ((source(0)-x(0))*(source(0)-x(0)) +
105 | 					   (source(1)-x(1))*(source(1)-x(1))));
106 |   grad(3) = 10*param(1)*(source(1)-x(1))/(log(10)*
107 | 					  ((source(0)-x(0))*(source(0)-x(0)) +
108 | 					   (source(1)-x(1))*(source(1)-x(1))));
109 |   */
110 | 
111 |   grad(0)=1;
112 | 
113 |   grad(1)=(-10*log(sqrt(pow(source(0) - x(0),2) + pow(source(1) - x(1),2))))/log(10);
114 | 
115 |   grad(2)=(-10*param(1)*(source(0) - x(0)))/((pow(source(0) - x(0),2) + pow(source(1) - x(1),2))*log(10));
116 | 
117 |   grad(3)=(-10*param(1)*(source(1) - x(1)))/((pow(source(0) - x(0),2) + pow(source(1) - x(1),2))*log(10));
118 | 
119 |   grad = grad % this->grad_mult;
120 | }
121 | 
122 | void LogMean::Hessian(Mat<REAL> &H, const Col<REAL>& x)
123 | {
124 |   H.set_size(param.n_elem, param.n_elem);
125 | 
126 | 
127 |   /*
128 |   // d[grad(0)]/d[param(i)]
129 |   H(0,0) = 0;
130 |   H(0,1) = 0;
131 |   H(0,2) = 0;
132 |   H(0,3) = 0;
133 | 
134 |   // d[grad(1)]/d[param(i)]
135 |   H(1,0) = 0;
136 |   H(1,1) = 0;
137 |   H(1,2) = -10*(param(2)-x(0))/
138 |     (((param(2)-x(0))*(param(2)-x(0)) + (param(3)-x(1))*(param(3)-x(1)))*log(10));
139 |   H(1,3) = -10*(param(3)-x(1))/
140 |     (((param(2)-x(0))*(param(2)-x(0)) + (param(3)-x(1))*(param(3)-x(1)))*log(10));
141 | 
142 |   // d[grad(2)]/d[param(i)]
143 |   H(2,0) = 0;
144 |   H(2,1) = 10*(source(0)-x(0))/(log(10)*
145 | 				((source(0)-x(0))*(source(0)-x(0)) +
146 | 				 (source(1)-x(1))*(source(1)-x(1))));
147 |   H(2,2) = 10*param(1)*
148 |     ((param(3)-x(1))*(param(3)-x(1)) - (param(2)-x(0))*(param(2)-x(0)))/
149 |     pow(((param(2)-x(0))*(param(2)-x(0)) + (param(3)-x(1))*(param(3)-x(1))),2);
150 |   H(2,3) = -20*param(1)*(param(2)-x(0))*(param(3)-x(1))/
151 |     pow(((param(2)-x(0))*(param(2)-x(0)) + (param(3)-x(1))*(param(3)-x(1))),2);
152 | 
153 |   // d[grad(3)]/d[param(i)]
154 |   H(3,0) = 0;
155 |   H(3,1) = 10*(source(1)-x(1))/(log(10)*
156 | 				((source(0)-x(0))*(source(0)-x(0)) +
157 | 				 (source(1)-x(1))*(source(1)-x(1))));
158 |   H(3,2) = -20*param(1)*(param(2)-x(0))*(param(3)-x(1))/
159 |     pow(((param(2)-x(0))*(param(2)-x(0)) + (param(3)-x(1))*(param(3)-x(1))),2);
160 |   H(3,3) = 10*param(1)*(pow(param(2)-x(0),2) - pow(param(3)-x(1),2))/
161 |     pow(((param(2)-x(0))*(param(2)-x(0)) + (param(3)-x(1))*(param(3)-x(1))),2);
162 | 
163 |   */
164 | 
165 |   H(0,0)=0;
166 | 
167 |   H(0,1)=0;
168 | 
169 |   H(0,2)=0;
170 | 
171 |   H(0,3)=0;
172 | 
173 |   H(1,0)=0;
174 | 
175 |   H(1,1)=0;
176 | 
177 |   H(1,2)=(-10*(source(0) - x(0)))/((pow(source(0) - x(0),2) + pow(source(1) - x(1),2))*log(10));
178 | 
179 |   H(1,3)=(-10*(source(1) - x(1)))/((pow(source(0) - x(0),2) + pow(source(1) - x(1),2))*log(10));
180 | 
181 |   H(2,0)=0;
182 | 
183 |   H(2,1)=(-10*(source(0) - x(0)))/((pow(source(0) - x(0),2) + pow(source(1) - x(1),2))*log(10));
184 | 
185 |   H(2,2)=(20*param(1)*pow(source(0) - x(0),2))/(pow(pow(source(0) - x(0),2) + pow(source(1) - x(1),2),2)*log(10)) - (10*param(1))/((pow(source(0) - x(0),2) + pow(source(1) - x(1),2))*log(10));
186 | 
187 |   H(2,3)=(20*param(1)*(source(0) - x(0))*(source(1) - x(1)))/(pow(pow(source(0) - x(0),2) + pow(source(1) - x(1),2),2)*log(10));
188 | 
189 |   H(3,0)=0;
190 | 
191 |   H(3,1)=(-10*(source(1) - x(1)))/((pow(source(0) - x(0),2) + pow(source(1) - x(1),2))*log(10));
192 | 
193 |   H(3,2)=(20*param(1)*(source(0) - x(0))*(source(1) - x(1)))/(pow(pow(source(0) - x(0),2) + pow(source(1) - x(1),2),2)*log(10));
194 | 
195 |   H(3,3)=(-10*param(1))/((pow(source(0) - x(0),2) + pow(source(1) - x(1),2))*log(10)) + (20*param(1)*pow(source(1) - x(1),2))/(pow(pow(source(0) - x(0),2) + pow(source(1) - x(1),2),2)*log(10));
196 | 
197 | 
198 | }
199 | 
200 | 
201 | void LogMean::GradX(Col<REAL> &grad, const Col<REAL>& x)
202 | {
203 |   grad.set_size(x.n_elem);
204 |   REAL dist = norm(source-x, 2);
205 |   if(dist < 1e-6)
206 |     grad.fill(0);
207 |   else {
208 |     grad(0) = -10*param(1)*(x(0)-source(0))/(dist*log(10));
209 |     grad(1) = -10*param(1)*(x(1)-source(1))/(dist*log(10));
210 |   }
211 | }
212 | 


--------------------------------------------------------------------------------
/src/MeanFunction.h:
--------------------------------------------------------------------------------
 1 | #ifndef _MEAN_FUNCTION_
 2 | #define _MEAN_FUNCTION_
 3 | 
 4 | #include "armadillo"
 5 | using namespace arma;
 6 | 
 7 | #include "myTypes.h"
 8 | 
 9 | class MeanFunction {
10 |  public:
11 |   virtual ~MeanFunction() {}
12 |   virtual REAL Eval(const Col<REAL>& x)=0;
13 |   virtual void Grad(Col<REAL> &grad, const Col<REAL>& x)=0;
14 |   virtual void Hessian(Mat<REAL> &hessian, const Col<REAL>& x)=0;
15 |   virtual void SetParams(const Col<REAL>& param);
16 |   virtual Col<REAL> GetParams();
17 |   virtual void GradX(Col<REAL> &grad, const Col<REAL>& x)=0;
18 |   int GetParamDim();
19 |   void SetGradMult(const Col<REAL>& grad_mult);
20 |   
21 |  protected:
22 |   int param_dim;
23 |   Col<REAL> param;
24 |   Col<REAL> grad_mult;
25 | };
26 | 
27 | class ConstantMean : public MeanFunction {
28 |  public:
29 |   ConstantMean();
30 |   ConstantMean(const Col<REAL>& param);
31 |   REAL Eval(const Col<REAL>& x);
32 |   void Grad(Col<REAL> &grad, const Col<REAL>& x);
33 |   void Hessian(Mat<REAL> &hessian, const Col<REAL>& x);
34 |   void GradX(Col<REAL> &grad, const Col<REAL>& x);
35 | };
36 | 
37 | class LogMean : public MeanFunction {
38 |  public:
39 |   LogMean();
40 |   LogMean(const Col<REAL>& param);
41 |   void SetParams(const Col<REAL>& param);
42 |   REAL Eval(const Col<REAL>& x);
43 |   void Grad(Col<REAL> &grad, const Col<REAL>& x);
44 |   void Hessian(Mat<REAL> &hessian, const Col<REAL>& x);
45 |   void GradX(Col<REAL> &grad, const Col<REAL>& x);
46 |  private:
47 |   Col<REAL> source;
48 | };
49 | 
50 | #endif
51 | 


--------------------------------------------------------------------------------
/src/SPGP.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | using namespace std;
  3 | 
  4 | #include "SPGP.h"
  5 | #include "CGOptimizer.h"
  6 | #include "armadillo_backsub.h"
  7 | 
  8 | SPGP::SPGP(REAL s2_n, KernelFunction *kernel, MeanFunction *mean)
  9 |   : GP(s2_n, kernel, mean)
 10 | {
 11 |   // Nothing special to do here
 12 | 
 13 |   del = 1e-6;
 14 | }
 15 | 
 16 | SPGP::~SPGP()
 17 | {
 18 | 
 19 | }
 20 | 
 21 | void SPGP::
 22 | SetDel(REAL del)
 23 | {
 24 |   this->del = del;
 25 | }
 26 | 
 27 | void SPGP::
 28 | SetKernelFuncParams(const Col<REAL>& param)
 29 | {
 30 | 
 31 |   GP::SetKernelFuncParams(param);
 32 |   this->new_kernel = true;
 33 | }
 34 | 
 35 | void SPGP::
 36 | SetMeanFuncParams(const Col<REAL>& param)
 37 | {
 38 |   GP::SetMeanFuncParams(param);
 39 |   this->new_mean = true;
 40 | }
 41 | 
 42 | void SPGP::
 43 | SetPseudoInputs(const Mat<REAL>& Xb)
 44 | {
 45 |   this->Xb = Xb;
 46 |   MatrixMap(kxbxb, Xb, Xb);
 47 | 
 48 |   this->n = Xb.n_cols;
 49 | 
 50 |   this->new_pi = true;
 51 | }
 52 | 
 53 | void SPGP::
 54 | SetTraining(const Mat<REAL>& X, const Row<REAL> &y)
 55 | {
 56 |   this->X = X;
 57 |   this->y = y;
 58 | 
 59 |   this->N = X.n_cols;
 60 |   this->dim = X.n_rows;
 61 | 
 62 |   this->meanvals.set_size(X.n_cols);
 63 | 
 64 |   this->new_training = true;
 65 | }
 66 | 
 67 | void SPGP::
 68 | ComputeLm()
 69 | {
 70 |   if(new_training || new_pi || new_kernel) {
 71 |     MatrixMap(kxbx, Xb, X);
 72 |   }
 73 | 
 74 |   if(new_pi || new_kernel) {
 75 |     MatrixMap(kxbxb, Xb, Xb);
 76 |   }
 77 | 
 78 |   //kxbx.print("kxbx: ");
 79 |   //kxbxb.print("kxbxb: ");
 80 | 
 81 |   //printf("Computing L\n");
 82 |   L = trans(chol(kxbxb + del*eye<Mat<REAL> >(kxbxb.n_cols, kxbxb.n_cols)));
 83 |   //L.print("L: ");
 84 | 
 85 |   solve_tri(V, L, kxbx, false);
 86 |   
 87 |   //V.print("V: ");
 88 |   //V = trans(V);
 89 | 
 90 |   Col<REAL> tmp(this->dim);
 91 |   tmp.zeros();
 92 | 
 93 |   ep = 1 + (this->kernel->Eval(tmp, tmp) - trans(sum(V%V)))/s2_n;
 94 | 
 95 |   //ep.print("ep:");
 96 | 
 97 |   V = V / (repmat(trans(sqrt(ep)), n, 1));
 98 | 
 99 |   //printf("Computing Lm\n");
100 |   Lm = trans(chol(s2_n*eye<Mat<REAL> >(n, n) + V*trans(V)));
101 | 
102 |   //Lm.print("Lm");
103 | }
104 | 
105 | void SPGP::
106 | ComputeBet()
107 | {
108 |   if(new_mean || new_training) {
109 |     for(unsigned int i=0; i < this->N; ++i)
110 |       this->meanvals(i) = this->mean->Eval(X.col(i));
111 |   }
112 | 
113 |   Col<REAL> ytmp = trans(y-meanvals)/sqrt(ep);
114 | 
115 |   bet.zeros(Lm.n_cols, ytmp.n_cols);
116 | 
117 |   //bet.print("bet (init)");
118 |   Mat<REAL> tmp = V*ytmp;
119 |   //tmp.print("V*ytmp");
120 | 
121 |   solve_tri(bet, Lm, V*ytmp, false);
122 | 
123 |   //bet.print("bet");
124 | }
125 | 
126 | REAL SPGP::ComputeLikelihood()
127 | {
128 |   ComputeLm();
129 |   ComputeBet();
130 | 
131 |   Mat<REAL> tmp;
132 |   loglikelihood = 0;
133 | 
134 |   tmp = sum(log(Lm.diag()));
135 |   //tmp.print("sum(log(Lm.diag()))");
136 | 
137 |   loglikelihood += tmp(0,0);
138 | 
139 |   tmp = (dot(y,y) - dot(bet, bet) + sum(log(ep)))/2;
140 |   //tmp.print("(dot(y,y) - dot(bet, bet) + sum(log(ep)))/2");
141 | 
142 |   loglikelihood += tmp(0,0);
143 | 
144 |   /*
145 |   loglikelihood = sum(log(Lm.diag())) +
146 |     (X.n_cols - Xb.n_cols)/2*log(s2_n) +
147 |     (dot(y,y) - dot(bet, bet) + sum(log(ep)))/2 +
148 |     0.5*X.n_cols*log(2*Math<REAL>::pi());
149 |   */
150 | 
151 |   loglikelihood += (X.n_cols - Xb.n_cols)/2*log(s2_n) +
152 |     0.5*X.n_cols*log(2*Math<REAL>::pi());
153 | 
154 |   return loglikelihood;
155 | }
156 | 
157 | void SPGP::Predict(const Mat<REAL> &Xs, Row<REAL> &mu)
158 | {
159 |   ComputeLm();
160 |   ComputeBet();
161 | 
162 |   Mat<REAL> kstar(Xb.n_cols, Xs.n_cols);
163 |   //kstar.print("kstar(init)");
164 |   MatrixMap(kstar, Xb, Xs);
165 |   //kstar.print("kstar");
166 |   Mat<REAL> lst;
167 |   solve(lst, trimatu(L), kstar);
168 |   //lst.print("lst");
169 |   Mat<REAL> lmst;
170 |   solve(lmst, trimatu(Lm), lst);
171 |   //lmst.print("lmst");
172 | 
173 |   Row<REAL> meanxs(Xs.n_cols);
174 |   for(unsigned int i=0; i < Xs.n_cols; ++i)
175 |     meanxs(i) = this->mean->Eval(Xs.col(i));
176 | 
177 |   mu = trans(bet*lmst) + meanxs;
178 | }
179 | 
180 | void SPGP::Predict(const Mat<REAL> &Xs, Row<REAL> &mu, Row<REAL> &var)
181 | {
182 |   ComputeLm();
183 |   ComputeBet();
184 | 
185 |   Mat<REAL> kstar(Xb.n_cols, Xs.n_cols);
186 |   //kstar.print("kstar(init)");
187 |   MatrixMap(kstar, Xb, Xs);
188 |   //kstar.print("kstar");
189 |   Mat<REAL> lst;
190 |   solve_tri(lst, L, kstar, false);
191 |   //lst.print("lst");
192 |   Mat<REAL> lmst;
193 |   solve_tri(lmst, Lm, lst, false);
194 |   //lmst.print("lmst");
195 | 
196 |   Row<REAL> meanxs(Xs.n_cols);
197 |   for(unsigned int i=0; i < Xs.n_cols; ++i)
198 |     meanxs(i) = this->mean->Eval(Xs.col(i));
199 | 
200 |     //printf("trans(bet): %d x %d, lmst: %d x %d \n", bet.n_cols, bet.n_rows, lmst.n_rows, lmst.n_cols);
201 |   mu = trans(bet)*lmst + meanxs;
202 | 
203 |     //mu.print("mu");
204 | 
205 |   Col<REAL> tmp(this->dim);
206 |   tmp.zeros();
207 |   var = this->kernel->Eval(tmp, tmp) -
208 |     sum(lst % lst) +
209 |     s2_n*sum(lmst % lmst);
210 | }
211 | 
212 | 
213 | void SPGP::Predict(const Col<REAL> &Xs, REAL &mu)
214 | {
215 |   Mat<REAL> XsMat(Xs.n_rows, 1);
216 |   XsMat.col(0) = Xs;
217 | 
218 |   Predict(XsMat, mu);
219 | }
220 | 
221 | void SPGP::Predict(const Col<REAL> &Xs, REAL &mu, REAL &var)
222 | {
223 |   Mat<REAL> XsMat(Xs.n_rows, 1);
224 |   XsMat.col(0) = Xs;
225 | 
226 |   Predict(XsMat, mu, var);
227 | }
228 | 
229 | void SPGP::
230 | GradLikelihoodPseudoInputs(Mat<REAL> &grad)
231 | {
232 |   // note: we make some assumptions here about the kernel function being sq-exp
233 |   ComputeLm();
234 |   ComputeBet();
235 | 
236 |   Mat<REAL> Q = kxbxb + del*eye<Mat<REAL> >(kxbxb.n_cols, kxbxb.n_cols);
237 | 
238 |   Mat<REAL> K = kxbx;
239 |   K = K / (repmat(trans(sqrt(ep)), n, 1));
240 | 
241 |   Mat<REAL> invLmV;
242 |   solve(invLmV, trimatu(Lm), V);
243 | 
244 |   Mat<REAL> Lt = L*Lm;
245 |   Mat<REAL> B1; solve(B1, trimatl(trans(Lt)), invLmV);
246 |   Col<REAL> b1; solve(b1, trimatl(trans(Lt)), bet);
247 |   Mat<REAL> invLV; solve(invLV, trimatl(trans(L)), V);
248 |   Mat<REAL> invL = inv(L);
249 |   Mat<REAL> invQ = trans(invL)*invL;
250 |   Mat<REAL> invLt = inv(Lt);
251 |   Mat<REAL> invA = trans(invLt)*invLt;
252 | 
253 |   Col<REAL> mu; solve(mu, trimatl(trans(Lm)), bet); mu = trans(trans(mu)*V);
254 |   Col<REAL> sumVsq = trans(sum(V%V));
255 | 
256 |   Col<REAL> bigsum = trans(y) % trans((trans(bet)*invLmV))/s2_n -
257 |     trans(sum(invLmV % invLmV))/2 -
258 |     (trans(y % y) + mu % mu)/2/s2_n + 0.5;
259 | 
260 |   Mat<REAL> TT = invLV*(trans(invLV)%repmat(bigsum, 1, n));
261 | 
262 |   Mat<REAL> dnnQ, dNnK, epdot;
263 |   Col<REAL> epPmod;
264 |   Mat<REAL> dfxb(n, dim);
265 | 
266 |   Col<REAL> kernel_param = this->kernel->GetParams();
267 | 
268 |   for(unsigned int i=0; i < this->dim; ++i) {
269 |     dnnQ = (repmat(trans(Xb.row(i)), 1, n) - repmat(Xb.row(i), n, 1)) % Q;
270 |     dNnK = (repmat(X.row(i), n, 1) - repmat(trans(Xb.row(i)), 1, N)) % K;
271 | 
272 |     epdot = -2/s2_n*(dNnK % invLV);
273 |     epPmod = trans(-sum(epdot));
274 | 
275 |     dfxb.col(i) = -b1 % (dNnK*(trans(y)-mu)/s2_n + dnnQ*b1)
276 |       + sum((invQ - invA*s2_n)%dnnQ, 1)
277 |       + epdot*bigsum - 2/s2_n*sum(dnnQ % TT, 1);
278 | 
279 |     dfxb.col(i) = dfxb.col(i)/sqrt(kernel_param(1));
280 |   }
281 | 
282 |   grad = dfxb;
283 | }
284 | 
285 | void SPGP::GetPseudoInputs(Mat<REAL> &Xb)
286 | {
287 |   Xb.set_size(this->Xb.n_rows, this->Xb.n_cols);
288 |   //Xb = Mat<REAL>(this->Xb);
289 |   Xb = this->Xb;
290 | }
291 | 
292 | void SPGP::
293 | OptimizePseudoInputs(Mat<REAL> &pseudoinputs, int max_iterations, double step, double eps)
294 | {
295 |   CGOptimizer opt(reinterpret_cast<void*>(this));
296 | 
297 |   Col<REAL> opt_state(pseudoinputs.n_rows*pseudoinputs.n_cols);
298 |   int idx=0;
299 |   for(unsigned int i=0; i < pseudoinputs.n_rows; ++i) {
300 |     for(unsigned int j=0; j < pseudoinputs.n_cols; ++j) {
301 |       opt_state(idx++) = pseudoinputs(i, j);
302 |     }
303 |   }
304 | 
305 |   opt.Initialize(opt_state, &f_eval_pi, &df_eval_pi, &fdf_eval_pi, step, eps);
306 |   opt.Optimize(opt_state, max_iterations);
307 | 
308 |   idx=0;
309 |   for(unsigned int i=0; i < pseudoinputs.n_rows; ++i) {
310 |     for(unsigned int j=0; j < pseudoinputs.n_cols; ++j) {
311 |       pseudoinputs(i, j) = opt_state(idx++);
312 |     }
313 |   }
314 | }
315 | 
316 | double f_eval_pi(const gsl_vector *x, void *param)
317 | {
318 |   SPGP *gp_obj = reinterpret_cast<SPGP*>(param);
319 | 
320 |   Mat<REAL> pinput; gp_obj->GetPseudoInputs(pinput);
321 |   unsigned int dim = pinput.n_cols;
322 |   unsigned int n = pinput.n_rows;
323 | 
324 |   int k=0;
325 |   for(unsigned int i=0; i < n; ++i) {
326 |     for(unsigned int j=0; j < dim; ++j) {
327 |       pinput(i, j) = gsl_vector_get(x, k++);
328 |     }
329 |   }
330 | 
331 |   gp_obj->SetPseudoInputs(pinput);
332 | 
333 |   double ret = gp_obj->ComputeLikelihood();
334 | 
335 |   return ret;
336 | }
337 | 
338 | void df_eval_pi(const gsl_vector *x, void *param, gsl_vector *g)
339 | {
340 | 
341 |   SPGP *gp_obj = reinterpret_cast<SPGP*>(param);
342 | 
343 |   Mat<REAL> pinput; gp_obj->GetPseudoInputs(pinput);
344 |   unsigned int dim = pinput.n_cols;
345 |   unsigned int n = pinput.n_rows;
346 | 
347 |   int k=0;
348 |   for(unsigned int i=0; i < n; ++i) {
349 |     for(unsigned int j=0; j < dim; ++j) {
350 |       pinput(i, j) = gsl_vector_get(x, k++);
351 |     }
352 |   }
353 | 
354 |   gp_obj->SetPseudoInputs(pinput);
355 | 
356 |   Mat<REAL> grad;
357 |   gp_obj->GradLikelihoodPseudoInputs(grad);
358 | 
359 |   k=0;
360 |   for(unsigned int i=0; i < n; ++i) {
361 |     for(unsigned int j=0; j < dim; ++j) {
362 | 
363 |       gsl_vector_set(g, k++, -grad(j, i));
364 | 
365 |     }
366 |   }
367 | }
368 | 
369 | void fdf_eval_pi(const gsl_vector *x, void *param, double *f, gsl_vector *g)
370 | {
371 |   *f = f_eval_pi(x, param);
372 |   df_eval_pi(x, param, g);
373 | }
374 | 


--------------------------------------------------------------------------------
/src/SPGP.h:
--------------------------------------------------------------------------------
 1 | #ifndef _SPGP_H_
 2 | #define _SPGP_H_
 3 | 
 4 | #include "GP.h"
 5 | 
 6 | double f_eval_pi(const gsl_vector *x, void *param);
 7 | void df_eval_pi(const gsl_vector *x, void *param, gsl_vector *g);
 8 | void fdf_eval_pi(const gsl_vector *x, void *param, double *f, gsl_vector *g);
 9 | 
10 | class SPGP : public GP
11 | {
12 |  public:
13 |   SPGP(REAL s2_n, KernelFunction *kernel, MeanFunction *mean);
14 |   virtual ~SPGP();
15 | 
16 | void SetDel(REAL del);
17 |   void SetKernelFuncParams(const Col<REAL>& param);
18 |   void SetMeanFuncParams(const Col<REAL>& param);
19 | 
20 |   void SetTraining(const Mat<REAL>& X, const Row<REAL> &y);
21 |   void SetPseudoInputs(const Mat<REAL>& Xb);
22 | 
23 |   void Predict(const Mat<REAL> &Xs, Row<REAL> &mu);
24 |   void Predict(const Mat<REAL> &Xs, Row<REAL> &mu, Row<REAL> &var);
25 |   void Predict(const Col<REAL> &Xs, REAL &mu);
26 |   void Predict(const Col<REAL> &Xs, REAL &mu, REAL &var);
27 | 
28 |   void ComputeLm();
29 |   void ComputeBet();
30 | 
31 |   void GradLikelihoodPseudoInputs(Mat<REAL> &grad);
32 | void OptimizePseudoInputs(Mat<REAL> &pseudoinputs, int max_iterations, double step, double eps);
33 |   
34 |   REAL ComputeLikelihood();
35 | 
36 | void GetPseudoInputs(Mat<REAL> &Xb);
37 | 
38 |  private:
39 |   // State variables
40 |   Mat<REAL> Xb; //! Pseudo input locations
41 |   
42 |   // Misc
43 |   Mat<REAL> Lm, V, ep, bet;
44 | 
45 |   Mat<REAL> kxbxb, kxbx;
46 | 
47 |   bool new_training, new_mean, new_kernel, new_pi;
48 | 
49 |   unsigned int N, n, dim;
50 |   REAL del;
51 | 
52 | };
53 | 
54 | #endif
55 | 


--------------------------------------------------------------------------------
/src/armadillo_backsub.cc:
--------------------------------------------------------------------------------
 1 | #include "armadillo_backsub.h"
 2 | 
 3 | #include <clapack.h>
 4 | 
 5 | namespace arma
 6 | {
 7 |   extern "C"
 8 |   {
 9 | 
10 |     void arma_fortran_prefix(spotrs)(char* uplo, blas_int* n, blas_int *nrhs, float* a, blas_int* lda, float* b, blas_int* ldb, blas_int* info)
11 |     {
12 |       arma_fortran_noprefix(spotrs)(uplo, n, nrhs, a, lda, b, ldb, info);
13 |     }
14 | 
15 |     void arma_fortran_prefix(dpotrs)(char* uplo, blas_int* n, blas_int *nrhs, double* a, blas_int* lda, double* b, blas_int* ldb, blas_int* info)
16 |     {
17 |       arma_fortran_noprefix(dpotrs)(uplo, n, nrhs, a, lda, b, ldb, info);
18 |     }
19 | 
20 |   }
21 | 
22 | }
23 | 


--------------------------------------------------------------------------------
/src/armadillo_backsub.h:
--------------------------------------------------------------------------------
  1 | #ifndef ARMADILLO_BACKSUB_H_
  2 | #define ARMADILLO_BACKSUB_H_
  3 | 
  4 | #include <armadillo>
  5 | using namespace arma;
  6 | 
  7 | namespace arma
  8 | {
  9 | 
 10 |   extern "C"
 11 |   {
 12 | // Cholesky decomposition Solve
 13 |     void arma_fortran(spotrs)(char* uplo, blas_int* n, blas_int *nrhs, float* a, blas_int* lda, float* b, blas_int* ldb, blas_int* info);
 14 |     void arma_fortran(dpotrs)(char* uplo, blas_int* n, blas_int *nrhs, double* a, blas_int* lda, double* b, blas_int* ldb, blas_int* info);
 15 |   }
 16 | 
 17 |   namespace lapack
 18 |   {
 19 | 
 20 |     template<typename eT>
 21 |       inline
 22 |       void
 23 |       potrs(char* uplo, blas_int* n, blas_int *nrhs, const eT* a, blas_int* lda, eT* b, blas_int* ldb, blas_int* info)
 24 |     {
 25 |       arma_type_check(( is_supported_blas_type<eT>::value == false ));
 26 | 
 27 |       if(is_float<eT>::value == true)
 28 |       {
 29 |         typedef float T;
 30 |         arma_fortran(spotrs)(uplo, n, nrhs, (T*)a, lda, (T*)b, ldb, info);
 31 |       }
 32 |       else
 33 |         if(is_double<eT>::value == true)
 34 |         {
 35 |           typedef double T;
 36 |           arma_fortran(dpotrs)(uplo, n, nrhs, (T*)a, lda, (T*)b, ldb, info);
 37 |         }
 38 |         else {
 39 |           
 40 |           /* if(is_supported_complex_float<eT>::value == true) */
 41 |           /* { */
 42 |           /*   typedef std::complex<float> T; */
 43 |           /*   arma_fortran(cpotrs)(uplo, n, nrhs, (T*)a, lda, (T*)b, ldb, info); */
 44 |           /* } */
 45 |           /* else */
 46 |           /*   if(is_supported_complex_double<eT>::value == true) */
 47 |           /*   { */
 48 |           /*     typedef std::complex<double> T; */
 49 |           /*     arma_fortran(zpotrs)(uplo, n, nrhs, (T*)a, lda, (T*)b, ldb, info); */
 50 |           /*   } */
 51 |         }
 52 |         
 53 |     }
 54 | 
 55 |   }
 56 | 
 57 |   template<typename eT>
 58 |     inline
 59 |     bool
 60 |     cholbacksub(Mat<eT>& out, const Mat<eT>& U, const Mat<eT>& B)
 61 |   {
 62 |     arma_extra_debug_sigprint();
 63 |     
 64 | #if defined(ARMA_USE_LAPACK)
 65 |     {
 66 |       char uplo = 'U';
 67 |       blas_int  n    = blas_int(U.n_rows);
 68 |       blas_int nrhs = blas_int(B.n_cols);
 69 |       blas_int info = 0;
 70 |                         
 71 |       Mat<eT> U_copy = U;
 72 |       
 73 |       out = B;
 74 |       lapack::potrs(&uplo, &n, &nrhs, U_copy.memptr(), &n, out.memptr(), &n, &info);
 75 |       
 76 |       return (info == 0);
 77 |     }
 78 | #else
 79 |     {
 80 |       arma_stop("auxlib::cholbacksub(): need LAPACK library");
 81 |       return false;
 82 |     }
 83 | #endif
 84 |   }
 85 | 
 86 |   template<typename eT, typename T1, typename T2>
 87 |     inline
 88 |     bool
 89 |     cholbacksub(Mat<eT>& out, const Base<eT,T1>& U, const Base<eT,T2>&b)
 90 |   {
 91 |     arma_extra_debug_sigprint();
 92 |     const unwrap<T1> tmp_A(U.get_ref());
 93 |     const unwrap<T2> tmp_B(b.get_ref());
 94 |           
 95 |     const Mat<eT>& A = tmp_A.M;
 96 |     const Mat<eT>& B = tmp_B.M;
 97 |               
 98 |     arma_debug_check( !A.is_square(), "cholbacksub(): given matrix is not square");
 99 |     arma_debug_check( A.n_rows != B.n_rows, "cholbacksub(): A and B must have same number of rows");
100 |               
101 |     return cholbacksub(out, A, B);
102 |   }
103 | 
104 | 
105 | //! Solve a system of linear equations
106 | //! where A is triangular
107 | //! Assumes that A.n_rows = A.n_cols
108 | //! and B.n_rows = A.n_rows
109 |   template<typename eT>
110 |     inline
111 |     bool
112 |     solve_tri(Mat<eT>& out, const Mat<eT>& A, const Mat<eT>& B, const bool upper)
113 |   {
114 |     arma_extra_debug_sigprint();
115 |     
116 | #if defined(ARMA_USE_LAPACK)
117 |     {
118 |       char uplo = (upper ? 'U' : 'L');
119 |       char trans = 'N';
120 |       char diag = 'N';
121 |       blas_int n    = blas_int(A.n_rows);
122 |       blas_int lda  = blas_int(A.n_rows);
123 |       blas_int ldb  = blas_int(A.n_rows);
124 |       blas_int nrhs = blas_int(B.n_cols);
125 |       blas_int info = 0;
126 |       out = B;
127 |       Mat<eT> A_copy = A;
128 |           
129 |       lapack::trtrs<eT>(&uplo, &trans, &diag, &n, &nrhs,
130 |                         A_copy.memptr(), &lda,
131 |                         out.memptr(), &ldb, &info);
132 |       return (info == 0);
133 |     }
134 | #else
135 |     {
136 |       arma_stop("auxlib::solve_tri(): need LAPACK library");
137 |       return false;
138 |     }
139 | #endif
140 |   }  
141 | 
142 | 
143 | //! Solve a system of linear equations, i.e., A*X = B, where X is unknown.
144 | //! and X is upper or lower triangular
145 |   template<typename eT, typename T1, typename T2>
146 |     inline
147 |     bool
148 |     solve_tri(Mat<eT>& X, const Base<eT,T1>& A_in, const Base<eT,T2>& B_in, const bool upper=true)
149 |   {
150 |     arma_extra_debug_sigprint();
151 |  
152 |     const unwrap<T1> tmp1(A_in.get_ref());
153 |     const unwrap<T2> tmp2(B_in.get_ref());
154 | 
155 |     const Mat<eT>& A = tmp1.M;
156 |     const Mat<eT>& B = tmp2.M;
157 | 
158 |     arma_debug_check( (A.n_rows != B.n_rows), "solve_tri(): number of rows in A and B must be the same" );
159 | 
160 |     arma_debug_check( (A.n_rows != A.n_cols), "solve_tri(): A must be square" );
161 | 
162 |     return solve_tri(X, A, B, upper);
163 | 
164 |   }
165 | 
166 |   template<typename eT, typename T1, typename T2>
167 |     inline
168 |     Mat<eT>
169 |     solve_tri(const Base<eT,T1>& A_in, const Base<eT,T2>& B_in, const bool upper=true)
170 |   {
171 |     arma_extra_debug_sigprint();
172 | 
173 |     Mat<eT> X;
174 |     bool info = solve_tri(X, A_in, B_in, upper);
175 | 
176 |     if(info == false)
177 |     {
178 |       arma_print("solve_tri(): solution not found");
179 |     }
180 | 
181 |     return X;
182 |   }
183 | 
184 | }
185 | 
186 | #endif
187 | 


--------------------------------------------------------------------------------
/src/myTypes.h:
--------------------------------------------------------------------------------
1 | #ifndef _MY_TYPES_
2 | #define _MY_TYPES_
3 | 
4 | typedef float REAL;
5 | //typedef double REAL;
6 | 
7 | #endif
8 | 


--------------------------------------------------------------------------------
/src/test.cc:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include "GP.h"
 3 | 
 4 | using namespace std;
 5 | 
 6 | int main(int argc, char **argv)
 7 | {
 8 |   Col<REAL> kernel_param = "1.0 4.0";
 9 |   SqExpKernel kernel(kernel_param);
10 |   ConstantMean mean("10");
11 |   GP gp(0.1, &kernel, &mean);
12 | 
13 |   Mat<REAL> X(1,10);
14 |   Row<REAL> y;
15 |   y.set_size(X.n_cols);
16 | 
17 |   for(int i=0; i < X.n_cols; ++i) {
18 |     X(0,i) = i;
19 |     y(i) = 10+sin(i/(2*Math<REAL>::pi()*X.n_cols));
20 |   }
21 | 
22 |   Mat<REAL> Xs = X;
23 |   Row<REAL> mu, var;
24 |   REAL mu1, var1;
25 | 
26 |   cout << "Setting training data\n";
27 |   gp.SetTraining(X, y);
28 |   cout << "Making predictions\n";
29 |   gp.Predict(Xs, mu, var);
30 |   // gp.Predict(Xs.col(0), mu1, var1);
31 | 
32 |   Col<REAL> grad;
33 |   gp.GradLikelihoodKernelParams(grad);
34 |   grad.print("Likelihood Gradient (kernel)");
35 | 
36 |   cout << "X:\n" << X << endl;
37 |   cout << "y:\n" << y << endl;
38 |   cout << "Xs:\n" << Xs << endl;
39 |   cout << "mu:\n" << mu << endl;
40 | 
41 |   Row<REAL> error = y - mu;
42 |   cout << "error:\n" << norm(error,2) << endl;
43 | 
44 |   cout << "Log-likelihood: " << gp.ComputeLikelihood() << endl;
45 | 
46 |   return 0;
47 | }
48 | 


--------------------------------------------------------------------------------
/src/test_spgp.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <fstream>
  3 | 
  4 | #include "SPGP.h"
  5 | 
  6 | using namespace std;
  7 | 
  8 | int main(int argc, char **argv)
  9 | {
 10 |   Col<REAL> kernel_param = "1.0 2.0";
 11 |   SqExpKernel kernel(kernel_param);
 12 |   ConstantMean mean("0");
 13 |   SPGP gp(0.1, &kernel, &mean);
 14 |   
 15 |   Mat<REAL> X(1,200);
 16 |   Row<REAL> y;
 17 |   y.set_size(X.n_cols);
 18 |   
 19 |   for(int i=0; i < X.n_cols; ++i) {
 20 |     X(0,i) = (2.0*Math<REAL>::pi())*(REAL)i/(REAL)X.n_cols;
 21 |     y(i) = 1*sin(X(0,i));
 22 |   }
 23 |   
 24 |   Mat<REAL> Xb(1,X.n_cols/10);
 25 |   for(int i=0; i < Xb.n_cols; ++i) {
 26 |     Xb(0,i) = (0.25*Math<REAL>::pi())*(REAL)i/(REAL)Xb.n_cols;
 27 |   }
 28 | 
 29 |   gp.SetPseudoInputs(Xb);
 30 | 
 31 |   Mat<REAL> Xs(1,400);
 32 |   for(int i=0; i < Xs.n_cols; ++i) {
 33 |     Xs(0,i) = (2.0*Math<REAL>::pi())*(REAL)i/(REAL)X.n_cols;
 34 |   }
 35 | 
 36 |   Row<REAL> mu, var;
 37 |   REAL mu1, var1;
 38 | 
 39 |   cout << "Setting training data\n";
 40 |   gp.SetTraining(X, y);
 41 |   cout << "Making predictions\n";
 42 |   gp.Predict(Xs, mu, var);
 43 |   //gp.Predict(Xs.col(0), mu1, var1);
 44 | 
 45 |   /*
 46 |   Col<REAL> grad;
 47 |   gp.GradLikelihoodKernelParams(grad);
 48 |   grad.print("Likelihood Gradient (kernel)");
 49 |   */
 50 | 
 51 |   cout << "X:\n" << X << endl;
 52 |   cout << "y:\n" << y << endl;
 53 |   cout << "Xb:\n" << Xb << endl;
 54 |   cout << "Xs:\n" << Xs << endl;
 55 |   cout << "mu:\n" << mu << endl;
 56 | 
 57 |   // Output csv file for plotting
 58 | 
 59 | 
 60 |   ofstream out("test.txt");
 61 |   for(int i=0; i < Xs.n_cols; ++i) {
 62 |     out << Xs(0, i) << ", " << mu(i) << ", " << var(i) << "\n";
 63 |   }
 64 |   out.close();
 65 | 
 66 |   out.open("training.txt");
 67 |   for(int i=0; i < X.n_cols; ++i) {
 68 |     out << X(0, i) << ", " << y(i) << "\n";
 69 |   }
 70 |   out.close();
 71 | 
 72 |   out.open("pseudoinputs.txt");
 73 |   for(int i=0; i < Xb.n_cols; ++i) {
 74 |     out << Xb(0, i) << ", 0\n";
 75 |   }
 76 |   out.close();
 77 | 
 78 |   /*
 79 |   Row<REAL> error = y - mu;
 80 |   cout << "error:\n" << norm(error,2) << endl;
 81 |   */
 82 |   cout << "Log-likelihood: " << gp.ComputeLikelihood() << endl;
 83 | 
 84 |   gp.OptimizePseudoInputs(Xb, 20, 10, 0.001);
 85 |   gp.SetPseudoInputs(Xb);  
 86 |   cout << "Xb*:\n" << Xb << endl;
 87 | 
 88 |   cout << "Log-likelihood: " << gp.ComputeLikelihood() << endl;  
 89 | 
 90 |   gp.Predict(Xs, mu, var);
 91 |   
 92 |   out.open("test2.txt");
 93 |   for(int i=0; i < Xs.n_cols; ++i) {
 94 |     out << Xs(0, i) << ", " << mu(i) << ", " << var(i) << "\n";
 95 |   }
 96 |   out.close();
 97 | 
 98 |   out.open("pseudoinputs2.txt");
 99 |   for(int i=0; i < Xb.n_cols; ++i) {
100 |     out << Xb(0, i) << ", 0\n";
101 |   }
102 |   out.close();
103 | 
104 |   for(int i=0; i < Xb.n_cols; ++i) {
105 |     Xb(0,i) = (2.0*Math<REAL>::pi())*(REAL)i/(REAL)Xb.n_cols;
106 |   }
107 | 
108 |   gp.SetPseudoInputs(Xb);
109 |   gp.Predict(Xs, mu, var);
110 |   cout << "Log-likelihood: " << gp.ComputeLikelihood() << endl;  
111 | 
112 |   out.open("test3.txt");
113 |   for(int i=0; i < Xs.n_cols; ++i) {
114 |     out << Xs(0, i) << ", " << mu(i) << ", " << var(i) << "\n";
115 |   }
116 |   out.close();
117 | 
118 |   out.open("pseudoinputs3.txt");
119 |   for(int i=0; i < Xb.n_cols; ++i) {
120 |     out << Xb(0, i) << ", 0\n";
121 |   }
122 |   out.close();
123 | 
124 |   gp.OptimizePseudoInputs(Xb, 20, 10, 0.001);
125 |   gp.SetPseudoInputs(Xb);  
126 | 
127 |   gp.Predict(Xs, mu, var);
128 |   cout << "Log-likelihood: " << gp.ComputeLikelihood() << endl;  
129 | 
130 |   out.open("test4.txt");
131 |   for(int i=0; i < Xs.n_cols; ++i) {
132 |     out << Xs(0, i) << ", " << mu(i) << ", " << var(i) << "\n";
133 |   }
134 |   out.close();
135 | 
136 |   out.open("pseudoinputs4.txt");
137 |   for(int i=0; i < Xb.n_cols; ++i) {
138 |     out << Xb(0, i) << ", 0\n";
139 |   }
140 |   out.close();
141 | 
142 |   return 0;
143 | }
144 | 


--------------------------------------------------------------------------------