├── .gitignore ├── src ├── utility │ ├── index_median.cpp │ ├── meshgrid.cpp │ └── sample_from.cpp ├── data_science │ ├── scores.cpp │ ├── pairwise_kernel.cpp │ ├── ridge.cpp │ ├── polyfeat.cpp │ ├── splines.cpp │ ├── bin_data.cpp │ ├── k_fold.cpp │ └── logistic_regression.cpp ├── ode │ ├── rk4.cpp │ ├── am2.cpp │ ├── cheb.cpp │ ├── rk45.cpp │ ├── rk5i.cpp │ ├── poisson2d.cpp │ ├── rk34i.cpp │ ├── bvp_k.cpp │ ├── bvp_cheb.cpp │ ├── ivp.cpp │ └── diffmat.cpp ├── optimization │ ├── mgd.cpp │ ├── broyd.cpp │ ├── mix_fpi.cpp │ ├── gradient_optimizer.cpp │ ├── lmlsqr.cpp │ ├── lbfgs.cpp │ ├── simplex.cpp │ ├── quasi_newton.cpp │ ├── bfgs.cpp │ ├── newton.cpp │ ├── wolfe_step.cpp │ ├── nelder_mead.cpp │ ├── fmin.cpp │ ├── trust_newton.cpp │ └── fzero.cpp ├── derivatives │ ├── deriv.cpp │ ├── directional_grad.cpp │ ├── grad.cpp │ ├── spectral_deriv.cpp │ └── approx_jacobian.cpp ├── integrals │ ├── chebyshev_integral.cpp │ ├── lobatto_integral.cpp │ └── simpson_integral.cpp ├── interpolation │ ├── sinc_interp.cpp │ ├── lagrange_interp.cpp │ ├── cubic_interp.cpp │ └── hspline_interp.cpp └── neural_network │ ├── layer.cpp │ └── optimize.cpp ├── examples ├── optimization │ ├── simplex_ex.cpp │ ├── fmin_ex.cpp │ ├── pcg_ex.cpp │ ├── genetic_ex.cpp │ ├── gmres_ex.cpp │ ├── fzero_ex.cpp │ ├── newton_ex.cpp │ ├── minimize_ex.cpp │ └── lmlsqr_ex.cpp ├── neural_network │ ├── linear_reg_sgd_ex.cpp │ ├── linear_clf_sgd_ex.cpp │ ├── nn_classifier_ex.cpp │ ├── nn_regression_ex.cpp │ └── model_ex.cpp ├── utility │ ├── polynomials_ex.cpp │ └── sample_ex.cpp ├── derivatives │ ├── spectral_deriv_ex.cpp │ └── finite_dif_ex.cpp ├── data_science │ ├── kernel_smooth_ex.cpp │ ├── logistic_regression_ex.cpp │ ├── splines_ex.cpp │ ├── knn_ex.cpp │ ├── k_fold_ex.cpp │ ├── lasso_ex.cpp │ ├── kde_ex.cpp │ ├── ridge_ex.cpp │ ├── knn_classifier_ex.cpp │ └── kmeans_ex.cpp ├── ode │ ├── poisson_ex.cpp │ ├── ode_ex.cpp │ └── bvp_ex.cpp ├── interpolation │ └── interp_ex.cpp └── integration │ └── integrate_ex.cpp ├── include ├── numerics.hpp └── numerics │ ├── integrals.hpp │ ├── derivatives.hpp │ ├── utility.hpp │ └── interpolation.hpp ├── references.txt ├── CMakeLists.txt └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # ignore directories personal to my make build process and IDE setup 2 | /build 3 | /CMakeFiles 4 | /.vscode 5 | /archived_functions 6 | Makefile 7 | makefile 8 | *.cmake 9 | CMakeCache.txt 10 | install_manifest.txt 11 | libnumerics.so 12 | research.md 13 | *.pdf -------------------------------------------------------------------------------- /src/utility/index_median.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | uint numerics::index_median(const arma::vec& x) { 4 | std::vector> y; 5 | for (uint i=0; i < x.n_elem; ++i) { 6 | y.push_back( {x(i),i} ); 7 | } 8 | int nhalf = y.size()/2; 9 | std::nth_element(y.begin(), y.begin()+nhalf, y.end(), [](const std::pair& a, std::pair& b) -> bool {return a.first < b.first;}); 10 | return y.at(nhalf).second; 11 | } -------------------------------------------------------------------------------- /src/data_science/scores.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | double numerics::mse_score(const arma::vec& y, const arma::vec& yhat) { 4 | return arma::mean(arma::square(arma::conv_to::from(y - yhat))); 5 | } 6 | 7 | double numerics::r2_score(const arma::vec& y, const arma::vec& yhat) { 8 | return 1.0 - mse_score(y, yhat)/arma::var(arma::conv_to::from(y),1); 9 | } 10 | 11 | double numerics::accuracy_score(const arma::uvec& y, const arma::uvec& yhat) { 12 | return (long double)arma::sum(y == yhat) / y.n_elem; 13 | } -------------------------------------------------------------------------------- /src/ode/rk4.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | double numerics::ode::rk4::_step(double k, double& t1, arma::vec& u1, arma::vec& f1, const odefunc& f, const odejacobian* jacobian) { 4 | const arma::vec& u = _prev_u.back(); 5 | const double& t = _prev_t.back(); 6 | 7 | arma::vec v = k*_prev_f.back(); 8 | arma::vec p = u + rk4b[0]*v; 9 | for (short i=1; i < 5; ++i) { 10 | v = rk4a[i]*v + k*f(t+rk4c[i]*k, p); 11 | p += rk4b[i]*v; 12 | } 13 | 14 | t1 = t + k; 15 | u1 = std::move(p); 16 | f1 = f(t1,u1); 17 | return _cstep; 18 | } -------------------------------------------------------------------------------- /examples/optimization/simplex_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -g -Wall -o simplex simplex_ex.cpp -O3 -lnumerics -larmadillo 4 | 5 | int main() { 6 | arma::vec f = {1,3}; // z(x) = 1x + 3y 7 | arma::mat conRHS = {{1,1}, // x + y 8 | {5,2}, // 5x + 2y 9 | {1,2}}; // x + 2y 10 | arma::vec conLHS = {10, // x + y <= 10 11 | 20, // 5x + 2y <= 20 12 | 36}; // x + 2y <= 36 13 | arma::vec x; 14 | double y = numerics::optimization::simplex(x, f, conRHS, conLHS); 15 | std::cout << "max val: " << y << std::endl << "occuring at:\n" << x.t() << std::endl; 16 | return 0; 17 | } -------------------------------------------------------------------------------- /examples/optimization/fmin_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -g -Wall -o fmin fmin_ex.cpp -lnumerics -larmadillo 4 | 5 | double f(double x) { 6 | return 1 + std::exp(-100*x) + (0.99-x)/(x-1); // unimodal, very large derivative near minimum and singluar at one of the end points 7 | } 8 | 9 | int main() { 10 | double a=0, b=1; // bounds, appropriate to this function 11 | double x0=0; 12 | 13 | std::cout << "f(x) = 1 + exp(-100x) + (0.99-x)/(x-1)\n" 14 | << "\ttrue minimum: x = " << 0.0902125 << "\n" 15 | << "\tusing fminbnd: x = " << numerics::optimization::fminbnd(f,a,b) << "\n" 16 | << "\tusing fminsearch: x = " << numerics::optimization::fminsearch(f,x0) << "\n"; 17 | return 0; 18 | } -------------------------------------------------------------------------------- /src/optimization/mgd.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | bool numerics::optimization::MomentumGD::_step(arma::vec& dx, arma::vec& g1, const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 4 | if (_line_min) { 5 | _alpha = numerics::optimization::fminsearch([this,&x,&f](double a)->double{return f(x - std::abs(a)*_g);}, _alpha); 6 | _alpha = std::abs(_alpha); 7 | dx = -_alpha*_g; 8 | } else { 9 | arma::vec tmp; 10 | if (_y.is_empty()) tmp = x; 11 | else tmp = _y; 12 | 13 | _y = x - _alpha*_g; 14 | dx = -_alpha*_g + _damping_param*(_y - tmp); 15 | } 16 | 17 | if (dx.has_nan()) return false; 18 | 19 | g1 = df(x + dx); 20 | return true; 21 | } 22 | -------------------------------------------------------------------------------- /src/utility/meshgrid.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | /* MESHGRID : produces meshgrids from two different input vectors. 4 | * --- xgrid : matrix to assign x values mesh to. 5 | * --- ygrid : matrix to assign y values mesh to. 6 | * --- x : x values. 7 | * --- y : y values. */ 8 | void numerics::meshgrid(arma::mat& xgrid, arma::mat& ygrid, const arma::vec& x, const arma::vec& y) { 9 | xgrid = arma::repmat(x.t(), y.n_elem, 1); 10 | ygrid = arma::repmat(y, 1, x.n_elem); 11 | } 12 | 13 | /* MESHGRID : produces meshgrid from single input vector, correspondingly ygrid = xgrid.t() 14 | * --- xgrid : matrix to assign x values to. 15 | * --- x : x values. */ 16 | void numerics::meshgrid(arma::mat& xgrid, const arma::vec& x) { 17 | xgrid = arma::repmat(x.t(), x.n_elem, 1); 18 | } -------------------------------------------------------------------------------- /examples/neural_network/linear_reg_sgd_ex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // g++ -g -Wall -O3 -o sgd_regression linear_reg_sgd_ex.cpp -lnumerics -larmadillo 4 | 5 | int main() { 6 | int N = 10000; 7 | arma::mat x = arma::randn(N, 5); 8 | arma::vec w = 10*arma::randu(5)-5; 9 | double b = 3.14; 10 | 11 | arma::vec y = x*w + b + 2.5*arma::randn(N); 12 | 13 | std::string loss = "mse"; 14 | long max_iter = 200; 15 | double tol = 1e-4; 16 | double l2 = 1e-4; 17 | double l1 = 0; 18 | std::string optimizer = "adam"; 19 | bool verbose = true; 20 | 21 | numerics::LinearRegressorSGD model(loss, max_iter, tol, l2, l1, optimizer, verbose); 22 | model.fit(x,y); 23 | 24 | std::cout << "R2 : " << std::fixed << std::setprecision(2) << model.score(x,y) << "\n"; 25 | return 0; 26 | } -------------------------------------------------------------------------------- /src/ode/am2.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | double numerics::ode::am2::_step(double k, double& t1, arma::vec& u1, arma::vec& f1, const odefunc& f, const odejacobian* jacobian) { 4 | arma::vec z = _prev_u.back() + 0.5*k*_prev_f.back(); 5 | 6 | double t = _prev_t.back(); 7 | 8 | auto am2f = [t,k,&z,&f,&f1](const arma::vec& v) -> arma::vec { 9 | f1 = f(t+k,v); 10 | return v - (z + 0.5*k*f1); 11 | }; 12 | 13 | u1 = z; 14 | if (jacobian == nullptr) solver->fsolve(u1, am2f); 15 | else { 16 | auto am2J = [t,k,&z,&jacobian](const arma::vec& v) -> arma::mat { 17 | arma::mat J = (*jacobian)(t+k, v); 18 | return arma::eye(arma::size(J)) - 0.5*k*J; 19 | }; 20 | solver->fsolve(u1, am2f, am2J); 21 | } 22 | t1 = t + k; 23 | return _cstep; 24 | } -------------------------------------------------------------------------------- /src/optimization/broyd.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | bool numerics::optimization::Broyden::_step(arma::vec& dx, arma::vec& F1, const arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 4 | if (_J.has_nan()) return false; 5 | 6 | bool success = arma::solve(dx, _J, -_F); 7 | if (not success) { 8 | if (jacobian == nullptr) _J = approx_jacobian(f, x); 9 | else _J = (*jacobian)(x); 10 | 11 | success = arma::solve(dx, _J, -_F); 12 | if (not success) return false; 13 | } 14 | 15 | auto line_f = [&F1,&x,&f,&dx](double a) -> double { 16 | F1 = f(x + a*dx); 17 | return arma::norm(F1); 18 | }; 19 | if (line_f(1.0) > 0.99*arma::norm(_F)) { 20 | double a = fminbnd(line_f, 0.0, 1.0, 1e-2); 21 | dx *= a; 22 | } 23 | 24 | arma::vec y = F1 - _F; 25 | _J += (y - _J*dx)*dx.t() / arma::dot(dx, dx); 26 | 27 | return true; 28 | } -------------------------------------------------------------------------------- /src/utility/sample_from.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | int numerics::sample_from(const arma::vec& pdf, const arma::uvec& labels) { 4 | int n = pdf.n_elem; 5 | int i; 6 | double cdf = 0, rval = arma::randu(); 7 | for (i=0; i < n; ++i) { 8 | if (cdf < rval && rval <= cdf + pdf(i)) break; 9 | cdf += pdf(i); 10 | } 11 | if ( labels.is_empty() ) return i; 12 | else return labels(i); 13 | } 14 | 15 | arma::uvec numerics::sample_from(int n, const arma::vec& pdf, const arma::uvec& labels) { 16 | int m = pdf.n_elem; 17 | arma::vec rvals = arma::randu(n); 18 | arma::uvec samples = arma::zeros(n); 19 | double cdf = 0; 20 | double i; 21 | for (i=0; i < m; ++i) { 22 | samples(arma::find(cdf < rvals && rvals <= cdf + pdf(i))).fill(i); 23 | cdf += pdf(i); 24 | } 25 | if (labels.is_empty()) return samples; 26 | else return labels(samples); 27 | } -------------------------------------------------------------------------------- /examples/utility/polynomials_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -g -Wall -o polynomials polynomials_ex.cpp -O3 -lnumerics -larmadillo 4 | 5 | int main() { 6 | arma::vec p = {1, 1, 1, 1}; // x^3 + x^2 + x + 1 7 | 8 | arma::vec dp = numerics::polyder(p); // should return 3x^2 + 2x + 1 9 | arma::vec d2p = numerics::polyder(p,2); // should return 6x + 2 10 | 11 | arma::vec ip = numerics::polyint(p); // should return x^4/4 + x^3/3 + x^2/2 + x + 0 12 | arma::vec ip_2 = numerics::polyint(p,2); // should return x^4/4 + x^3/3 + x^2/2 + x + 2 13 | 14 | std::cout << "original polynomial: " << p.t() << std::endl 15 | << "first derivative: " << dp.t() << std::endl 16 | << "second derivative: " << d2p.t() << std::endl 17 | << "integral (with coefficient +0):" << ip.t() << std::endl 18 | << "integral (with coefficient +2):" << ip_2.t() << std::endl; 19 | 20 | return 0; 21 | } -------------------------------------------------------------------------------- /src/derivatives/deriv.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /* deriv(f, x, h, catch_zero) : computes the approximate derivative of a function of a single variable. 4 | * --- f : f(x) whose derivative to approximate. 5 | * --- x : point to evaluate derivative. 6 | * --- h : finite difference step size; method is O(h^4). 7 | * --- catch_zero: rounds near zero elements to zero. */ 8 | double numerics::deriv(const std::function& f, double x, double h, bool catch_zero, short npt) { 9 | double df; 10 | if (npt == 1) df = (f(x + h) - f(x))/h; 11 | else if (npt == 2) df = (f(x + h) - f(x - h))/(2*h); 12 | else if (npt == 4) df = (f(x - 2*h) - 8*f(x - h) + 8*f(x + h) - f(x + 2*h))/(12*h); 13 | else { 14 | throw std::invalid_argument("only 1, 2, and 4 point FD derivatives supported (not " + std::to_string(npt) + ")."); 15 | } 16 | 17 | if (catch_zero and (std::abs(df) < h/2)) return 0; 18 | 19 | return df; 20 | } -------------------------------------------------------------------------------- /src/data_science/pairwise_kernel.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | arma::mat numerics::cubic_kernel(const arma::mat& x) { 4 | arma::mat K; 5 | K.set_size(x.n_rows, x.n_rows); 6 | for (u_int i=0; i < x.n_rows; ++i) { 7 | for (u_int j=0; j < i; ++j) { 8 | K(i,j) = std::pow(arma::norm(x.row(i) - x.row(j)), 3); 9 | K(j,i) = K(i,j); 10 | } 11 | K(i,i) = 0; 12 | } 13 | return K; 14 | } 15 | 16 | arma::mat numerics::cubic_kernel(const arma::mat& x1, const arma::mat& x2) { 17 | if (x1.n_cols != x2.n_cols) { 18 | throw std::invalid_argument("require x1.n_cols (=" + std::to_string(x1.n_cols) + ") == x2.n_cols (=" + std::to_string(x2.n_cols) + ")"); 19 | } 20 | arma::mat K; 21 | K.set_size(x2.n_rows, x1.n_rows); 22 | for (u_int i=0; i < x2.n_rows; ++i) { 23 | for (u_int j=0; j < x1.n_rows; ++j) { 24 | K(i,j) = std::pow(arma::norm(x1.row(j) - x2.row(i)), 3); 25 | } 26 | } 27 | return K; 28 | } -------------------------------------------------------------------------------- /examples/neural_network/linear_clf_sgd_ex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // g++ -g -Wall -O3 -o sgd_classifier linear_clf_sgd_ex.cpp -lnumerics -larmadillo 4 | 5 | arma::uvec gen_classes(const arma::mat& x, int n_classes) { 6 | arma::mat xx = x + 2.5*arma::randn(arma::size(x)); 7 | 8 | numerics::KMeansSGD km(n_classes); 9 | return km.fit_predict(xx); 10 | } 11 | 12 | int main() { 13 | int N = 10000; 14 | arma::mat x = arma::randn(N, 5); 15 | 16 | 17 | arma::uvec y = gen_classes(x, 3); 18 | 19 | std::string loss = "categorical_crossentropy"; 20 | long max_iter = 200; 21 | double tol = 1e-4; 22 | double l2 = 1e-4; 23 | double l1 = 0; 24 | std::string optimizer = "adam"; 25 | bool verbose = true; 26 | 27 | numerics::LinearClassifierSGD model(loss, max_iter, tol, l2, l1, optimizer, verbose); 28 | model.fit(x,y); 29 | 30 | std::cout << "accuracy : " << std::fixed << std::setprecision(2) << model.score(x,y) << "\n"; 31 | return 0; 32 | } -------------------------------------------------------------------------------- /src/integrals/chebyshev_integral.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | double numerics::chebyshev_integral(const std::function& f, double a, double b, uint m) { 4 | arma::vec x = arma::regspace(0,m-1)/(m-1); 5 | x = 0.5*(1+arma::cos(M_PI * x))*(b-a) + a; 6 | 7 | arma::vec y = x; 8 | y.for_each([&f](arma::vec::elem_type& u) -> void {u = f(u);}); 9 | y = arma::join_cols(y, arma::flipud(y.rows(1,y.n_rows-2))); 10 | arma::vec c = arma::real(arma::fft(y)); 11 | c = arma::flipud(c.rows(0,m-1))/(m-1); 12 | c(0) /= 2; 13 | c(m-1) /= 2; 14 | 15 | arma::vec p1 = arma::zeros(m); p1(m-1) = 1; 16 | arma::vec p2 = arma::zeros(m); p2(m-2) = 1; 17 | arma::vec p = c(m-1)*p1 + c(m-2)*p2; 18 | for (uint i=m-3; i > 0; --i) { 19 | arma::vec temp = p2; 20 | p2 = arma::shift(2*p2,-1) - p1; 21 | p1 = temp; 22 | p += c(i) * p2; 23 | } 24 | p = polyint(p); 25 | return (b-a)/2 * arma::as_scalar(arma::diff(arma::polyval(p,arma::vec({-1,1})))); 26 | } -------------------------------------------------------------------------------- /src/derivatives/directional_grad.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | arma::vec numerics::directional_grad(const std::function& f, const arma::vec& x, const arma::vec& v, double h, bool catch_zero, short npt) { 4 | if (x.n_elem != v.n_elem) { 5 | throw std::invalid_argument( 6 | "cannot compute derivative of f at x in the direction v when x.n_elem (=" 7 | + std::to_string(x.n_elem) + ") does not match v.n_elem (=" 8 | + std::to_string(v.n_elem) + ")." 9 | ); 10 | } 11 | 12 | arma::vec Jv; 13 | 14 | double C = h / arma::norm(v); 15 | if (npt == 1) Jv = (f(x+C*v) - f(x)) / C; 16 | else if (npt == 2) Jv = (f(x+C*v) - f(x-C*v)) / (2*C); 17 | else if (npt == 4) Jv = (f(x - 2*C*v) - 8*f(x - C*v) + 8*f(x + C*v) - f(x + 2*C*v)) / (12*C); 18 | else { 19 | throw std::invalid_argument("only 1, 2, and 4 point FD derivatives supported (not " + std::to_string(npt) + ")."); 20 | } 21 | 22 | if (catch_zero) Jv(arma::find(arma::abs(Jv) < h/2)).zeros(); 23 | 24 | return Jv; 25 | } -------------------------------------------------------------------------------- /src/ode/cheb.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /* cheb(D, x, L, R, m) : constructs the Chebyshev spectral differentiation matrix. 4 | * --- D : cheb matrix storage. 5 | * --- x : grid value storage. 6 | * --- L,R : limits on x. 7 | * --- m : number of points. */ 8 | void numerics::ode::cheb(arma::mat& D, arma::vec& x, double L, double R, uint m) { 9 | m = m-1; 10 | x = arma::regspace(0,m); 11 | x = -1*arma::cos(M_PI * x / m); // standard cheb nodes on [-1,1] 12 | x = (R - L) * x/2 + (L + R)/2; // transformation from [-1,1] -> [L,R] 13 | 14 | arma::vec c = arma::ones(m+1); c(0) = 2; c(m) = 2; 15 | c( arma::regspace(1,2,m) ) *= -1; // every other element negative 16 | 17 | D = arma::repmat(x,1,m+1); 18 | D -= D.t(); 19 | D = c * ( 1/c.t() ) / (D + arma::eye(m+1, m+1)); 20 | D -= arma::diagmat(arma::sum(D,1)); 21 | } 22 | 23 | /* cheb(D, x, m) : constructs the Chebyshev matrix on the unit interval [-1,1]. 24 | * --- D : cheb matrix storage. 25 | * --- x : grid storage. 26 | * --- m : number of points on interval. */ 27 | void numerics::ode::cheb(arma::mat& D, arma::vec& x, uint m) { 28 | cheb(D,x,-1,1,m); 29 | } -------------------------------------------------------------------------------- /examples/derivatives/spectral_deriv_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o spectralD spectral_deriv_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector ddvec; 7 | 8 | double f(double x) { 9 | return std::exp(-x*x); 10 | } 11 | 12 | arma::vec df(arma::vec x) { 13 | return (-2*x) % arma::exp(-x%x); 14 | } 15 | 16 | int main() { 17 | double a = -3; double b = 3; double m = 50; 18 | 19 | arma::vec x = arma::linspace(a,b); 20 | arma::vec y = df(x); 21 | 22 | numerics::PolyInterp dy = numerics::spectral_deriv(f,a,b,m); 23 | arma::vec v = dy.predict(x); 24 | 25 | std::cout << "max error : " << arma::norm(v - y, "inf") << std::endl; 26 | 27 | ddvec xx = arma::conv_to::from(x); 28 | ddvec yy = arma::conv_to::from(y); 29 | ddvec vv = arma::conv_to::from(v); 30 | 31 | matplotlibcpp::title("spectral derivative"); 32 | matplotlibcpp::named_plot("actual derivative", xx, yy, "--k"); 33 | matplotlibcpp::named_plot("spectral approximation", xx, vv, "-r"); 34 | matplotlibcpp::legend(); 35 | matplotlibcpp::show(); 36 | 37 | return 0; 38 | } -------------------------------------------------------------------------------- /src/derivatives/grad.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /* GRAD : computes the gradient of a function of multiple variables. 4 | * --- f : f(x) whose gradient to approximate. 5 | * --- x : vector to evaluate gradient at. 6 | * --- h : finite difference step size. method is O(h^4). 7 | * --- catch_zero: rounds near zero elements to zero. */ 8 | arma::vec numerics::grad(const std::function& f, const arma::vec& x, double h, bool catch_zero, short npt) { 9 | uint n = x.n_elem; 10 | arma::vec g(n); 11 | if (npt == 1) { // specialize this instance to minimize repeated calculations. 12 | g.fill(-f(x)); 13 | arma::vec y = x; 14 | for (uint i=0; i < n; ++i) { 15 | y(i) += h; 16 | g(i) += f(y); 17 | y(i) = x(i); 18 | } 19 | g /= h; 20 | g(arma::find(arma::abs(g) < h/2)).zeros(); 21 | } else { 22 | for (uint i=0; i < n; ++i) { 23 | auto ff = [&f,x,i](double t) -> double { 24 | arma::vec y = x; 25 | y(i) = t; 26 | return f(y); 27 | }; 28 | g(i) = deriv(ff, x(i), h, catch_zero, npt); 29 | } 30 | } 31 | 32 | return g; 33 | } -------------------------------------------------------------------------------- /src/optimization/mix_fpi.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::optimization::MixFPI::fix(arma::vec& x, const VecFunc& f) { 4 | u_long n = x.n_elem; 5 | 6 | arma::mat F = arma::zeros(n, _steps_to_remember); 7 | arma::mat X = arma::zeros(n, _steps_to_remember); 8 | 9 | arma::mat FF = arma::ones(n+1, _steps_to_remember); 10 | 11 | arma::vec b = arma::zeros(n+1); 12 | b(n) = 1; 13 | 14 | _n_iter = 0; 15 | u_long head; 16 | VerboseTracker T(_max_iter); 17 | if (_v) T.header("max|x-f(x)|"); 18 | while (true) { 19 | if (_v) T.iter(_n_iter, arma::norm(F.col(head) - x,"inf")); 20 | 21 | head = _n_iter % _steps_to_remember; 22 | 23 | F.col(head) = f(x); 24 | X.col(head) = x; 25 | 26 | if (arma::norm(F.col(head) - x.col(head),"inf") < _xtol) { 27 | _exit_flag = 0; 28 | if (_v) T.success_flag(); 29 | break; 30 | } 31 | 32 | FF.submat(0,head,n-1,head) = F.col(head) - X.col(head); 33 | if (_n_iter < _steps_to_remember) x = F.cols(0,_n_iter) * arma::solve(FF.cols(0,_n_iter), b); 34 | else x = F * arma::solve(FF, b); 35 | 36 | _n_iter++; 37 | 38 | if (_n_iter >= _max_iter) { 39 | _exit_flag = 2; 40 | if (_v) T.max_iter_flag(); 41 | return; 42 | } 43 | } 44 | } -------------------------------------------------------------------------------- /src/interpolation/sinc_interp.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | /* sinc_interp(x, y, u) : interpolate data points with sinc functions 4 | * --- x : uniformly spaced domain data 5 | * --- y : points to interpolate 6 | * --- u : points to evaluate interpolation on */ 7 | arma::mat numerics::sinc_interp(const arma::vec& x, const arma::mat& y, const arma::vec& u) { 8 | int n = x.n_elem; 9 | if (x.n_elem != y.n_rows) { // dimension error 10 | std::cerr << "sinc_interp() error: interpolation could not be constructed, x and y vectors must be the same length." << std::endl 11 | << "\tx has " << x.n_elem << " elements, y has " << y.n_elem << " elements." << std::endl; 12 | return {NAN}; 13 | } 14 | 15 | for (int i(0); i < n - 1; ++i) { // repeated x error 16 | for (int j(i+1); j < n; ++j) { 17 | if (std::abs(x(i) - x(j)) < arma::datum::eps) { 18 | std::cerr << "sinc_interp() error: one or more x values are repeating." << std::endl; 19 | return {NAN}; 20 | } 21 | } 22 | } 23 | 24 | double h = x(1) - x(0); 25 | arma::mat v(u.n_elem, y.n_cols, arma::fill::zeros); 26 | for (int i(0); i < n; ++i) { 27 | arma::mat s = arma::repmat(y.row(i), u.n_elem, 1); 28 | s.each_col() %= arma::sinc( (u - x(i))/h ); 29 | v += s; 30 | } 31 | 32 | return v; 33 | } -------------------------------------------------------------------------------- /include/numerics.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NUMERICS_HPP 2 | #define NUMERICS_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | #define ARMA_USE_SUPERLU 1 // optional, but really should be used when handling sparse matrices 16 | #include 17 | 18 | /* Copyright 2019 Amit Rotem 19 | 20 | Licensed under the Apache License, Version 2.0 (the "License"); 21 | you may not use this file except in compliance with the License. 22 | You may obtain a copy of the License at 23 | 24 | http://www.apache.org/licenses/LICENSE-2.0 25 | 26 | Unless required by applicable law or agreed to in writing, software 27 | distributed under the License is distributed on an "AS IS" BASIS, 28 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | See the License for the specific language governing permissions and 30 | limitations under the License. */ 31 | 32 | namespace numerics { 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | } 42 | 43 | #endif -------------------------------------------------------------------------------- /src/data_science/ridge.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::RidgeCV::fit(const arma::mat& X, const arma::vec& y) { 4 | _check_xy(X,y); 5 | _dim = X.n_cols; 6 | uint n_obs = X.n_rows; 7 | 8 | arma::mat covariance; 9 | arma::vec yp; 10 | arma::mat P; 11 | 12 | if (_fit_intercept) { 13 | P = _add_intercept(X); 14 | covariance = P.t() * P; 15 | yp = P.t() * y; 16 | } else { 17 | covariance = X.t() * X; 18 | yp = X.t() * y; 19 | } 20 | arma::eig_sym(_eigvals, _eigvecs, covariance); 21 | yp = _eigvecs.t() * yp; 22 | auto GCV = [&](double lam) -> double { 23 | lam = std::pow(10.0, lam); 24 | _w = _eigvecs * (yp / (_eigvals + lam)); 25 | 26 | _df = arma::sum(_eigvals / (_eigvals + lam)); 27 | 28 | double mse; 29 | if (_fit_intercept) mse = mse_score(y, P*_w); 30 | else mse = mse_score(y, X*_w); 31 | 32 | return mse * std::pow(n_obs / (n_obs-_df), 2); 33 | }; 34 | _lambda = optimization::fminbnd(GCV, -8, 4); 35 | _lambda = std::pow(10.0, _lambda); 36 | _split_weights(); 37 | } 38 | 39 | arma::vec numerics::RidgeCV::predict(const arma::mat& X) const { 40 | _check_x(X); 41 | return _b + X * _w; 42 | } 43 | 44 | double numerics::RidgeCV::score(const arma::mat& x, const arma::vec& y) const { 45 | _check_xy(x, y); 46 | return r2_score(y, predict(x)); 47 | } -------------------------------------------------------------------------------- /examples/utility/sample_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -Wall -g -o sample_from sample_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | using namespace numerics; 7 | typedef std::vector ddvec; 8 | 9 | const int n = 10; 10 | const double p = 0.3; 11 | 12 | int coef[n+1][n+1]; 13 | 14 | void choose() { 15 | for (int i=0; i <= n; ++i) { 16 | for (int j=0; j <= i; ++j) { 17 | if (j == 0 || j == i) coef[i][j] = 1; 18 | else coef[i][j] = coef[i - 1][j - 1] + coef[i - 1][j]; 19 | } 20 | } 21 | } 22 | 23 | double binom_pdf(int k) { 24 | return coef[n][k] * std::pow(p, k) * std::pow(1-p, n-k); 25 | } 26 | 27 | int main() { 28 | choose(); 29 | 30 | arma::vec x = arma::regspace(0,n); 31 | arma::vec pdf = arma::zeros(n+1); 32 | for (int i=0; i <= n; ++i) pdf(i) = binom_pdf(i); 33 | 34 | arma::vec sample = sample_from(1000, pdf); 35 | 36 | ddvec xx = arma::conv_to::from(arma::linspace(0,n)); 37 | ddvec hist = arma::conv_to::from(sample); 38 | ddvec pdf0 = arma::conv_to::from( 39 | cubic_interp(x,1000*pdf)(arma::linspace(0,n)) 40 | ); 41 | 42 | matplotlibcpp::named_hist("sample", hist, 30); 43 | matplotlibcpp::named_plot("pdf",xx, pdf0, "-r"); 44 | matplotlibcpp::legend(); 45 | matplotlibcpp::show(); 46 | 47 | return 0; 48 | } -------------------------------------------------------------------------------- /examples/neural_network/nn_classifier_ex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // g++ -g -Wall -O3 -o nn_classifier nn_classifier_ex.cpp -lnumerics -larmadillo 4 | 5 | arma::uvec gen_classes(const arma::mat& x, int n_classes) { 6 | arma::mat xx = x + 0.1*arma::randn(arma::size(x)); 7 | 8 | numerics::KMeansSGD km(n_classes*5); 9 | arma::uvec y = km.fit_predict(xx); 10 | arma::umat cc = arma::randperm(n_classes*5); 11 | cc.reshape(n_classes,5); 12 | std::map cs; 13 | for (int i=0; i < 5; ++i) { 14 | for (int j=0; j < n_classes; ++j) { 15 | cs[cc(j,i)] = i; 16 | } 17 | } 18 | y.transform([&cs](arma::uword yi)->arma::uword{return cs[yi];}); 19 | return y; 20 | } 21 | 22 | int main() { 23 | int N = 1000; 24 | arma::mat x = arma::randn(N, 2); 25 | 26 | arma::uvec y = gen_classes(x, 3); 27 | 28 | std::string loss = "categorical_crossentropy"; 29 | std::vector> layers = {{100,"relu"}}; 30 | long max_iter = 200; 31 | double tol = 1e-2; 32 | double l2 = 1e-4; 33 | double l1 = 0; 34 | std::string optimizer = "adam"; 35 | bool verbose = true; 36 | 37 | numerics::NeuralNetClassifier model(layers, loss, max_iter, tol, l2, l1, optimizer, verbose); 38 | model.fit(x,y); 39 | 40 | std::cout << "accuracy : " << std::fixed << std::setprecision(2) << model.score(x,y) << "\n"; 41 | return 0; 42 | } 43 | -------------------------------------------------------------------------------- /examples/derivatives/finite_dif_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -Wall -g -o finite_dif finite_dif_ex.cpp -O3 -lnumerics -larmadillo 4 | 5 | using namespace numerics; 6 | 7 | int main() { 8 | auto f = [](double x) -> double {return std::sin(x);}; 9 | auto df = [](double x) -> double {return std::cos(x);}; 10 | 11 | double df0 = df(0.5); 12 | double df1 = deriv(f,0.5); 13 | 14 | std::cout << "the derivative of sin(0.5) is: " << df0 << std::endl 15 | << "\tthe approximation of that derivative with h = 1e-2 is: " << df1 << std::endl 16 | << "\ttrue error: " << std::abs(df0 - df1) << std::endl; 17 | 18 | auto g = [](const arma::vec& x) -> double {return arma::norm(x);}; 19 | auto dg = [](const arma::vec& x) -> arma::vec { 20 | arma::vec v = arma::zeros(3); 21 | v(0) = x(0)/arma::norm(x); 22 | v(1) = x(1)/arma::norm(x); 23 | v(2) = x(2)/arma::norm(x); 24 | return v; 25 | }; 26 | arma::vec x = {0.4, 0.5, 0.33}; 27 | 28 | arma::mat dg1 = grad(g, x).t(); 29 | arma::mat dg0 = dg(x).t(); 30 | 31 | std::cout << "g(x,y,z) = sqrt(x^2 + y^2 + z^2)" << std::endl 32 | << "\tthe true gradient of g([0.4 0.5 0.33]) is:" << dg0 33 | << "\tthe approximate gradient of g with h = 1e-2 is:" << dg1 34 | << "\t\t||true error||: " << arma::norm(dg0 - dg1, "Inf") << std::endl; 35 | 36 | return 0; 37 | 38 | } -------------------------------------------------------------------------------- /examples/optimization/pcg_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -g -Wall -o pcg pcg_ex.cpp -O3 -lnumerics -larmadillo -lsuperlu 4 | 5 | 6 | int main() { 7 | int n = 2000; 8 | arma::arma_rng::set_seed_random(); 9 | arma::sp_mat A = arma::sprandn(5*n,n,0.001); // the sparser the system the greater the difference in performance b/w pcg() and spsolve() 10 | A = A.t() * A; 11 | std::cout << "nonzeros / n = " << (double)A.n_nonzero / A.n_elem << std::endl; 12 | 13 | arma::vec b = arma::randn(n); // we can solve multiple equations at once 14 | 15 | auto tic = std::chrono::high_resolution_clock::now(); 16 | arma::mat x = arma::spsolve(A,b); 17 | auto toc = std::chrono::high_resolution_clock::now(); 18 | double dur = std::chrono::duration_cast(toc-tic).count()/1000.0; 19 | 20 | std::cout << "For the matrix of order " << n << ", the direct solver took " << dur << " seconds" << std::endl << std::endl; 21 | 22 | arma::vec y; 23 | tic = std::chrono::high_resolution_clock::now(); 24 | numerics::optimization::pcg(y, A, b); 25 | toc = std::chrono::high_resolution_clock::now(); 26 | dur = std::chrono::duration_cast(toc-tic).count()/1000.0; 27 | 28 | std::cout << "Conjugate gradient method took " << dur << " seconds" << std::endl 29 | << "the relative error was: " << arma::norm(A*y - b) / arma::norm(b) << std::endl << std::endl; 30 | 31 | return 0; 32 | } -------------------------------------------------------------------------------- /src/optimization/gradient_optimizer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::optimization::GradientOptimizer::_initialize(const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 4 | _g = df(x); 5 | } 6 | 7 | void numerics::optimization::GradientOptimizer::_solve(arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 8 | VerboseTracker T(_max_iter); 9 | if (_v) T.header("f"); 10 | 11 | _initialize(x, f, df, hessian); 12 | arma::vec dx, g1; 13 | 14 | _n_iter = 0; 15 | while (true) { 16 | bool successful_step = _step(dx, g1, x, f, df, hessian); 17 | 18 | if (not successful_step) { 19 | _exit_flag = 3; 20 | if (_v) T.nan_flag(); 21 | return; 22 | } 23 | double xtol = _xtol*std::max(1.0, arma::norm(x,"inf")); 24 | x += dx; 25 | _n_iter++; 26 | 27 | _g = std::move(g1); 28 | 29 | if (_v) T.iter(_n_iter, f(x)); 30 | 31 | if (arma::norm(_g,"inf") < _ftol) { 32 | _exit_flag = 0; 33 | if (_v) T.success_flag(); 34 | return; 35 | } 36 | 37 | if (arma::norm(dx,"inf") < xtol) { 38 | _exit_flag = 1; 39 | if (_v) T.success_flag(); 40 | return; 41 | } 42 | 43 | if (_n_iter >= _max_iter) { 44 | _exit_flag = 2; 45 | if (_v) T.max_iter_flag(); 46 | return; 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /examples/data_science/kernel_smooth_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o kernel_smooth kernel_smooth_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector dvec; 7 | 8 | arma::vec f(arma::vec& X) { 9 | arma::vec y = arma::zeros(arma::size(X)); 10 | for (int i=1; i < 10; ++i) { 11 | y += arma::sin(i*X)/i; 12 | } 13 | return 0.5 - y/M_PI; 14 | } 15 | 16 | int main() { 17 | arma::vec x = 5*arma::randu(200)-2.5; 18 | arma::vec y = f(x) + 0.1*arma::randn(arma::size(x)); 19 | 20 | std::string k = "square"; // gaussian, square, triangle, parabolic 21 | bool data_to_bins = false; 22 | 23 | numerics::KernelSmooth model(k, data_to_bins); 24 | model.fit(x,y); 25 | arma::vec t = arma::linspace(-3,3,300); 26 | arma::vec yhat = model.predict(t); 27 | 28 | std::cout << "bandwidth : " << model.bandwidth << std::endl; 29 | std::cout << "model r^2 : " << model.score(x,y) << std::endl; 30 | 31 | dvec xx = arma::conv_to::from(x); 32 | dvec yy = arma::conv_to::from(y); 33 | dvec tt = arma::conv_to::from(t); 34 | dvec uu = arma::conv_to::from(yhat); 35 | dvec vv = arma::conv_to::from(f(t)); 36 | 37 | matplotlibcpp::plot(xx,yy,"or"); 38 | matplotlibcpp::named_plot("kernel fit", tt, uu, "-b"); 39 | matplotlibcpp::named_plot("actual function", tt, vv, "--k"); 40 | matplotlibcpp::legend(); 41 | matplotlibcpp::show(); 42 | 43 | return 0; 44 | } -------------------------------------------------------------------------------- /examples/data_science/logistic_regression_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o logistic_regression logistic_regression_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | std::vector conv(const arma::mat& u) { 7 | return arma::conv_to>::from(u); 8 | } 9 | std::vector conv(const arma::umat& u) { 10 | return arma::conv_to>::from(u); 11 | } 12 | 13 | int main() { 14 | arma::mat X = (2*arma::randu(100,1) - 1)*M_PI; 15 | arma::uvec y = (arma::abs(X)+0.1*arma::randn(arma::size(X)) > 1.5); 16 | 17 | numerics::PolyFeatures poly(3); 18 | arma::mat Xp3 = poly.fit_predict(X); 19 | numerics::LogisticRegression model; 20 | // model.set_lambda(1e-2); 21 | model.fit(Xp3,y); 22 | 23 | arma::vec xgrid = arma::linspace(-M_PI,M_PI,1000); 24 | arma::mat p = model.predict_proba(poly.predict(xgrid)); 25 | arma::uvec yh = model.predict(Xp3); 26 | 27 | std::cout << "lambda:" << model.lambda << "\n" 28 | << "accuracy: " << model.score(Xp3,y) << "\n"; 29 | 30 | auto xx = conv(X); 31 | auto y1 = conv(y); 32 | auto xxgrid = conv(xgrid); 33 | auto p1 = conv(p.col(1)); 34 | auto c1 = conv(yh); 35 | 36 | matplotlibcpp::named_plot("observed classes",xx,y1,"o"); 37 | matplotlibcpp::named_plot("fit",xxgrid,p1,"-"); 38 | matplotlibcpp::named_plot("predicted classes",xx,c1,"."); 39 | matplotlibcpp::legend(); 40 | matplotlibcpp::show(); 41 | 42 | return 0; 43 | } -------------------------------------------------------------------------------- /src/derivatives/spectral_deriv.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /* spectral_deriv(f, a, b, sample_points) : compute spectrally accurate derivative of function over an interval 4 | * --- f : f(x) function to compute derivative of 5 | * --- [a,b] : interval over which to evaluate derivative over 6 | * --- sample_points: number of points to sample (more->more accurate) {default = 50} */ 7 | numerics::Polynomial numerics::spectral_deriv(const std::function& f, double a, double b, uint sample_points) { 8 | arma::cx_double i(0,1); // i^2 = -1 9 | int N = sample_points - 1; 10 | 11 | arma::vec y = arma::cos( arma::regspace(0,N)*M_PI/N ); 12 | arma::vec v = y; 13 | v.for_each([&f,&b,&a](arma::vec::elem_type& u){u = f(0.5*(u+1)*(b-a)+a);}); 14 | 15 | arma::uvec ii = arma::regspace(0,N-1); 16 | v = arma::join_cols(v, arma::reverse(v.rows(1,N-1))); 17 | v = arma::real(arma::fft(v)); 18 | arma::cx_vec u = i*arma::join_cols(arma::join_cols(arma::regspace(0,N-1), arma::vec({0})), arma::regspace(1-N, -1)); 19 | arma::vec W = arma::real(arma::ifft(u%v)); 20 | W.rows(1,N-1) = -W.rows(1,N-1) / arma::sqrt(1 - arma::square(y.rows(1,N-1))); 21 | W(0) = 0.5*N*v(N) + arma::accu(arma::square(ii) % v.rows(ii+1)) / N; 22 | arma::vec j = arma::ones(N); j.rows(arma::regspace(1,2,N-1)) *= 2; 23 | W(N) = 0.5*std::pow(-1,N+1)*N*v(N) + arma::accu(j % arma::square(ii) % v.rows(ii+1)) / N; 24 | W = W.rows(0,N); 25 | W /= (b-a)/2; 26 | 27 | return Polynomial(0.5*(y+1)*(b-a) + a, W); 28 | } -------------------------------------------------------------------------------- /src/optimization/lmlsqr.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::optimization::LmLSQR::_initialize(const arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 4 | _F = f(x); 5 | if (jacobian == nullptr) _J = approx_jacobian(f,x); 6 | else _J = (*jacobian)(x); 7 | 8 | _lam = _damping_param * arma::norm(_J.diag(), "inf"); 9 | } 10 | 11 | bool numerics::optimization::LmLSQR::_step(arma::vec& dx, arma::vec& F1, const arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 12 | arma::vec D; arma::mat U,V; 13 | bool success = arma::svd(U, D, V, _J); 14 | if (not success) { 15 | if (jacobian == nullptr) _J = approx_jacobian(f, x); 16 | else _J = (*jacobian)(x); 17 | success = arma::svd(U, D, V, _J); 18 | if (not success) return false; 19 | } 20 | double rho; 21 | double f0 = std::pow(arma::norm(_F),2); 22 | arma::vec JF = -_J.t() * _F; 23 | arma::vec UF = -U.t() * _F; 24 | while (true) { 25 | dx = V * (D/(arma::square(D)+_lam) % UF); 26 | F1 = f(x + dx); 27 | if (F1.has_nan()) return false; 28 | double f1 = std::pow(arma::norm(F1),2); 29 | rho = (f0-f1) / arma::dot(dx, _lam*dx + JF); 30 | if (rho > 0) { 31 | if (jacobian == nullptr) _J = approx_jacobian(f, x+dx); 32 | else _J = (*jacobian)(x+dx); 33 | 34 | _lam *= std::max(0.33, 1 - std::pow(2*rho-1,3)); 35 | 36 | break; 37 | } else { 38 | _lam *= _damping_scale; 39 | } 40 | } 41 | return true; 42 | } -------------------------------------------------------------------------------- /examples/data_science/splines_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o splines splines_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | using namespace numerics; 7 | typedef std::vector ddvec; 8 | 9 | arma::vec f(const arma::vec& X) { 10 | arma::vec y = arma::zeros(arma::size(X)); 11 | for (int i=1; i < 10; ++i) { 12 | y += arma::sin(i*X)/i; 13 | } 14 | return 0.5 - y/M_PI; 15 | } 16 | 17 | int main() { 18 | arma::arma_rng::set_seed_random(); 19 | arma::mat X = 5*arma::randu(100,1) - 2.5; 20 | arma::vec y = f(X) + 0.05*arma::randn(100,1); 21 | 22 | numerics::Splines model; 23 | // model.set_df(15); 24 | // model.set_lambda(c1); 25 | model.fit(X,y); 26 | std::cout << "lambda : " << model.lambda << std::endl 27 | << "df : " << model.eff_df << std::endl; 28 | 29 | int N = 200; 30 | arma::mat xgrid = arma::linspace(-2.5,2.5,N); 31 | arma::vec yHat = model.predict(xgrid); 32 | 33 | ddvec X1 = arma::conv_to::from(X); 34 | ddvec Y1 = arma::conv_to::from(y); 35 | ddvec xx = arma::conv_to::from(xgrid); 36 | ddvec yy = arma::conv_to::from(yHat); 37 | ddvec ff = arma::conv_to::from(f(xgrid)); 38 | 39 | matplotlibcpp::plot(X1, Y1, "ok"); 40 | matplotlibcpp::named_plot("spline fit",xx,yy, "-b"); 41 | matplotlibcpp::named_plot("actual function", xx, ff,"--m"); 42 | matplotlibcpp::legend(); 43 | matplotlibcpp::show(); 44 | 45 | return 0; 46 | } -------------------------------------------------------------------------------- /examples/data_science/knn_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o kNN knn_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector dvec; 7 | 8 | arma::vec f(arma::vec& X) { 9 | arma::vec y = arma::zeros(arma::size(X)); 10 | for (int i=1; i < 10; ++i) { 11 | y += arma::sin(i*X)/i; 12 | } 13 | return 0.5 - y/M_PI; 14 | } 15 | 16 | int main() { 17 | arma::arma_rng::set_seed(123); 18 | arma::vec x = 5*arma::randu(200)-2.5; 19 | arma::vec y = f(x) + 0.1*arma::randn(arma::size(x))%arma::round(arma::randu(arma::size(x))*1); 20 | 21 | bool distance_weights = false; 22 | arma::uvec k_set = arma::regspace(2,20); 23 | 24 | numerics::KNeighborsRegressor model(k_set,2,distance_weights); 25 | model.fit(x,y); 26 | 27 | arma::vec t = arma::linspace(-3,3,500); 28 | arma::vec yhat = model.predict(t); 29 | 30 | std::cout << "optimal k : " << model.k << '\n' 31 | << "model R^2 : " << model.score(x,y) << '\n'; 32 | 33 | dvec xx = arma::conv_to::from(x); 34 | dvec yy = arma::conv_to::from(y); 35 | dvec tt = arma::conv_to::from(t); 36 | dvec uu = arma::conv_to::from(yhat); 37 | dvec vv = arma::conv_to::from(f(t)); 38 | 39 | matplotlibcpp::plot(xx,yy,"or"); 40 | matplotlibcpp::named_plot("kNN fit", tt, uu, "-b"); 41 | matplotlibcpp::named_plot("actual function", tt, vv, "--m"); 42 | matplotlibcpp::legend(); 43 | matplotlibcpp::show(); 44 | 45 | return 0; 46 | } -------------------------------------------------------------------------------- /src/optimization/lbfgs.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::optimization::LBFGS::_lbfgs_update(arma::vec& p) { 4 | long k = _S.size(); 5 | if (k > 0) { 6 | arma::vec ro = arma::zeros(k); 7 | for (long i=0; i < k; ++i) { 8 | ro(i) = 1 / arma::dot(_S.at(i),_Y.at(i)); 9 | } 10 | 11 | arma::vec q = p; 12 | arma::vec alpha = arma::zeros(k); 13 | 14 | for (long i(k-1); i >= 0; --i) { 15 | alpha(i) = ro(i) * arma::dot(_S.at(i),q); 16 | q -= alpha(i) * _Y.at(i); 17 | } 18 | 19 | arma::vec r = q * _hdiag; 20 | 21 | for (long i(0); i < k; ++i) { 22 | double beta = ro(i) * arma::dot(_Y.at(i),r); 23 | r += _S.at(i) * (alpha(i) - beta); 24 | } 25 | 26 | p = r; 27 | } 28 | } 29 | 30 | void numerics::optimization::LBFGS::_initialize(const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 31 | _g = df(x); 32 | _S.clear(); 33 | _Y.clear(); 34 | } 35 | 36 | bool numerics::optimization::LBFGS::_step(arma::vec& dx, arma::vec& g1, const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 37 | arma::vec p = -_g; 38 | _lbfgs_update(p); 39 | 40 | double alpha = wolfe_step(f, df, x, p, _wolfe_c1, _wolfe_c2); 41 | 42 | dx = alpha*p; 43 | 44 | if (dx.has_nan()) return false; 45 | 46 | g1 = df(x + dx); 47 | arma::vec y = g1 - _g; 48 | 49 | _hdiag = arma::dot(dx, y) / arma::dot(y,y); 50 | _S.push(dx); 51 | _Y.push(std::move(y)); 52 | 53 | return true; 54 | } -------------------------------------------------------------------------------- /examples/optimization/genetic_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -Wall -g -o genetic genetic_ex.cpp -O3 -lnumerics -larmadillo 4 | 5 | using namespace numerics; 6 | 7 | int main() { 8 | auto f = [](const arma::vec& x){ return -arma::accu(arma::pow(x,4)-16*arma::pow(x,2)+5*x)/2; }; 9 | arma::vec xMin = {-5,-5,-5,-5}; 10 | arma::vec xMax = {5,5,5,5}; 11 | arma::vec x; 12 | std::cout << "Let's try to maximize f(x) = sum(-x^4 - 16x^2 + 5x)/2 where x is a 4 dimensional vector." << std::endl 13 | << "We will find the global max of the constrained problem [-5 -5 -5 -5] <= x <= [5 5 5 5]." << std::endl; 14 | 15 | numerics::optimization::GeneticOptimizer genOptim; 16 | 17 | clock_t t = clock(); 18 | genOptim.maximize(x, f, xMin, xMax); 19 | t = clock() - t; 20 | std::cout << "global maximum at: " << x.t() << "the value is: " << f(x) << std::endl; 21 | std::cout << "computation time: " << (float)t/CLOCKS_PER_SEC << std::endl << std::endl; 22 | 23 | std::cout << "We will try to solve the unconstrained problem with initial guess x = [0 0 0 0] and search radius of 2.0" << std::endl; 24 | arma::vec x0 = {0,0,0,0}; 25 | genOptim.set_search_radius(2.0); 26 | t = clock(); 27 | genOptim.maximize(x0, f); 28 | t = clock() - t; 29 | std::cout << "local max at: " << x0.t() << "the value is: " << f(x0) << std::endl; 30 | std::cout << "computation time: " << (float)t/CLOCKS_PER_SEC << std::endl << std::endl; 31 | 32 | double u = -2.904; 33 | arma::vec U = {u,u,u,u}; 34 | std::cout << "actual max at: " << U.t() << "actual max is: " << f(U) << std::endl; 35 | 36 | return 0; 37 | } -------------------------------------------------------------------------------- /examples/ode/poisson_ex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -Wall -g -o pois poisson_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector> ddvec; 7 | 8 | arma::vec potential(const arma::vec& x, const arma::vec& y) { 9 | return 20*arma::sinc( 4*arma::pow(x-1, 2) + 4*arma::pow(y-2, 2) ); 10 | } 11 | 12 | arma::mat bc(const arma::mat& x, const arma::mat& y) { 13 | arma::mat B = arma::zeros(arma::size(x)); 14 | B( arma::find(x==-1) ).zeros(); 15 | B( arma::find(y==0) ) = arma::sin(M_PI*x(arma::find(y==0))); 16 | B( arma::find(x==3) ) = arma::sinc(2*y(arma::find(x==3)) - 4); 17 | B( arma::find(y==4) ) = -0.5*arma::exp( -arma::pow(x(arma::find(y==4))-0.75,2)*5 ); 18 | return B; 19 | // return 0*x; 20 | } 21 | 22 | using namespace numerics::ode; 23 | 24 | int main() { 25 | int num_pts = 32; 26 | double root_eig = 5.5; 27 | 28 | arma::mat X = {-1,3}, 29 | Y = {0,2}; 30 | 31 | arma::mat U; 32 | 33 | poisson_helmholtz_2d(X, Y, U, potential, bc, root_eig, num_pts); 34 | 35 | ddvec xx(num_pts), yy(num_pts), zz(num_pts); 36 | for (uint i=0; i < X.n_rows; ++i) { 37 | for (uint j=0; j < Y.n_cols; ++j) { 38 | xx.at(i).push_back(X(i,j)); 39 | yy.at(i).push_back(Y(i,j)); 40 | zz.at(i).push_back(U(i,j)); 41 | } 42 | } 43 | 44 | std::map keys; 45 | keys["cmap"] = "plasma"; 46 | matplotlibcpp::plot_surface(xx,yy,zz,keys); 47 | matplotlibcpp::title("2D Poisson"); 48 | matplotlibcpp::show(); 49 | 50 | return 0; 51 | } -------------------------------------------------------------------------------- /src/interpolation/lagrange_interp.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | /* lagrange_interp(x, y, u, normalize) : lagrange polynomial interpolation of data points 4 | * --- x : x values 5 | * --- y : y values 6 | * --- u : points to evaluate interpolant 7 | * --- normalize : whether to improve conditioning of interpolant by scaling distant according to exp(-x^2) */ 8 | arma::mat numerics::lagrange_interp(const arma::vec& x, const arma::mat& y, const arma::vec& u) { 9 | int nx = x.n_elem; 10 | if (x.n_elem != y.n_rows) { // dimension error 11 | std::cerr << "lagrange_interp() error: interpolation could not be constructed, x and y vectors must be the same length." << std::endl 12 | << "\tx has " << x.n_elem << " elements, y has " << y.n_elem << " elements." << std::endl; 13 | return {NAN}; 14 | } 15 | 16 | for (int i(0); i < nx - 1; ++i) { // repeated x error 17 | for (int j(i+1); j < nx; ++j) { 18 | if (std::abs(x(i) - x(j)) < arma::datum::eps) { 19 | std::cerr << "lagrange_interp() error: one or more x values are repeating." << std::endl; 20 | return {NAN}; 21 | } 22 | } 23 | } 24 | 25 | int nu = u.n_elem; 26 | arma::mat v(nu, y.n_cols, arma::fill::zeros); 27 | double var; 28 | 29 | for (int i(0); i < nx; ++i) { 30 | arma::mat P(nu, y.n_cols, arma::fill::ones); 31 | for (int j(0); j < nx; ++j) { 32 | if (j != i) { 33 | P.each_col() %= (u - x(j))/(x(i) - x(j)); 34 | } 35 | } 36 | P.each_row() %= y.row(i); 37 | v += P; 38 | } 39 | 40 | return v; 41 | } -------------------------------------------------------------------------------- /src/optimization/simplex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | double numerics::optimization::simplex(arma::vec& x, const arma::vec& f, const arma::mat& conRHS, const arma::vec& conLHS) { 4 | u_long numCons = conRHS.n_elem; 5 | arma::mat A = arma::join_cols(conRHS, -f.as_row()); // A = [RHS; f] 6 | A = arma::join_rows(A, arma::eye(A.n_rows,A.n_rows)); // A = [A , I] 7 | arma::vec z = {0}; 8 | arma::vec LHS = arma::join_cols(conLHS,z); // LHS = [LHS; 0] 9 | A = arma::join_rows(A,LHS); // A = [A, LHS] 10 | 11 | u_long numRows = A.n_rows; 12 | u_long numCols = A.n_cols; 13 | 14 | while ( not arma::all(A.row(numRows-1) >= 0) ) { 15 | u_long pivCol = arma::index_min( A.row(numRows-1) ); // min of last row 16 | u_long pivRow = arma::index_min( A(arma::span(0,numRows-2), numCols-1) / A(arma::span(0,numRows-2),pivCol) ); // min of (piv col)/(end col); 17 | A.row(pivRow) /= A(pivRow,pivCol); 18 | for (u_long i(0); i < numRows; ++i) { 19 | if (i != pivRow) { 20 | A.row(i) -= A.row(pivRow) * A(i,pivCol); // row reduce set all non-pivot positions in pivot col to zero 21 | } 22 | } 23 | } 24 | arma::urowvec nonbas = (A(numRows-1, arma::span(0,numCols-2)) == 0); // non-basic variables 25 | arma::mat B = A(arma::span(0,numRows-2), arma::span(0,numCols-2)); // constraint matrix with out LHS 26 | for (u_long i(0); i <= numRows-2; ++i) { 27 | B.col(i) *= nonbas(i); 28 | } 29 | x = solve(B, A(arma::span(0,numRows-2), numCols-1)); // solve B\LHS 30 | x = x(arma::span(0, numCols-numRows-2)); //return where min occurs 31 | return arma::dot(f,x); 32 | } -------------------------------------------------------------------------------- /examples/data_science/k_fold_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o k_fold k_fold_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector dvec; 7 | 8 | int main() { 9 | arma::vec x = 4*arma::randu(100)-2; 10 | arma::vec y = 2*x + arma::randn(100); 11 | 12 | int num_folds = 4; 13 | 14 | numerics::KFolds2Arr split(num_folds); 15 | split.fit(x,y); 16 | 17 | std::map keys_train; 18 | keys_train["marker"] = "o"; 19 | keys_train["mfc"] = "none"; 20 | keys_train["markersize"] = "4"; 21 | keys_train["ls"] = "none"; 22 | keys_train["label"] = "train data"; 23 | std::map keys_test; 24 | keys_test["marker"] = "o"; 25 | keys_test["markersize"] = "7"; 26 | keys_test["label"] = "test data"; 27 | keys_test["ls"] = "none"; 28 | 29 | for (int i=0; i < num_folds; ++i) { 30 | dvec xtrain = arma::conv_to::from( split.trainX(i) ); 31 | dvec ytrain = arma::conv_to::from( split.trainY(i) ); 32 | dvec xtest = arma::conv_to::from( split.testX(i) ); 33 | dvec ytest = arma::conv_to::from( split.testY(i) ); 34 | matplotlibcpp::subplot(2,2,i+1); 35 | matplotlibcpp::plot(xtrain, ytrain, keys_train); 36 | matplotlibcpp::plot(xtest, ytest, keys_test); 37 | matplotlibcpp::title("fold #"+std::to_string(i+1)); 38 | if (i==0) matplotlibcpp::legend(); 39 | } 40 | matplotlibcpp::tight_layout(); 41 | matplotlibcpp::show(); 42 | 43 | return 0; 44 | } -------------------------------------------------------------------------------- /examples/optimization/gmres_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -g -Wall -o gmres gmres_ex.cpp -O3 -lnumerics -larmadillo -lsuperlu 4 | 5 | 6 | int main() { 7 | int n = 1000; 8 | arma::arma_rng::set_seed_random(); 9 | arma::sp_mat A = arma::speye(n,n); 10 | A.diag(-1) = arma::randu(n-1)*0.5-0.25; 11 | A.diag(1) = arma::randu(n-1)*0.5-0.25; 12 | arma::sp_mat B = arma::sprandu(n,n,0.05)*0.2; 13 | B.for_each([](arma::sp_mat::elem_type& elem)->void{elem -= 0.1;}); 14 | A += B; 15 | // almost tridiagonal matrix. 16 | 17 | std::cout << "nonzeros / n = " << (double)A.n_nonzero / A.n_elem << std::endl; 18 | std::cout << "rcond(A) = " << arma::rcond(arma::mat(A)) << std::endl; 19 | 20 | arma::vec b = arma::randn(n); // we can solve multiple equations at once 21 | 22 | auto tic = std::chrono::high_resolution_clock::now(); 23 | arma::vec x = arma::spsolve(A,b); 24 | auto toc = std::chrono::high_resolution_clock::now(); 25 | double dur = std::chrono::duration_cast(toc-tic).count()/1000.0; 26 | 27 | std::cout << "For the matrix of order " << n << ", the direct solver took " << dur << " seconds" << std::endl << std::endl; 28 | 29 | arma::vec y; 30 | tic = std::chrono::high_resolution_clock::now(); 31 | numerics::optimization::gmres(y, A, b); 32 | toc = std::chrono::high_resolution_clock::now(); 33 | dur = std::chrono::duration_cast(toc-tic).count()/1000.0; 34 | 35 | std::cout << "gmres method took " << dur << " seconds" << std::endl 36 | << "the relative error was: " << arma::norm(A*y - b) / arma::norm(b) << std::endl << std::endl; 37 | 38 | return 0; 39 | } -------------------------------------------------------------------------------- /src/optimization/quasi_newton.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::optimization::QausiNewton::_initialize(const arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 4 | _F = f(x); 5 | if (jacobian == nullptr) _J = numerics::approx_jacobian(f,x); 6 | else _J = (*jacobian)(x); 7 | } 8 | 9 | void numerics::optimization::QausiNewton::_solve(arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 10 | numerics::optimization::VerboseTracker T(_max_iter); 11 | if (_v) T.header("max|f|"); 12 | 13 | _initialize(x, f, jacobian); 14 | arma::vec dx, F1; 15 | 16 | _n_iter = 0; 17 | while (true) { 18 | bool successful_step = _step(dx, F1, x, f, jacobian); 19 | 20 | if (not successful_step) { 21 | _exit_flag = 3; 22 | if (_v) T.nan_flag(); 23 | return; 24 | } 25 | double xtol = _xtol*std::max(1.0, arma::norm(x,"inf")); 26 | double df = arma::norm(F1 - _F, "inf"); 27 | double ftol = _ftol*std::max(1.0, arma::norm(_F,"inf")); 28 | 29 | x += dx; 30 | _n_iter++; 31 | 32 | _F = std::move(F1); 33 | 34 | if (_v) T.iter(_n_iter, arma::norm(_F,"inf")); 35 | 36 | if (df < ftol) { 37 | _exit_flag = 0; 38 | if (_v) T.success_flag(); 39 | return; 40 | } 41 | 42 | if (arma::norm(dx,"inf") < xtol) { 43 | _exit_flag = 1; 44 | if (_v) T.success_flag(); 45 | return; 46 | } 47 | 48 | if (_n_iter >= _max_iter) { 49 | _exit_flag = 2; 50 | if (_v) T.max_iter_flag(); 51 | return; 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /src/ode/rk45.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | void numerics::ode::rk45::_next_step_size(double& k, double res, double tol) { 3 | k *= std::min(10.0, std::max(0.1, 0.9*std::pow(tol/res,0.2))); 4 | if (k < _min_step) throw std::runtime_error("rk45 could not converge b/c current step-size (=" + std::to_string(k) + ") < minimum step size (=" + std::to_string(_min_step) + ")."); 5 | } 6 | 7 | double numerics::ode::rk45::_step(double k, double& t1, arma::vec& u1, arma::vec& f1, const odefunc& f, const odejacobian* jacobian) { 8 | arma::vec V1, V2, V3, V4, V5, V6, rk4; 9 | double res = 0; 10 | 11 | const arma::vec& u = _prev_u.back(); 12 | const double& t = _prev_t.back(); 13 | 14 | double tol = std::max(_atol, _rtol*arma::norm(u,"inf")); 15 | 16 | do { 17 | V1 = k * _prev_f.back(); 18 | V2 = k * f( t + 0.2*k, u + 0.2*V1 ); 19 | V3 = k * f( t + 0.3*k, u + (3.0/40)*V1 + (9.0/40)*V2 ); 20 | V4 = k * f( t + 0.8*k, u + (44.0/45)*V1 - (56.0/15)*V2 + (32.0/9)*V3 ); 21 | V5 = k * f( t + (8.0/9)*k, u + (19372.0/6561)*V1 - (25360.0/2187)*V2 + (64448.0/6561)*V3 - (212.0/729)*V4 ); 22 | V6 = k * f( t + k, u + (9017.0/3168)*V1 - (355.0/33)*V2 + (46732.0/5247)*V3 + (49.0/176)*V4 - (5103.0/18656)*V5 ); 23 | rk4 = u + (35.0/384)*V1 + (500.0/1113)*V3 + (125.0/192)*V4 - (2187.0/6784)*V5 + (11.0/84)*V6; 24 | u1 = u + (5179.0/57600)*V1 + (7571.0/16695)*V3 + (393.0/640)*V4 - (92097.0/339200)*V5 + (187.0/2100)*V6 + (1.0/40)*k*f(t + k, rk4); 25 | res = arma::norm(u1 - rk4, "inf"); 26 | t1 = t + k; 27 | _next_step_size(k, res, tol); 28 | } while (res > tol); 29 | 30 | f1 = f(t1, u1); 31 | return k; 32 | } 33 | -------------------------------------------------------------------------------- /examples/neural_network/nn_regression_ex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | // g++ -g -Wall -O3 -o nn_regression nn_regression_ex.cpp -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | arma::mat f(arma::mat& X) { 7 | arma::mat y = arma::zeros(arma::size(X)); 8 | for (int i=1; i < 5; ++i) { 9 | y += arma::sin(i*X)/i; 10 | } 11 | y = 0.5 - y/M_PI; 12 | return y; 13 | } 14 | 15 | using namespace numerics; 16 | typedef std::vector dvec; 17 | 18 | int main() { 19 | int N = 1000; 20 | arma::mat x = 6*arma::randu(N,1) - 3; 21 | arma::vec y = f(x) + 0.2*arma::randn(arma::size(x)); 22 | 23 | std::string loss = "mse"; 24 | std::vector> layers = {{100,"relu"}}; 25 | long max_iter = 200; 26 | double tol = 1e-4; 27 | double l2 = 1e-4; 28 | double l1 = 0; 29 | std::string optimizer = "adam"; 30 | bool verbose = true; 31 | 32 | NeuralNetRegressor model(layers, loss, max_iter, tol, l2, l1, optimizer, verbose); 33 | model.fit(x,y); 34 | 35 | std::cout << "R^2 : " << std::fixed << std::setprecision(2) << model.score(x,y) << "\n"; 36 | 37 | arma::vec t = arma::linspace(-3,3); 38 | arma::vec yh = model.predict(t); 39 | 40 | dvec xx = arma::conv_to::from(x); 41 | dvec yy = arma::conv_to::from(y); 42 | dvec tt = arma::conv_to::from(arma::vec(t)); 43 | dvec yyh = arma::conv_to::from(arma::vec(yh.col(0))); 44 | 45 | std::map ls = {{"label","data"},{"ls","none"},{"marker","o"}}; 46 | matplotlibcpp::plot(xx, yy, ls); 47 | ls["label"] = "fit"; ls["marker"] = ""; ls["ls"] = "-"; 48 | matplotlibcpp::plot(tt, yyh, ls); 49 | matplotlibcpp::show(); 50 | 51 | return 0; 52 | } -------------------------------------------------------------------------------- /src/optimization/bfgs.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::optimization::BFGS::_initialize(const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 4 | _g = df(x); 5 | if (hessian == nullptr) { 6 | if (_use_fd) { 7 | _H = arma::symmatu(numerics::approx_jacobian(df,x)); 8 | bool chol = arma::inv_sympd(_H, _H); 9 | if (!chol) _H = arma::pinv(_H); 10 | } else _H = arma::eye(x.n_elem, x.n_elem); 11 | } else { 12 | _H = (*hessian)(x); 13 | bool chol = arma::inv_sympd(_H, _H); 14 | if (!chol) _H = arma::pinv(_H); 15 | } 16 | } 17 | 18 | bool numerics::optimization::BFGS::_step(arma::vec& dx, arma::vec& g1, const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 19 | dx = -(_H*_g); 20 | 21 | if (dx.has_nan()) { 22 | if (hessian == nullptr) { 23 | if (_use_fd) { 24 | _H = arma::symmatu(numerics::approx_jacobian(df,x)); 25 | bool chol = arma::inv_sympd(_H, _H); 26 | if (!chol) _H = arma::pinv(_H); 27 | } else _H = arma::eye(x.n_elem, x.n_elem); 28 | } else { 29 | _H = (*hessian)(x); 30 | bool chol = arma::inv_sympd(_H, _H); 31 | if (!chol) _H = arma::pinv(_H); 32 | } 33 | dx = -(_H*_g); 34 | if (dx.has_nan()) return false; 35 | } 36 | 37 | double alpha = wolfe_step(f, df, x, dx, _wolfe_c1, _wolfe_c2); 38 | 39 | dx *= alpha; 40 | g1 = df(x+dx); 41 | 42 | arma::vec y = g1 - _g; 43 | 44 | double sy = arma::dot(dx,y); 45 | arma::vec Hy = _H*y; 46 | double yHy = arma::dot(y,Hy); 47 | arma::mat sHy = dx*Hy.t() / sy; 48 | 49 | _H += (sy + yHy)/std::pow(sy,2) * dx*dx.t() - sHy - sHy.t(); 50 | 51 | return true; 52 | } 53 | -------------------------------------------------------------------------------- /examples/data_science/lasso_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | 5 | // g++ -g -Wall -o lasso lasso_ex.cpp -larmadillo -lnumerics -I/usr/include/python3.8 -lpython3.8 6 | 7 | int main() { 8 | int n=200; 9 | arma::arma_rng::set_seed(123); 10 | arma::vec x = 2*arma::randu(n)-1; 11 | arma::vec y = 1 - 0.1*x + x%x + 0.3*arma::randn(n); // random data is quadratic in x... 12 | 13 | numerics::PolyFeatures poly(20); 14 | arma::mat P = poly.fit_predict(x); 15 | 16 | numerics::LassoCV lcv; 17 | lcv.fit(P,y); 18 | arma::vec w = lcv.linear_coefs; 19 | std::cout << "optimal lambda: " << lcv.lambda << "\n"; 20 | 21 | // print + plot results 22 | std::stringstream model; 23 | model << "model : y = " << std::fixed << std::setprecision(3) << lcv.intercept; 24 | for (uint i=0; i < w.n_elem; ++i) if (w(i) != 0) model << " + " << w(i) << "*x^" << i+1; 25 | std::string model_str = model.str(); 26 | std::cout << model_str + "\n"; 27 | 28 | arma::vec t = arma::linspace(x.min(), x.max()); 29 | arma::mat Pt = poly.predict(t); 30 | arma::vec yh = lcv.predict(Pt); 31 | 32 | std::vector xx = arma::conv_to>::from(x); 33 | std::vector yy = arma::conv_to>::from(y); 34 | std::vector tt = arma::conv_to>::from(t); 35 | std::vector pp = arma::conv_to>::from(yh); 36 | std::vector ptrue = arma::conv_to>::from(1 - 0.1*t+t%t); 37 | 38 | matplotlibcpp::named_plot("data", xx, yy, ".b"); 39 | matplotlibcpp::named_plot("true function", tt, ptrue, "--k"); 40 | matplotlibcpp::named_plot("predicted function", tt, pp, "-r"); 41 | matplotlibcpp::legend(); 42 | matplotlibcpp::title(model_str); 43 | matplotlibcpp::show(); 44 | 45 | return 0; 46 | } -------------------------------------------------------------------------------- /examples/data_science/kde_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o kde kde_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector dvec; 7 | 8 | int main() { 9 | arma::arma_rng::set_seed(123); 10 | 11 | int sample_size = 300; 12 | 13 | arma::vec sample(sample_size); 14 | sample.rows(0,sample_size/2-1) = arma::randn(sample_size/2); 15 | sample.rows(sample_size/2,sample_size-1) = 2*arma::randn(sample_size/2) + 10; 16 | 17 | std::string K = "gaussian"; // gaussian, square, triangle, parabolic 18 | std::string estimator = "grid_cv"; // rule_of_thumb, min_sd_iqr, plug_in, grid_cv 19 | bool enable_binning = true; 20 | numerics::KDE pdf_hat(K, estimator, enable_binning); 21 | // kde pdf_hat(0.8, K, enable_binning); 22 | pdf_hat.fit(sample); 23 | 24 | arma::vec x = arma::linspace(-5,20,1000); 25 | arma::vec p_hat = pdf_hat.predict(x); 26 | 27 | std::cout << "selected bandwidth: " << pdf_hat.bandwidth << std::endl; 28 | 29 | int N = 1000; 30 | arma::vec sample_new = pdf_hat.sample(N); // we can resample using our KDE 31 | 32 | int n_bins = 30; 33 | 34 | auto xx = arma::conv_to::from(x); 35 | auto hist = arma::conv_to::from(sample); 36 | auto bootstrap = arma::conv_to::from(sample_new); 37 | auto p_hat0 = arma::conv_to::from(sample_size*p_hat); 38 | 39 | matplotlibcpp::named_hist("sample", hist, n_bins); 40 | matplotlibcpp::named_plot("kde",xx, p_hat0, "-k"); 41 | matplotlibcpp::legend(); 42 | matplotlibcpp::show(); 43 | 44 | p_hat0 = arma::conv_to::from(N*p_hat); 45 | matplotlibcpp::named_hist("bootstrap sample", bootstrap, n_bins); 46 | matplotlibcpp::named_plot("kde",xx, p_hat0, "-k"); 47 | matplotlibcpp::legend(); 48 | matplotlibcpp::show(); 49 | 50 | return 0; 51 | } -------------------------------------------------------------------------------- /src/data_science/polyfeat.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::PolyFeatures::fit(const arma::mat& x) { 4 | _dim = x.n_cols; 5 | _scale.set_size(_dim); 6 | for (u_long i=0; i < _dim; ++i) { 7 | _scale(i) = arma::norm(x.col(i),"inf"); 8 | } 9 | std::queue> Q; 10 | std::set> S; 11 | for (u_int i=0; i < x.n_cols; ++i) { 12 | std::vector str = {i}; 13 | Q.push(str); 14 | } 15 | while ( !Q.empty() ) { 16 | std::vector str = Q.front(); 17 | if (str.size() > _deg) { // invalid monomial 18 | Q.pop(); 19 | continue; 20 | } 21 | if (S.count(str) > 0) { // discovered 22 | Q.pop(); 23 | continue; 24 | } else { // new node 25 | S.insert(str); 26 | for (uint i=0; i < x.n_cols; ++i) { 27 | std::vector str2 = str; 28 | str2.push_back(i); 29 | std::sort( str2.begin(), str2.end() ); 30 | Q.push(str2); 31 | } 32 | Q.pop(); 33 | } 34 | } 35 | _monomials.clear(); 36 | for (const std::vector& str : S) _monomials.push_back(str); 37 | } 38 | 39 | arma::mat numerics::PolyFeatures::predict(const arma::mat& x) const { 40 | _check_x(x); 41 | u_int n = x.n_rows; 42 | u_int num_mons = _monomials.size(); 43 | 44 | arma::mat P; 45 | if (_intercept) P = arma::ones(n, num_mons+1); 46 | else P = arma::ones(n, num_mons); 47 | 48 | u_int start = 0; 49 | if (_intercept) start = 1; 50 | for (u_int i=0; i < num_mons; ++i) { 51 | for (u_int r : _monomials.at(i) ) { 52 | P.col(i+start) %= x.col(r) / _scale(r); 53 | } 54 | } 55 | return P; 56 | } 57 | 58 | arma::mat numerics::PolyFeatures::fit_predict(const arma::mat& x) { 59 | fit(x); 60 | return predict(x); 61 | }; -------------------------------------------------------------------------------- /examples/data_science/ridge_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o ridge ridge_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | using namespace numerics; 7 | namespace plt = matplotlibcpp; 8 | 9 | typedef std::vector dvec; 10 | 11 | arma::mat f(arma::vec& X) { 12 | arma::vec y = arma::zeros(arma::size(X)); 13 | for (int i=1; i < 10; ++i) { 14 | y += arma::sin(i*X)/i; 15 | } 16 | return y/M_PI; 17 | } 18 | 19 | int main() { 20 | // arma::arma_rng::set_seed(123); 21 | int n_obs = 100; 22 | arma::vec X = 5*arma::randu(n_obs) - 2.5; 23 | arma::vec y = f(X) + 0.15*arma::randn(n_obs); 24 | 25 | uint n_poly = 50; 26 | arma::vec centers = arma::linspace(-2.5,2.5,n_poly); 27 | arma::mat features = numerics::cubic_kernel(centers, X); 28 | 29 | arma::mat c_overfit = arma::solve(features, y); 30 | arma::vec xgrid = arma::linspace(-2.5, 2.5, 500); 31 | arma::mat features_grid = numerics::cubic_kernel(centers, xgrid); 32 | arma::vec yhat_overfit = features_grid * c_overfit; 33 | 34 | numerics::RidgeCV model; 35 | model.fit(features, y); 36 | arma::vec yhat = model.predict(features_grid); 37 | 38 | std::cout << "lambda: " << model.lambda << std::endl 39 | << "R^2: " << model.score(features,y) << std::endl 40 | << "effective degrees of freedom: " << model.eff_df << std::endl; 41 | 42 | dvec xx = arma::conv_to::from(X); 43 | dvec yy = arma::conv_to::from(y); 44 | dvec xxg = arma::conv_to::from(xgrid); 45 | dvec yoverfit = arma::conv_to::from(yhat_overfit); 46 | dvec yyhat = arma::conv_to::from(yhat); 47 | 48 | plt::plot(xx,yy,"o"); 49 | plt::named_plot("overfit", xxg, yoverfit); 50 | plt::named_plot("regularized fit", xxg, yyhat); 51 | plt::ylim(-1.2, 1.2); 52 | plt::legend(); 53 | plt::show(); 54 | 55 | return 0; 56 | } -------------------------------------------------------------------------------- /references.txt: -------------------------------------------------------------------------------- 1 | Author: Amit Rotem 2 | Last modified: 10/25/2019 3 | 4 | I used the following sources: 5 | Armadillo: 6 | [1] Conrad Sanderson and Ryan Curtin. 7 | Armadillo: a template-based C++ library for linear algebra. 8 | Journal of Open Source Software, Vol. 1, pp. 26, 2016. 9 | 10 | [2] Conrad Sanderson and Ryan Curtin. 11 | A User-Friendly Hybrid Sparse Matrix Class in C++. 12 | Lecture Notes in Computer Science (LNCS), Vol. 10931, pp. 422-430, 2018. 13 | 14 | matplotlibcpp: 15 | [3] https://github.com/lava/matplotlib-cpp 16 | 17 | algorithms: 18 | [4] Many of the methods here were adapted from algorithms taught at various classes I took (or am currently taking) at the Colorado School of Mines within the Applied Mathematics and Statistics department. 19 | 20 | [5] LeVeque, Randall J. "Finite difference methods for ordinary and partial differential equations". SIAM, 2007. 21 | 22 | [6] Lloyd N. Trefethen "Spectral Methods in MATLAB". SIAM, 2000. 23 | 24 | [7] Manolis I. A.Lourakis "A Brief Description of the Levenberg-Marquardt Algorithm Implemened by levmar", 2005. 25 | 26 | [8] http://www.scholarpedia.org/article/Nelder-Mead_algorithm accessed Oct 2019 27 | 28 | [9] Fornberg, Bengt "Calculation of weights in finite difference formulas". SIAM, 1998 29 | 30 | [10] E. Hairer, G. Wanner, "Solving Ordinary Differential Equations II". Springer-Verlag, 1993. 31 | 32 | [11] Gramacki A. "Nonparametric Kernel Density Estimation and Its Computational Aspects". Springer International Publishing, 2018. 33 | 34 | [12] Brent, R. P. "Algorithms for Minimization Without Derivatives". Prentice-Hall, 1972. 35 | 36 | [13] Phillips, Steven J. "Acceleration of K-Means and Related Clustering Algorithms". Springer, 2002. 37 | 38 | [14] D. Sculley. "Web-scale k-means clustering". ACM, 2010. 39 | 40 | [15] J. Nocedal, S. Wright "Numerical Optimization". Springer, 1999. 41 | 42 | [16] Y. Saad "Iterative Methods for Sparse Linear Systems". SIAM, 2003. 43 | 44 | [*] a lot of Wikipedia... 45 | -------------------------------------------------------------------------------- /src/ode/rk5i.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | arma::vec numerics::ode::rk5i::_solve_v2(double k, double t, const arma::vec& z, const odefunc& f, const odejacobian* jacobian) { 4 | arma::vec v2 = z; 5 | auto v2f = [t,k,&z,&f](const arma::vec& v) -> arma::vec { 6 | return v - f(t+0.25*k, z + 0.125*k*v); 7 | }; 8 | if (jacobian == nullptr) solver->fsolve(v2, v2f); 9 | else { 10 | auto v2J = [t,k,&z,&jacobian](const arma::vec& v) -> arma::mat { 11 | arma::mat J = (*jacobian)(t+0.25*k, z + 0.125*k*v); 12 | J = arma::eye(arma::size(J)) - 0.125*k*J; 13 | return J; 14 | }; 15 | solver->fsolve(v2, v2f, v2J); 16 | } 17 | return v2; 18 | } 19 | 20 | arma::vec numerics::ode::rk5i::_solve_v3(double k, double t, const arma::vec& z, const odefunc& f, const odejacobian* jacobian) { 21 | arma::vec v3 = z; 22 | auto v3f = [t,k,&f,&z](const arma::vec& v) -> arma::vec { 23 | return v - f(t + 0.7*k, z + 0.15*k*v); 24 | }; 25 | if (jacobian == nullptr) solver->fsolve(v3, v3f); 26 | else { 27 | auto v3J = [t,k,&jacobian,&z](const arma::vec& v) -> arma::mat { 28 | arma::mat J = (*jacobian)(t + 0.7*k, z + 0.15*k*v); 29 | J = arma::eye(arma::size(J)) - 0.15*k*J; 30 | return J; 31 | }; 32 | solver->fsolve(v3, v3f, v3J); 33 | } 34 | return v3; 35 | } 36 | 37 | double numerics::ode::rk5i::_step(double k, double& t1, arma::vec& u1, arma::vec& f1, const odefunc& f, const odejacobian* jacobian) { 38 | arma::vec v2, v3, v4, z; 39 | 40 | const arma::vec& v1 = _prev_f.back(); 41 | const arma::vec& u = _prev_u.back(); 42 | const double& t = _prev_t.back(); 43 | 44 | z = u + 0.125*k*v1; 45 | v2 = _solve_v2(k,t,z,f,jacobian); 46 | 47 | z = u + k*(-0.01*v1 + 0.56*v2); 48 | v3 = _solve_v3(k,t,z,f,jacobian); 49 | 50 | v4 = f(t+k, u + k*((2.0/7)*v1 + (5.0/7)*v3)); 51 | z = u + k * ( (1.0/14)*v1 + (32.0/81)*v2 + (250.0/567)*v3 + (5.0/54)*v4 ); 52 | 53 | t1 = t + k; 54 | u1 = std::move(z); 55 | f1 = f(t1,u1); 56 | return _cstep; 57 | } -------------------------------------------------------------------------------- /src/optimization/newton.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::optimization::Newton::_initialize(const arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 4 | _F = f(x); 5 | } 6 | 7 | bool numerics::optimization::Newton::_step(arma::vec& dx, arma::vec& F1, const arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 8 | bool success; 9 | if (jacobian == nullptr) { 10 | double h = 1e-6*std::max(1.0, arma::norm(x)); 11 | VecFunc JacMult = [this, h, &F1, &f, &x](const arma::vec& v) -> arma::vec { 12 | double C = std::max(1.0, arma::norm(v)) / h; 13 | return C * (f(x + v/C) - _F); 14 | }; 15 | success = numerics::optimization::gmres(dx, JacMult, -_F, _xtol); 16 | } else { 17 | _J = (*jacobian)(x); 18 | if (_J.has_nan()) return false; 19 | success = arma::solve(dx, _J, -_F); 20 | } 21 | if (not success) return false; 22 | auto line_f = [&F1,&x,&f,&dx](double a) -> double { 23 | F1 = f(x + a*dx); 24 | return arma::norm(F1); 25 | }; 26 | if (line_f(1.0) > 0.99*arma::norm(_F)) { 27 | double a = numerics::optimization::fminbnd(line_f, 0.0, 1.0, 1e-2); 28 | dx *= a; 29 | } 30 | return true; 31 | } 32 | 33 | bool numerics::optimization::NewtonMin::_step(arma::vec& dx, arma::vec& g1, const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 34 | bool success; 35 | if (hessian == nullptr) { 36 | double h = 1e-6*std::max(1.0, arma::norm(x)); 37 | VecFunc HessMult = [this,&df,&x,h](const arma::vec& v) -> arma::vec { 38 | double C = std::max(1.0, arma::norm(v)) / h; 39 | return C * (df(x + v/C) - _g); 40 | }; 41 | success = pcg(dx, HessMult, -_g, _xtol); 42 | if (not success) success = gmres(dx, HessMult, -_g, _xtol); 43 | } else { 44 | arma::mat H = (*hessian)(x); 45 | success = arma::solve(dx, H, -_g); 46 | } 47 | 48 | if (not success) return false; 49 | 50 | double alpha = wolfe_step(f, df, x, dx); 51 | 52 | dx *= alpha; 53 | g1 = df(x+dx); 54 | 55 | return true; 56 | } -------------------------------------------------------------------------------- /examples/data_science/knn_classifier_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o knn_classifier knn_classifier_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | std::vector conv(const arma::mat& u) { 7 | return arma::conv_to>::from(u); 8 | } 9 | std::vector conv(const arma::umat& u) { 10 | return arma::conv_to>::from(u); 11 | } 12 | 13 | int main() { 14 | int sample_size = 100; 15 | arma::mat X = arma::zeros(sample_size,1); 16 | X.rows(0,sample_size/2-1) = arma::randn(sample_size/2)-1.5; 17 | X.rows(sample_size/2,sample_size-1) = arma::randn(sample_size/2)+1.5; 18 | arma::uvec Y = arma::zeros(sample_size); 19 | Y.rows(sample_size/2,sample_size-1) += 1; 20 | 21 | bool distance_weights = false; 22 | arma::uvec k_set = arma::regspace(2,20); 23 | numerics::KNeighborsClassifier model(k_set, 2, distance_weights); 24 | model.fit(X,Y); 25 | 26 | arma::vec xgrid = arma::linspace(-M_PI,M_PI,1000); 27 | arma::mat p = model.predict_proba(xgrid); 28 | arma::uvec pred_categories = model.predict(X); 29 | 30 | std::cout << "optimal k: " << model.k << '\n' 31 | << "accuracy : " << model.score(X, Y) << '\n'; 32 | 33 | auto xx = conv(X); 34 | auto y1 = conv(Y); 35 | auto xxgrid = conv(xgrid); 36 | auto p1 = conv(p.col(1)); 37 | auto c1 = conv(pred_categories); 38 | 39 | std::map keys; 40 | keys["marker"] = "o"; 41 | keys["ls"] = "none"; 42 | keys["label"] = "actual categories"; 43 | matplotlibcpp::plot(xx,y1,keys); 44 | keys["marker"] = "none"; 45 | keys["ls"] = "-"; 46 | keys["label"] = "predicted probabilities"; 47 | matplotlibcpp::plot(xxgrid,p1,"-"); 48 | keys["marker"] = "o"; 49 | keys["mfc"] = "none"; 50 | keys["ls"] = "none"; 51 | keys["markersize"] = "10"; 52 | keys["label"] = "predicted categories"; 53 | matplotlibcpp::plot(xx,c1,keys); 54 | matplotlibcpp::legend(); 55 | matplotlibcpp::show(); 56 | 57 | return 0; 58 | } -------------------------------------------------------------------------------- /examples/data_science/kmeans_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -Wall -g -o kmeans kmeans_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector dvec; 7 | 8 | arma::mat gen_balls(int n_balls, int n_obs) { 9 | arma::arma_rng::set_seed_random(); 10 | arma::mat m = 5*arma::randn(n_balls,2); 11 | arma::mat data = arma::randn(n_obs,2); 12 | for (int i=0; i < n_balls; ++i) data.rows(i*n_obs/n_balls, (i+1)*n_obs/n_balls-1).each_row() += m.row(i); 13 | return data; 14 | } 15 | 16 | void plot_classes(int k, const arma::mat& data, const arma::uvec& labels, const arma::mat centroids) { 17 | arma::vec X = data.col(0), Y = data.col(1); 18 | for (int i=0; i < k; ++i) { 19 | arma::uvec cluster_i = arma::find(labels == i); 20 | dvec x = arma::conv_to::from(X(cluster_i)); 21 | dvec y = arma::conv_to::from(Y(cluster_i)); 22 | matplotlibcpp::named_plot("cluster " + std::to_string(i), x, y, "o"); 23 | std::vector cx = {centroids(i,0)}, cy = {centroids(i,1)}; 24 | matplotlibcpp::plot(cx, cy, "*k"); 25 | } 26 | matplotlibcpp::legend(); 27 | matplotlibcpp::show(); 28 | } 29 | 30 | int main() { 31 | int k = 4; 32 | arma::mat data = gen_balls(k, 1000); 33 | 34 | arma::uvec labels; 35 | arma::mat centroids; 36 | 37 | bool use_sgd = false; 38 | if (use_sgd) { 39 | int batch_size = 50, max_iter = 10; 40 | numerics::KMeansSGD kmu(k,2,batch_size,0.01,max_iter); 41 | labels = kmu.fit_predict(data); 42 | // centroids = kmu.clusters; 43 | centroids = kmu.points_nearest_centers; // alternative to using the means, this returns actual data points which may be useful when as a data reduction technique 44 | } else { 45 | double tol = 1e-5; 46 | numerics::KMeans kmu(k,2,tol); 47 | labels = kmu.fit_predict(data); 48 | // centroids = kmu.clusters; 49 | centroids = kmu.points_nearest_centers; // alternative to using the means, this returns actual data points which may be useful when as a data reduction technique 50 | } 51 | 52 | plot_classes(k, data, labels, centroids); 53 | 54 | return 0; 55 | } -------------------------------------------------------------------------------- /src/ode/poisson2d.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /* poisson_helmholtz_2d(x, y, u, f, bc, k, m) : Solves the 2D Poisson/Helmholtz Equation: u_xx + u_yy + k^2 u = f(x,y) with dirichlet boundary conditions. 4 | * --- f : f(x,y). 5 | * --- bc : a function g(x,y) such that if x,y are on the boundary u(x,y) = g(x,y). 6 | * --- k : square root of Helmholtz eigenvalue. 7 | * --- m : number of points along each axis. */ 8 | void numerics::ode::poisson_helmholtz_2d(arma::mat& X, arma::mat& Y, arma::mat& U, 9 | const std::function& f, 10 | const std::function& bc, 11 | double k, 12 | int m 13 | ) { 14 | //--- (1) set up Chebyshev differentiation matrix 15 | arma::mat D; 16 | arma::vec x; 17 | numerics::ode::cheb(D,x,m); 18 | D *= D; // D^2 19 | 20 | if (X.n_elem < 2) { 21 | std::cerr << "poisson2d() error: bounds on X not defined.\n"; 22 | return; 23 | } 24 | if (Y.n_elem < 2) { 25 | std::cerr << "poisson2d() error: bounds on Y not defined.\n"; 26 | return; 27 | } 28 | double lower_x = X(0), upper_x = X(1), lower_y = Y(0), upper_y = Y(1); 29 | 30 | //--- (2) set up x,y mesh points 31 | numerics::meshgrid(X, x); 32 | Y = X.t(); 33 | X = arma::vectorise(X); 34 | Y = arma::vectorise(Y); 35 | 36 | //--- (3) set up Laplacian matrix operator 37 | arma::mat I = arma::eye(m, m); 38 | arma::mat L = arma::kron(I,D) + arma::kron(D,I) + k*k*arma::eye(m*m,m*m); 39 | 40 | //--- (4) apply boundary conditions 41 | arma::uvec b = arma::find( arma::abs(X) == 1 || arma::abs(Y) == 1 ); 42 | // L.rows(b) = arma::zeros( 4*m, std::pow(m+1,2) ); 43 | L.rows(b).zeros(); 44 | L(b,b) = arma::eye(4*m-4, 4*m-4); 45 | 46 | X = (upper_x - lower_x)*X/2 + (upper_x + lower_x)/2; 47 | Y = (upper_y - lower_y)*Y/2 + (upper_y + lower_y)/2; 48 | arma::vec F = f( X, Y ); 49 | F(b) = bc( X(b), Y(b) ); 50 | 51 | //--- (5) solve the system, reshape, and return 52 | U = arma::solve(L,F); 53 | U = arma::reshape(U, m, m); 54 | 55 | X = arma::reshape(X, m, m); 56 | Y = arma::reshape(Y, m, m); 57 | } -------------------------------------------------------------------------------- /src/ode/rk34i.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::ode::rk34i::_next_step_size(double& k, double res, double tol) { 4 | k *= std::min(10.0, std::max(0.1, 0.9*std::pow(tol/res,0.25))); 5 | if (k < _min_step) throw std::runtime_error("method could not converge b/c current step-size (=" + std::to_string(k) + ") < minimum step size (=" + std::to_string(_min_step) + ")."); 6 | } 7 | 8 | arma::vec numerics::ode::rk34i::_v_solve(double tv, double k, const arma::vec& z, arma::vec& f1, const odefunc& f, const odejacobian* jacobian) { 9 | arma::vec vv = z; 10 | auto vf = [tv,k,&f,&z,&f1](const arma::vec& v) -> arma::vec { 11 | f1 = f(tv, z + v/4); // stores this value in f1 for reuse at next time-step 12 | return v - k*f1; 13 | }; 14 | 15 | if (jacobian == nullptr) solver->fsolve(vv, vf); 16 | else { 17 | auto vJ = [tv,k,&f,&jacobian,&z](const arma::vec& v) -> arma::mat { 18 | arma::mat J = (*jacobian)(tv, z + v/4); 19 | return arma::eye(arma::size(J)) - k/4*J; 20 | }; 21 | solver->fsolve(vv, vf, vJ); 22 | } 23 | return vv; 24 | } 25 | 26 | double numerics::ode::rk34i::_step(double k, double& t1, arma::vec& u1, arma::vec& f1, const odefunc& f, const odejacobian* jacobian) { 27 | arma::vec v1, v2, v3, v4, v5, z, rk3; 28 | double res = 0; 29 | const arma::vec& u = _prev_u.back(); 30 | const double& t = _prev_t.back(); 31 | 32 | double tol = std::max(_atol, _rtol*arma::norm(u,"inf")); 33 | 34 | do { 35 | v1 = _v_solve(t+0.25*k, k, u, f1, f, jacobian); 36 | 37 | z = u + v1/2; 38 | v2 = _v_solve(t+0.75*k, k, z, f1, f, jacobian); 39 | 40 | z = u + (17.0/50)*v1 - (1.0/25)*v2; 41 | v3 = _v_solve(t+(11.0/20)*k, k, z, f1, f, jacobian); 42 | 43 | z = u + (371.0/1360)*v1 - (137.0/2720)*v2 + (15.0/544)*v3; 44 | v4 = _v_solve(t+0.5*k, k, z, f1, f, jacobian); 45 | 46 | z = u + (25.0/24)*v1 - (49.0/48)*v2 + (125.0/16)*v3 - (85.0/12)*v4; 47 | v5 = _v_solve(t+k, k, z, f1, f, jacobian); 48 | 49 | rk3 = u + (59.0/48)*v1 - (17.0/96)*v2 + (225.0/32)*v3 - (85.0/12)*v4; 50 | u1 = z + v5/4; 51 | t1 = t + k; 52 | 53 | res = arma::norm(u1 - rk3, "inf"); 54 | _next_step_size(k, res, tol); 55 | } while(res > tol); 56 | return k; 57 | } -------------------------------------------------------------------------------- /src/optimization/wolfe_step.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | double numerics::optimization::wolfe_step(const dFunc& f, const VecFunc& grad_f, const arma::vec& x, const arma::vec& p, double c1, double c2) { 4 | auto g = [&](double a) -> double { 5 | return f(x + a*a*p); 6 | }; 7 | uint best, worst, n_iter=0; 8 | double R=1.0, E=2.0, Co=0.5, Ci=0.5; 9 | double ar, fr, ae, fe, ac, fc; 10 | double a[2], fa[2]; 11 | 12 | a[0]=0; 13 | a[1]=1; 14 | fa[0]=g(a[0]); 15 | fa[1]=g(a[1]); 16 | 17 | while (true) { 18 | best = (fa[0] < fa[1]) ? (0) : (1); 19 | worst = not best; 20 | 21 | bool cond1 = g(a[best]) <= f(x) + c1*a[best]*fa[best]; 22 | bool cond2 = std::abs(arma::dot(p, grad_f(x + a[best]*p))) <= c2*std::abs(arma::dot(p, grad_f(x))); 23 | if ((cond1 && cond2) || n_iter >= 100) break; 24 | 25 | // attempt reflection step 26 | ar = (1+R)*a[best] - R*a[worst]; 27 | fr = g(ar); 28 | if (fa[best] < fr && fr < fa[worst]) { // the reflection is better that worst but not as good as best, so we replace worst with the reflection 29 | a[worst] = ar; 30 | fa[worst] = fr; 31 | } else if (fr < fa[best]) { // the reflection is better than the best, so we try an even bigger step size 32 | ae = (1+E)*a[best] - E*a[worst]; 33 | fe = g(ae); 34 | if (fe < fr) { // the expansion was successful 35 | a[worst] = ae; 36 | fa[worst] = fe; 37 | } else { // the expansion was not successful 38 | a[worst] = ar; 39 | fa[worst] = fr; 40 | } 41 | } else { // the reflection step was worse than the worst, so we take a smaller step size 42 | ac = (1+Co)*a[best] - Co*a[worst]; 43 | fc = g(ac); 44 | if (fc < fr) { // contraction is better than the reflection so we keep it. 45 | a[worst] = ac; 46 | fa[worst] = fc; 47 | } else { // the contraction is worse so we replace worst with a point closer to best 48 | a[worst] = (1-Ci)*a[best] + Ci*a[worst]; 49 | fa[worst] = g(a[worst]); 50 | } 51 | } 52 | n_iter++; 53 | } 54 | if (fa[0] < fa[1]) return a[0]*a[0]; 55 | else return a[1]*a[1]; 56 | } -------------------------------------------------------------------------------- /include/numerics/integrals.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NUMERICS_INTEGRALS_HPP 2 | #define NUMERICS_INTEGRALS_HPP 3 | 4 | /* adaptive simpson's method, generally efficient. 5 | * --- fmap : if fmap is provided, all function evaluations will be stored here. 6 | * --- f : function to integrate. 7 | * --- a,b : interval [a,b] to evaluate integral over. 8 | * --- tol : error tolerance, i.e. stopping criterion */ 9 | double simpson_integral(std::map& fmap, const std::function& f, double a, double b, double tol = 1e-5); 10 | 11 | /* adaptive simpson's method, generally efficient. 12 | * --- f : function to integrate. 13 | * --- a,b : interval [a,b] to evaluate integral over. 14 | * --- tol : error tolerance, i.e. stopping criterion */ 15 | double simpson_integral(const std::function& f, double a, double b, double tol = 1e-5); 16 | 17 | /* adaptive gauss-Lobato's method, spectrally accurate for smooth functions 18 | * --- f : function to integrate. 19 | * --- a,b : interval [a,b] to evaluate integral over 20 | * --- tol : error tolerance, i.e. stopping criterion */ 21 | double lobatto_integral(const std::function& f, double a, double b, double tol = 1e-8); 22 | 23 | /* polynomial interpolation based integration, spectral method for integrating continuous functions 24 | * --- f : function to integrate. 25 | * --- a,b : interval [a,b] to evaluate integral over 26 | * --- m : exact number of function evaluations. method is O(m log m) in time (via the fft). */ 27 | double chebyshev_integral(const std::function& f, double a, double b, uint m = 32); 28 | 29 | /* general integration method. 30 | * --- f : function to integrate. 31 | * --- a,b : interval [a,b] to evaluate integral over 32 | * --- method : one of {lobatto, simpson, chebyshev}, specify method. */ 33 | inline double integrate(const std::function& f, double a, double b, const std::string& method="lobatto") { 34 | double integral = 0; 35 | if (method == "lobatto") integral = lobatto_integral(f,a,b); 36 | else if (method == "simpson") integral = simpson_integral(f,a,b); 37 | else if (method == "chebyshev") integral = chebyshev_integral(f,a,b); 38 | else { 39 | throw std::invalid_argument("integration method (=" + method + ") must be one of {lobatto, simpson, chebyshev}"); 40 | } 41 | return integral; 42 | } 43 | 44 | #endif -------------------------------------------------------------------------------- /examples/interpolation/interp_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o interp interp_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector ddvec; 7 | 8 | arma::vec f(const arma::vec& x) { 9 | return 0.5*x - 4*x%arma::exp(-arma::square(x))%(2*arma::square(x)-3); 10 | } 11 | 12 | int main() { 13 | arma::arma_rng::set_seed_random(); 14 | double a = -3; double b = 3; double m = 20; double n = 300; 15 | 16 | arma::vec x = (b-a)*arma::regspace(0,m)/m + a; 17 | arma::vec y = f(x); 18 | 19 | std::string extrapolation[5] = { 20 | "const", 21 | "boundary", 22 | "linear", 23 | "periodic", 24 | "polynomial" 25 | }; 26 | 27 | arma::vec u = arma::linspace(-5, 5, n); 28 | arma::vec v; 29 | 30 | numerics::PieceWisePoly cspline = numerics::natural_cubic_spline(x, y, extrapolation[3]); 31 | numerics::PieceWisePoly hspline = numerics::hermite_cubic_spline(x,y, extrapolation[2]); 32 | 33 | matplotlibcpp::suptitle("interpolation"); 34 | 35 | for (int i(0); i < 4; ++i) { 36 | std::string title; 37 | if (i==0) {v = cspline(u); title = "natural cubic spline with periodic extrapolation";} 38 | else if (i==1) {v = hspline(u); title = "Hermite spline with liear extrapolation";} 39 | else if (i==2) {v = numerics::lagrange_interp(x,y,u); title = "lagrange";} 40 | else if (i==3) {v = numerics::sinc_interp(x,y,u); title = "sinc";} 41 | 42 | ddvec xx = arma::conv_to::from(x); 43 | ddvec y1 = arma::conv_to::from(y.col(0)); 44 | ddvec y2; 45 | if (y.n_cols==2) y2 = arma::conv_to::from(y.col(1)); 46 | ddvec uu = arma::conv_to::from(u); 47 | ddvec v1 = arma::conv_to::from(v.col(0)); 48 | ddvec v2; 49 | if (y.n_cols==2) v2 = arma::conv_to::from(v.col(1)); 50 | 51 | matplotlibcpp::subplot(2,2,i+1); 52 | matplotlibcpp::title(title); 53 | matplotlibcpp::plot(xx, y1, "or"); 54 | if (y.n_cols==2) matplotlibcpp::plot(xx, y2, "ob"); 55 | 56 | matplotlibcpp::plot(uu, v1, "-r"); 57 | if (y.n_cols==2) matplotlibcpp::plot(uu, v2, "-b"); 58 | if (i > 0) matplotlibcpp::ylim(-5,5); 59 | } 60 | matplotlibcpp::tight_layout(); 61 | matplotlibcpp::show(); 62 | 63 | return 0; 64 | } -------------------------------------------------------------------------------- /examples/optimization/fzero_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -g -Wall -o fzero fzero_ex.cpp -O3 -lnumerics -larmadillo 4 | 5 | int main() { 6 | auto g = [](double x) -> double { return std::exp(x) - std::pow(x,2); }; 7 | auto dg = [](double x) -> double { return std::exp(x) - 2*x; }; 8 | double y; 9 | 10 | std::cout << "let's try to find roots of: f(x) = exp[x] - x^2" << std::endl; 11 | 12 | clock_t t = clock(); 13 | y = numerics::optimization::newton_1d(g, dg, 0); 14 | t = clock() - t; 15 | 16 | std::cout << "(1)\tfor newton's method we need the derivative of the function and a starting point:" << std::endl 17 | << "\t\tnewton() returned: " << y << std::endl 18 | << "\t\t|f(root)|: " << std::abs(g(y)) << std::endl 19 | << "\t\tnewton() took: " << (float)t/CLOCKS_PER_SEC << " seconds" << std::endl << std::endl; 20 | 21 | t = clock(); 22 | y = numerics::optimization::secant(g, -3, 3); 23 | t = clock() - t; 24 | 25 | std::cout << "(2)\tfor secant method we need to specify two starting points, or better yet, points that bracket the solution:" << std::endl 26 | << "\t\tsecant() returned: " << y << std::endl 27 | << "\t\t|f(root)|: " << std::abs(g(y)) << std::endl 28 | << "\t\tsecant() took: " << (float)t/CLOCKS_PER_SEC << " seconds" << std::endl << std::endl; 29 | 30 | t = clock(); 31 | y = numerics::optimization::bisect(g, -3, 3); 32 | t = clock() - t; 33 | 34 | std::cout << "(3)\tfor bisection method we need two end points of a seach region:" << std::endl 35 | << "\t\tbisect() returned: " << y << std::endl 36 | << "\t\t|f(root)|: " << std::abs(g(y)) << std::endl 37 | << "\t\tsecant() took: " << (float)t/CLOCKS_PER_SEC << " seconds" << std::endl << std::endl; 38 | 39 | t = clock(); 40 | y = numerics::optimization::fzero(g, -3, 3); 41 | t = clock() - t; 42 | 43 | std::cout << "(4)\tfor fzero() we need two end points again but we adaptively use two methods to narrow our search region:" << std::endl 44 | << "\t\troots() returned: " << y << std::endl 45 | << "\t\t|f(root)|: " << std::abs(g(y)) << std::endl 46 | << "\t\troots() took: " << (float)t/CLOCKS_PER_SEC << " seconds" << std::endl << std::endl; 47 | 48 | return 0; 49 | } -------------------------------------------------------------------------------- /examples/integration/integrate_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -Wall -g -o integrate integrate_ex.cpp -O3 -lnumerics -larmadillo 4 | 5 | double g(double x) { 6 | double y = 0; 7 | for (int i=1; i <= 10; ++i) { 8 | y += std::sin(i*x)/i; 9 | } 10 | return (0.5 - y/M_PI); 11 | } 12 | 13 | int main() { 14 | std::cout << "we will now try to integerate exp[-x^2] over [0,1]" << std::endl << std::endl; 15 | 16 | auto f = [](double x) -> double {return std::exp(-std::pow(x,2));}; // f(x) = exp[-x^2] 17 | 18 | long double val = 0.746824132812427025l; 19 | double I = numerics::integrate(f,0,1); 20 | std::cout << "\tintegrate() estimate: " << I << std::endl << "\tactual value: " << val << std::endl 21 | << "\terror: " << std::abs(I - val) << std::endl; 22 | 23 | std::cout << "\nnow we compare Simpson, Lobatto, and Chebyshev for the function: 0.5 - sum(sin(j*x)/j, {j,1,10})/pi over [-pi,pi]" << std::endl; 24 | 25 | val = M_PI; 26 | std::cout << "using |err| < 0.001" << std::endl; 27 | double y; 28 | clock_t t = clock(); 29 | y = numerics::simpson_integral(g, -M_PI, M_PI, 1e-3); 30 | t = clock() - t; 31 | std::cout << "simpson_integral() approx: " << y << " it took " << (float)t/CLOCKS_PER_SEC << " secs" << std::endl; 32 | std::cout << "\ttrue error: " << std::abs(val - y) << std::endl; 33 | 34 | t = clock(); 35 | y = numerics::lobatto_integral(g, -M_PI, M_PI, 1e-3); 36 | t = clock() - t; 37 | std::cout << "lobatto_integral() approx: " << y << " it took " << (float)t/CLOCKS_PER_SEC << " secs" << std::endl; 38 | std::cout << "\ttrue error: " << std::abs(val - y) << std::endl; 39 | 40 | t = clock(); 41 | y = numerics::chebyshev_integral(g, -M_PI, M_PI); 42 | t = clock() - t; 43 | std::cout << "chebyshev_integral() approx: " << y << " it took " << (float)t/CLOCKS_PER_SEC << " secs" << std::endl; 44 | std::cout << "\ttrue error: " << std::abs(val - y) << std::endl; 45 | 46 | std::cout << "\nsimpson_integral() stores intermediate function evaluations so the function is never evaluated twice at the same point, but may take longer to converge." << std::endl; 47 | std::cout << "\nlobatto_integral() is very accurate for smooth functions, but not recommended for discontinuous functions." << std::endl; 48 | std::cout << "\nchebyshev_integral() is spectrally accurate for smooth (analytic) functions, but not recommended for discontinuous functions." << std::endl; 49 | 50 | return 0; 51 | } -------------------------------------------------------------------------------- /src/integrals/lobatto_integral.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // double numerics::lobatto_integral(const std::function& f, double a, double b, double tol) { 4 | // tol = std::abs(tol); 5 | // double h = (b-a)/2; 6 | // double c = (b+a)/2; 7 | 8 | // double sum4(0), sum7(0); 9 | // for (int i(0); i < 7; i++) { 10 | // if (i < 4) { 11 | // sum4 += constants::lobatto_4pt_weights[i] * f(h * constants::lobatto_4pt_nodes[i] + c); 12 | // } 13 | // sum7 += constants::lobatto_7pt_weights[i] * f(h * constants::lobatto_7pt_nodes[i] + c); 14 | // } 15 | // sum4 *= h; 16 | // sum7 *= h; 17 | 18 | // if (std::abs(sum4 - sum7) < tol) return sum4; 19 | // else return lobatto_integral(f,a,c,tol/2) + lobatto_integral(f,c,b,tol/2); 20 | // } 21 | 22 | double lobatto4(const std::function& f, double a, double b) { 23 | double h = (b - a) / 2; 24 | double c = (b + a) / 2; 25 | double sum4 = 0; 26 | for (short i=0; i < 4; ++i) { 27 | sum4 += numerics::constants::lobatto_4pt_weights[i] * f(h * numerics::constants::lobatto_4pt_nodes[i] + c); 28 | } 29 | return sum4*h; 30 | } 31 | 32 | double lobatto7(const std::function& f, double a, double b) { 33 | double h = (b - a) / 2; 34 | double c = (b + a) / 2; 35 | double sum7 = 0; 36 | for (short i=0; i < 7; ++i) { 37 | sum7 += numerics::constants::lobatto_7pt_weights[i] * f(h * numerics::constants::lobatto_7pt_nodes[i] + c); 38 | } 39 | return sum7*h; 40 | } 41 | 42 | double numerics::lobatto_integral(const std::function& f, double a, double b, double tol) { 43 | if (tol <= 0) throw std::invalid_argument("require tol (=" + std::to_string(tol) + ") > 0"); 44 | if (b <= a) throw std::invalid_argument("(" + std::to_string(a) + ", " + std::to_string(b) + ") does not define an interval"); 45 | 46 | double integral = 0; 47 | 48 | std::queue aq; aq.push(a); 49 | std::queue bq; bq.push(b); 50 | std::queue tq; tq.push(tol); 51 | 52 | while (not aq.empty()) { 53 | a = aq.front(); aq.pop(); 54 | b = bq.front(); bq.pop(); 55 | tol = tq.front(); tq.pop(); 56 | 57 | double sum4 = lobatto4(f, a, b); 58 | double sum7 = lobatto7(f, a, b); 59 | if (std::abs(sum4 - sum7) < tol) integral += sum7; 60 | else { 61 | double mid = (a+b) / 2; 62 | aq.push(a); bq.push(mid); tq.push(tol/2); 63 | aq.push(mid); bq.push(b); tq.push(tol/2); 64 | } 65 | } 66 | return integral; 67 | } -------------------------------------------------------------------------------- /src/integrals/simpson_integral.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | double numerics::simpson_integral(std::map& fvals, const std::function& f, double a, double b, double tol) { 4 | if (tol <= 0) throw std::invalid_argument("require tol (=" + std::to_string(tol) + ") > 0"); 5 | if (b <= a) throw std::invalid_argument("(" + std::to_string(a) + ", " + std::to_string(b) + ") does not define a valid interval"); 6 | 7 | double integral = 0; 8 | double l, c, r, mid1, mid2, h; 9 | c = (a + b) / 2; 10 | l = (a + c) / 2; 11 | r = (c + b) / 2; 12 | 13 | std::queue> q; q.push({a,l,c,r,b}); 14 | std::queue tq; tq.push(tol); 15 | while (not q.empty()) { 16 | for (double val : q.front()) { 17 | if (fvals.count(val) == 0) fvals[val] = f(val); 18 | } 19 | a = q.front().at(0); 20 | l = q.front().at(1); 21 | c = q.front().at(2); 22 | r = q.front().at(3); 23 | b = q.front().at(4); 24 | q.pop(); 25 | tol = tq.front(); tq.pop(); 26 | 27 | h = (b - a) / 2; 28 | double s1 = (fvals[a] + 4*fvals[c] + fvals[b]) * h / 3; 29 | double s2 = (fvals[a] + 4*fvals[l] + 2*fvals[c] + 4*fvals[r] + fvals[b]) * h / 6; 30 | if (std::abs(s2 - s1) < 15*tol) integral += s2; 31 | else { 32 | mid1 = (a + l)/2; mid2 = (l + c)/2; 33 | q.push({a,mid1,l,mid2,c}); 34 | tq.push(tol/2); 35 | 36 | mid1 = (c + r)/2; mid2 = (r + b)/2; 37 | q.push({c,mid1,r,mid2,b}); 38 | tq.push(tol/2); 39 | } 40 | } 41 | return integral; 42 | } 43 | 44 | // double numerics::simpson_integral(const std::function& f, double a, double b, std::map& fmap, double tol) { 45 | // double l, c, r; 46 | // c = (a+b)/2; 47 | // l = (a+c)/2; 48 | // r = (c+b)/2; 49 | 50 | // if (fmap.count(a)==0) fmap[a] = f(a); 51 | // if (fmap.count(l)==0) fmap[l] = f(l); 52 | // if (fmap.count(c)==0) fmap[c] = f(c); 53 | // if (fmap.count(r)==0) fmap[r] = f(r); 54 | // if (fmap.count(b)==0) fmap[b] = f(b); 55 | 56 | // double h = (b-a)/2; 57 | 58 | // double s1 = (fmap[a] + 4*fmap[c] + fmap[b])*h/3; 59 | // double s2 = (fmap[a] + 4*fmap[l] + 2*fmap[c] + 4*fmap[r] + fmap[b])*h/6; 60 | 61 | // if (std::abs(s2-s1) < 15*tol) return s2; 62 | // else return simpson_integral(f,a,c,fmap,tol/2) + simpson_integral(f,c,b,fmap,tol/2); 63 | // } 64 | 65 | double numerics::simpson_integral(const std::function& f, double a, double b, double tol) { 66 | std::map fmap; 67 | return simpson_integral(fmap, f, a, b, tol); 68 | } -------------------------------------------------------------------------------- /examples/neural_network/model_ex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -O3 -o nn_model model_ex.cpp -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | arma::mat f(arma::mat& X) { 7 | arma::mat y = arma::zeros(arma::size(X)); 8 | for (int i=1; i < 10; ++i) { 9 | y += arma::sin(i*X)/i; 10 | } 11 | y = 0.5 - y/M_PI; 12 | return y; 13 | } 14 | 15 | using namespace numerics; 16 | typedef std::vector dvec; 17 | 18 | int main() { 19 | int N = 1000; 20 | arma::mat x = 6*arma::randu(N,1) - 3; 21 | arma::vec y = f(x) + 0.2*arma::randn(arma::size(x)); 22 | 23 | neuralnet::Layer input(1,15); // specify input shape and output shape 24 | input.name = "input layer"; // we can name layers for convinience 25 | input.set_activation("sqexp"); // specify activation by name 26 | 27 | neuralnet::Layer hidden(20); // specify only output shape, input shape is infered during compile() 28 | hidden.set_activation(neuralnet::Relu()); // specify activation by class which allows construction of custom activations that inherit from class Activation 29 | 30 | neuralnet::Layer output(1); // default activation is linear 31 | 32 | neuralnet::Model model(input); 33 | model.attach(hidden); // consecutively build model by attaching layers 34 | model.attach(output); 35 | model.compile(); // infers model dimensions and initializes empty layer parameters 36 | 37 | model.set_loss(neuralnet::MAE()); // set by name (e.g. "mae") or class 38 | model.set_optimizer("adam"); // set by name or class 39 | model.set_l1(0.001); // set regularization for weights 40 | 41 | neuralnet::fit_parameters fitp; // set parameters for fit 42 | fitp.max_iter = 1000; 43 | fitp.tol = 1e-4; 44 | fitp.verbose = true; 45 | 46 | model.fit(x, y, fitp); 47 | 48 | neuralnet::Layer L = model.layers.at(1); // we can get a subview of the model, and perhaps copy over layers for use in another model 49 | 50 | arma::vec yh = model.predict(x); 51 | 52 | std::cout << "R^2 : " << std::fixed << std::setprecision(2) << r2_score(y, yh) << "\n"; 53 | 54 | arma::vec t = arma::linspace(-3,3); 55 | yh = model.predict(t); 56 | 57 | dvec xx = arma::conv_to::from(x); 58 | dvec yy = arma::conv_to::from(y); 59 | dvec tt = arma::conv_to::from(arma::vec(t)); 60 | dvec yyh = arma::conv_to::from(arma::vec(yh.col(0))); 61 | 62 | std::map ls = {{"label","data"},{"ls","none"},{"marker","o"}}; 63 | matplotlibcpp::plot(xx, yy, ls); 64 | ls["label"] = "fit"; ls["marker"] = ""; ls["ls"] = "-"; 65 | matplotlibcpp::plot(tt, yyh, ls); 66 | matplotlibcpp::show(); 67 | 68 | return 0; 69 | } -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(numerics) 3 | 4 | set(CMAKE_CXX_STANDARD 14) 5 | 6 | find_package(OpenMP) 7 | 8 | include_directories(include) 9 | 10 | add_library(numerics SHARED 11 | include/numerics.hpp 12 | include/numerics/data_science.hpp 13 | include/numerics/derivatives.hpp 14 | include/numerics/integrals.hpp 15 | include/numerics/interpolation.hpp 16 | include/numerics/ode.hpp 17 | include/numerics/optimization.hpp 18 | include/numerics/utility.hpp 19 | 20 | src/data_science/kd_tree.cpp 21 | src/data_science/bin_data.cpp 22 | src/data_science/kmeans.cpp 23 | src/data_science/polyfeat.cpp 24 | src/data_science/lasso.cpp 25 | src/data_science/ridge.cpp 26 | src/data_science/splines.cpp 27 | src/data_science/bw.cpp 28 | src/data_science/kernel_smooth.cpp 29 | src/data_science/kde.cpp 30 | src/data_science/logistic_regression.cpp 31 | src/data_science/pairwise_kernel.cpp 32 | 33 | src/derivatives/approx_jacobian.cpp 34 | src/derivatives/deriv.cpp 35 | src/derivatives/grad.cpp 36 | src/derivatives/spectral_deriv.cpp 37 | 38 | src/integrals/chebyshev_integral.cpp 39 | src/integrals/lobatto_integral.cpp 40 | src/integrals/simpson_integral.cpp 41 | 42 | src/interpolation/cubic_interp.cpp 43 | src/interpolation/hspline_interp.cpp 44 | src/interpolation/lagrange_interp.cpp 45 | src/interpolation/sinc_interp.cpp 46 | src/interpolation/polynomials.cpp 47 | 48 | src/ode/am2.cpp 49 | src/ode/bvp_k.cpp 50 | src/ode/bvp_cheb.cpp 51 | src/ode/bvpIIIa.cpp 52 | src/ode/cheb.cpp 53 | src/ode/diffmat.cpp 54 | src/ode/ivp.cpp 55 | src/ode/rk4.cpp 56 | src/ode/rk5i.cpp 57 | src/ode/rk45.cpp 58 | src/ode/rk34i.cpp 59 | src/ode/poisson2d.cpp 60 | 61 | src/optimization/quasi_newton.cpp 62 | src/optimization/gradient_optimizer.cpp 63 | src/optimization/bfgs.cpp 64 | src/optimization/broyd.cpp 65 | src/optimization/pcg.cpp 66 | src/optimization/gmres.cpp 67 | src/optimization/fzero.cpp 68 | src/optimization/fmin.cpp 69 | src/optimization/lbfgs.cpp 70 | src/optimization/lmlsqr.cpp 71 | src/optimization/mgd.cpp 72 | src/optimization/mix_fpi.cpp 73 | src/optimization/nelder_mead.cpp 74 | src/optimization/newton.cpp 75 | src/optimization/trust_newton.cpp 76 | src/optimization/simplex.cpp 77 | src/optimization/wolfe_step.cpp 78 | 79 | src/utility/meshgrid.cpp 80 | src/utility/sample_from.cpp 81 | 82 | src/neural_network/layer.cpp 83 | src/neural_network/optimize.cpp 84 | src/neural_network/model.cpp 85 | ) 86 | 87 | if(OpenMP_FOUND) 88 | target_link_libraries(numerics PRIVATE OpenMP::OpenMP_CXX) 89 | endif() 90 | 91 | install(TARGETS numerics DESTINATION lib) 92 | install(FILES include/numerics.hpp DESTINATION include) 93 | install(DIRECTORY include/numerics DESTINATION include) -------------------------------------------------------------------------------- /examples/optimization/newton_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | // g++ -Wall -g -o newton newton_ex.cpp -O3 -lnumerics -larmadillo 4 | 5 | const std::string methods[] = {"newton","broyden","lm","trust"}; 6 | bool in_methods(const std::string& s) { 7 | return (std::count(methods, methods+4, s) > 0); 8 | } 9 | 10 | arma::vec f(const arma::vec& v) { 11 | arma::vec fv = {0,0,0}; 12 | double x = v(0), y = v(1), z = v(2); 13 | fv(0) = 4*std::pow(x,3) - std::pow(y,2)*z; 14 | fv(1) = -2*z*y*z + y; 15 | fv(2) = 2*z - x*std::pow(y,2); 16 | return fv; 17 | } 18 | 19 | arma::mat J(const arma::vec& v) { 20 | double x = v(0); 21 | double y = v(1); 22 | double z = v(2); 23 | arma::mat Jac = {{ 12*x*x, -2*y*z, -y*y}, 24 | {-2*y*z, -2*x*z+1, -2*x*y}, 25 | {-y*y, -2*x*y, 2}}; 26 | return Jac; 27 | } 28 | 29 | int main() { 30 | std::cout << "In this file you can test the nonlinear solvers: Newton's method, Broyden's method, and Levenberg-Marquardt." << std::endl 31 | << "Consider, f(x,y,z) = [4x^3 - z*y^2; -2xyz+y; 2z-x*y^2]" << std::endl 32 | << "We will use a random initial guess." << std::endl 33 | << "The solvers are:" << std::endl 34 | << "\t'newton' : Newton's method with line search." << std::endl 35 | << "\t'broyden' : Inexact Newton's method with line search using Broyden's update for the jacobian." << std::endl 36 | << "\t'lm' : Levenberg-Marquardt algorithm for nonlinear least-squares." << std::endl 37 | << "\t'trust' : Trust-region method constrained to 2D subspace minimization." << std::endl 38 | << "solver: "; 39 | 40 | std::string choice; 41 | do { 42 | std::cin >> choice; 43 | if (in_methods(choice)) break; 44 | else { 45 | std::cout << "solver must be one of {"; 46 | for (std::string m : methods) std::cout << m << ","; 47 | std::cout << "}, try again.\nsolver: "; 48 | } 49 | } while (true); 50 | 51 | arma::arma_rng::set_seed_random(); 52 | arma::vec x = arma::randn(3); 53 | 54 | numerics::optimization::QausiNewton *fsolver; 55 | 56 | if (choice == "newton") fsolver = new numerics::optimization::Newton(); 57 | else if (choice == "broyden") fsolver = new numerics::optimization::Broyden(); 58 | else if (choice == "lm") fsolver = new numerics::optimization::LmLSQR(); 59 | else fsolver = new numerics::optimization::TrustNewton(); 60 | 61 | clock_t t = clock(); 62 | // fsolver->fsolve(x,f,J); 63 | fsolver->fsolve(x, f); // broyd and lmlsqr do not need a jacobian function 64 | t = clock() - t; 65 | 66 | arma::vec F = f(x); 67 | std::string flag = fsolver->get_exit_flag(); 68 | 69 | std::cout << "results:" << std::endl << std::fixed << std::setprecision(4) 70 | << "\troot: [" << x(0) << ", " << x(1) << ", " << x(2) << "]" << std::endl 71 | << "\tf(root): [" << F(0) << ", " << F(1) << ", " << F(2) << "]" << std::endl 72 | << "\ttime: " << (float)t/CLOCKS_PER_SEC << " secs" << std::endl 73 | << "\ttotal iterations needed: " << fsolver->n_iter << std::endl 74 | << "\texit flag: " << flag << std::endl; 75 | 76 | return 0; 77 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Author: Amit Rotem 2 | 3 | Last modified: 10/24/2019 4 | 5 | These headers take advantage of the armadillo linear algebra library and require it for both compiling and linking. 6 | All the functions can be found in the header file and in the documentation file. 7 | 8 | ### basic installation instructions: 9 | I have only a simple CMakeLists.txt, so any modifications will have to be provided by you, the user. This library has only been tested on linux. Most recent versions of gcc should be able to compile OpenMP instructions, this library uses some OpenMP instructions here and there, so verifying `omp.h` exists on your system will significantly improve the performance of some of the data science functions. 10 | 1. install [Armadillo +v9.2](http://arma.sourceforge.net/) and [matplotlibcpp](https://github.com/lava/matplotlib-cpp) (optional). 11 | 1. `cd /numeric-lib/` 12 | 1. `cmake .` 13 | 1. `make` 14 | 1. `sudo make install` 15 | 16 | ### basic compiling instructions: 17 | This assumes you have both armadillo and numerics installed in the default locations that your compiler can find (e.g. `/usr/local/lib/`). 18 | ``` 19 | g++ main.cpp -O3 -lnumerics -larmadillo 20 | ``` 21 | The order is important because `libnumerics` also has to link against `libarmadillo`. It is also recommended that you use optimization during compile, such as `-O2`, or `-O3`. 22 | 23 | ### `numerics.hpp` is a scientific computing library hosting: 24 | * integration (4th order, 7-pt lobatto, spectral). 25 | 26 | * root finding methods (derivative, derivative free, and mixed methods). 27 | * error control, and approximation options for root finding passed to solver via class interface. 28 | 29 | * optimization methods (unconstrained and box constraints. Function, Gradient, and Hessian based methods). 30 | 31 | * interpolation schemes (lagrange, cubic, and fourier interpolation). 32 | 33 | * machine learning and data analysis tools: 34 | * train/test splitting. 35 | * k-means clusttering. 36 | * regularized linear regression. 37 | * kernel linear basis regression and classification. 38 | * kernel based smoothing. 39 | * kernel density estimation. 40 | * nearest neighbors regression and classification. 41 | * built in cross validation for automatic parameter selection for all models. 42 | 43 | * simple finite difference methods (for approximating derivatives). 44 | * uniform spectral derivatives over an interval. 45 | 46 | ### namespace `numerics::ode` for solving ordinary differential equations 47 | * Discrete differentiation matrices 48 | * explicit constant and adaptive step size IVP solvers (4th order) 49 | * implicit constant and adaptive step size IVP solvers (1st, 2nd, 4th, and 5th order) 50 | * implicit solvers use quasi-Newton methods making them more efficient than fixed point iteration. 51 | * event handling and other options passed to solver via class interface. 52 | * nonlinear BVP solver (4th, and upto spectral order) 53 | 54 | For an in depth overview, refer to the `documentation.md` file. There are also example files for most of the functions found in `/examples/` directory. 55 | 56 | Note, many of the examples rely on ["matplotlibcpp.h"](https://github.com/lava/matplotlib-cpp) which is used for visualising results of many of the algorithms. To use this feature install "matplotlibcpp.h" and make sure you have a developer version of python 2.7. -------------------------------------------------------------------------------- /src/data_science/splines.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | void numerics::Splines::set_lambda(double l) { 4 | if (_fitted) throw std::runtime_error("cannot set lambda after call to fit."); 5 | if (_use_df) throw std::runtime_error("cannot set lambda after setting df."); 6 | if (_use_lambda) throw std::runtime_error("lambda already set."); 7 | if (l < 0) throw std::invalid_argument("require lambda (=" + std::to_string(l) + ") >= 0"); 8 | 9 | _lambda = l; 10 | _use_lambda = true; 11 | } 12 | 13 | void numerics::Splines::set_df(double df) { 14 | if (_fitted) throw std::runtime_error("cannot set df after call to fit."); 15 | if (_use_df) throw std::runtime_error("df already set."); 16 | if (_use_lambda) throw std::runtime_error("cannot set df after setting lambda."); 17 | if (df < 1) throw std::invalid_argument("require df (=" + std::to_string(df) + ") >= 1."); 18 | 19 | _df = df; 20 | _use_df = true; 21 | } 22 | 23 | void numerics::Splines::fit(const arma::mat& x, const arma::vec& y) { 24 | if (_fitted) { 25 | throw std::runtime_error("Splines object already fitted."); 26 | } 27 | _check_xy(x,y); 28 | if (_use_df and (_df > x.n_rows)) { 29 | throw std::runtime_error("requested degrees of freedom (=" + std::to_string(_df) + ") exceeds the number of observations (=" + std::to_string(x.n_rows) + ")."); 30 | } 31 | _dim = x.n_cols; 32 | _X = x; 33 | arma::mat P = _add_intercept(_X); 34 | _w = arma::solve(P, y); 35 | 36 | arma::mat K = cubic_kernel(_X); 37 | arma::eig_sym(_eigvals, _eigvecs, K); 38 | arma::vec D2 = arma::pow(_eigvals,2); 39 | 40 | arma::vec res = y - P*_w; 41 | arma::vec Vres = _eigvecs.t() * res; 42 | _split_weights(); 43 | if (_use_df) { // df specified, infer lambda 44 | if (_df == _X.n_rows) _lambda = 0; 45 | else { 46 | auto g = [&](double L) -> double { 47 | return _df - arma::sum(D2 / (D2 + L)); 48 | }; 49 | double lower = D2.min(), upper = D2.max(); 50 | _lambda = optimization::fzero(g, lower, upper, std::min(1e-8, lower/2)); 51 | } 52 | } else if (not _use_lambda) { // neither df nor lambda specified, use generalized LOO cross-validation to find best lambda 53 | auto GCV = [&](double L) -> double { 54 | L = std::pow(10.0,L); 55 | _c = _eigvecs * ((_eigvals / (D2 + L)) % Vres); 56 | _df = arma::sum(D2 / (D2 + L)); 57 | 58 | double mse = mse_score(res, K*_c); 59 | return mse * std::pow(_X.n_rows / (_X.n_rows - _df), 2); 60 | }; 61 | _lambda = optimization::fminbnd(GCV, -8, 4); 62 | _lambda = std::pow(10.0, _lambda); 63 | } 64 | if (_use_lambda or _use_df) { // we have lambda, now compute _c 65 | _c = _eigvecs * arma::diagmat(_eigvals / (D2 + _lambda)) * Vres; 66 | if (_use_lambda) _df = arma::sum(D2 / (D2 + _lambda)); // infer df from lambda 67 | } 68 | _fitted = true; 69 | } 70 | 71 | arma::vec numerics::Splines::predict(const arma::mat& x) const { 72 | _check_x(x); 73 | if (not _fitted) { 74 | throw std::runtime_error("Splines object not fitted."); 75 | } 76 | return _b + x * _w + cubic_kernel(_X, x) * _c; 77 | } 78 | 79 | double numerics::Splines::score(const arma::mat& x, const arma::vec& y) const { 80 | _check_xy(x,y); 81 | return r2_score(y, predict(x)); 82 | } 83 | -------------------------------------------------------------------------------- /examples/optimization/minimize_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | using namespace numerics; 4 | 5 | // g++ -g -Wall -o minimize minimize_ex.cpp -O3 -lnumerics -larmadillo 6 | 7 | typedef std::vector vvd; 8 | 9 | double g(const arma::vec& x) { 10 | return std::pow(1-x(0),2) + 100*std::pow(x(1) - x(0)*x(0),2); 11 | } 12 | 13 | arma::vec dg(const arma::vec& x) { 14 | return {-2*(1-x(0)) - 400*x(0)*(x(1)-x(0)*x(0)), 15 | 200*(x(1)-x(0)*x(0)) }; 16 | } 17 | 18 | arma::mat H(const arma::vec& x) { 19 | return { 20 | {1200*x(0)*x(0) - 400*x(1) + 2, -400*x(0)}, 21 | { -400*x(0), 200} 22 | }; 23 | } 24 | 25 | const std::string methods[] = {"newton","trust","bfgs","lbfgs","gd"}; 26 | bool in_methods(const std::string& s) { 27 | return (std::count(methods, methods+5, s) > 0); 28 | } 29 | 30 | int main() { 31 | std::string choice; 32 | arma::arma_rng::set_seed_random(); 33 | 34 | std::cout << "we will minimize the Rosebrock function" << std::endl; 35 | 36 | arma::vec x = {-2.0,-1.0}; 37 | std::cout << "initial point: " << x.t() << std::endl; 38 | 39 | arma::vec tru_min = {1.0, 1.0}; 40 | 41 | std::cout << "The available methods are\n" 42 | << "\t'newton' : Newton's method using either exact Hessian inverse or PCG for computing a search direction while also performing a line search.\n\n" 43 | << "\t'trust' : Trust-Region method constrained to the 2D subspace spanned by gradient direction and newton direction.\n\n" 44 | << "\t'bfgs' : quasi-newton method which constructs an approximation to the Hessian inverse. This method performs a line search every iteration.\n\n" 45 | << "\t'lbfgs' : quasi-newton method which approximates the Hessian inverse, but is more efficient than BFGS per step, may require more steps.\n\n" 46 | << "\t'gd' : Momentum gradient descent. This method performs either an exact line minimization step or a constant step size with momentum. More efficient than quasi-newton methods per step, requires many steps.\n\n" 47 | << "solver: "; 48 | do { 49 | std::cin >> choice; 50 | if (in_methods(choice)) break; 51 | else { 52 | std::cout << "solver must be one of {"; 53 | for (std::string m : methods) std::cout << m << ","; 54 | std::cout << "}, try again.\nsolver: "; 55 | } 56 | } while (true); 57 | 58 | numerics::optimization::GradientOptimizer *solver; 59 | 60 | if (choice == "newton") solver = new numerics::optimization::NewtonMin(); 61 | else if (choice == "trust") solver = new numerics::optimization::TrustMin(); 62 | else if (choice == "bfgs") solver = new numerics::optimization::BFGS(); 63 | else if (choice == "lbfgs") solver = new numerics::optimization::LBFGS(); 64 | else solver = new numerics::optimization::MomentumGD(1e-3, 1e-6); 65 | 66 | clock_t tt = clock(); 67 | solver->minimize(x, g, dg); 68 | // solver->minimize(x, g, dg, H); 69 | tt = clock() - tt; 70 | 71 | std::string flag = solver->get_exit_flag(); 72 | int n_iter = solver->n_iter; 73 | 74 | std::cout << "\noptimization results:\t\t" << g(x) << std::endl 75 | << "true min:\t\t\t" << g(tru_min) << std::endl 76 | << "final x: " << x.t() << "true argmin: " << tru_min.t() 77 | << "minimize() took " << (float)tt/CLOCKS_PER_SEC << " seconds" << std::endl 78 | << "num iterations needed: " << n_iter << std::endl 79 | << "exit flag: " << flag << std::endl; 80 | return 0; 81 | } -------------------------------------------------------------------------------- /include/numerics/derivatives.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NUMERICS_DERIVATIVES_HPP 2 | #define NUMERICS_DERIVATIVES_HPP 3 | 4 | /* jacobian_diag(f, x, h, catch_zero, npt) : computes only the diagonal of a system of nonlinear equations. 5 | * --- f : f(x) system to approximate jacobian of. 6 | * --- x : vector to evaluate jacobian at. 7 | * --- h : finite difference step size. method is O(h^npt) 8 | * --- catch_zero : rounds near zero elements to zero. 9 | * --- npt : number of points to use in FD, npt = 1 uses f(x) and f(x+h), npt=2,4 do not use f(x) but instead use points f(x +/- h), f(x +/- 2h). Since npt=2 and npt=1 require the same number of f evals, npt 2 is used for its better accuracy. */ 10 | arma::vec jacobian_diag(const std::function& f, const arma::vec& x, double h = 1e-5, bool catch_zero=true, short npt=2); 11 | 12 | /* approx_jacobian(f, x, h, catch_zero, npt) : computes the jacobian of a system of nonlinear equations. 13 | * --- f : f(x) whose jacobian to approximate. 14 | * --- x : vector to evaluate jacobian at. 15 | * --- h : finite difference step size. method is O(h^npt) 16 | * --- catch_zero: rounds near zero elements to zero. 17 | * --- npt : number of number of points to use in FD, npt = 1 uses f(x) and f(x+h), npt=2,4 do not use f(x) but instead use points f(x +/- h), f(x +/- 2h). npt=1 is the default since it uses n+1 calls to f; npt=2 uses 2*n calls to f, and npt=4 uses 4*n calls to f. */ 18 | arma::mat approx_jacobian(const std::function& f, const arma::vec& x, double h=1e-5, bool catch_zero = true, short npt=1); 19 | 20 | /* grad(f, x, h, catch_zero, npt) : computes the gradient of a function of multiple variables. 21 | * --- f : f(x) whose gradient to approximate. 22 | * --- x : vector to evaluate gradient at. 23 | * --- h : finite difference step size. method is O(h^npt). 24 | * --- catch_zero: rounds near zero elements to zero. 25 | * --- npt : number of points to use in FD, npt = 1 uses f(x) and f(x+h), npt=2,4 do not use f(x) but instead use points f(x +/- h), f(x +/- 2h). npt=1 is the default since it uses n+1 calls to f; npt=2 uses 2*n calls to f, and npt=4 uses 4*n calls to f. */ 26 | arma::vec grad(const std::function& f, const arma::vec& x, double h=1e-5, bool catch_zero = true, short npt=1); 27 | 28 | /* deriv(f, x, h, catch_zero, npt) : computes the approximate derivative of a function of a single variable. 29 | * --- f : f(x) whose derivative to approximate. 30 | * --- x : point to evaluate derivative. 31 | * --- h : finite difference step size; method is O(h^npt). 32 | * --- catch_zero: rounds near zero elements to zero. 33 | * --- npt : number of points to use in FD, npt = 1 uses f(x) and f(x+h), npt=2,4 do not use f(x) but instead use points f(x +/- h), f(x +/- 2h). Since npt=2 and npt=1 require the same number of f evals, npt 2 is used for its better accuracy. */ 34 | double deriv(const std::function& f, double x, double h=1e-5, bool catch_zero = true, short npt=2); 35 | 36 | /* directional_grad(f, x, h, catch_zero, npt) : approximates J*v for J = jacobian of f; i.e. the gradient of f in the direction of v. 37 | * --- f : f(x) whose derivative to approximate 38 | * --- x : point to evaluate derivative at 39 | * --- v : direction of derivative 40 | * --- catch_zero : rounds near zero elements to zero. 41 | * --- npt : number of points to use in FD. */ 42 | arma::vec directional_grad(const std::function& f, const arma::vec& x, const arma::vec& v, double h=1e-5, bool catch_zero=true, short npt=1); 43 | 44 | Polynomial spectral_deriv(const std::function& f, double a, double b, uint sample_points = 50); 45 | 46 | #endif -------------------------------------------------------------------------------- /src/ode/bvp_k.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::ode::BVPk::solve_bvp(const odefunc& f, const boundary_conditions& bc, const arma::vec& x0, const arma::mat& U0) { 4 | auto jacobian = [&](double x, const arma::vec& u) -> arma::mat { 5 | return numerics::approx_jacobian( 6 | [&](const arma::vec& v) -> arma::vec { 7 | return f(x,v); 8 | }, 9 | u 10 | ); 11 | }; 12 | solve_bvp(f, jacobian, bc, x0, U0); 13 | } 14 | 15 | void numerics::ode::BVPk::solve_bvp(const odefunc& f, const odejacobian& jacobian, const boundary_conditions& bc, const arma::vec& x0, const arma::mat& U0) { 16 | u_long n = x0.n_elem; 17 | u_long dim = _check_dim(x0, U0); 18 | 19 | _check_x(x0); 20 | 21 | _x = x0; 22 | if (U0.n_rows == n) _u = U0.t(); 23 | else _u = U0; 24 | 25 | arma::sp_mat D; 26 | diffmat(D, _x, 1, k); 27 | 28 | arma::sp_mat DD = arma::kron(D, arma::speye(dim,dim)); 29 | 30 | arma::mat F(dim,n), du; 31 | u_long j = 0; 32 | u_long row1, row2, col1, col2; // for accessing submatrix views 33 | do { 34 | if (j >= _max_iter) { 35 | _flag = 1; 36 | break; 37 | } 38 | 39 | // set up system and jacobian 40 | arma::vec BC = bc(_u.col(0), _u.col(n-1)); 41 | if (BC.n_elem != dim) { 42 | throw std::runtime_error("number of boundary conditions (=" + std::to_string(BC.n_elem) + ") does not match the system dimensions (=" + std::to_string(dim) + ")"); 43 | } 44 | 45 | arma::mat bcJac_L = numerics::approx_jacobian( 46 | [&](const arma::vec& v) -> arma::vec { 47 | return bc(v,_u.col(n-1)); 48 | }, 49 | _u.col(0) 50 | ); 51 | arma::mat bcJac_R = numerics::approx_jacobian( 52 | [&](const arma::vec& v) -> arma::vec { 53 | return bc(_u.col(0),v); 54 | }, 55 | _u.col(n-1) 56 | ); 57 | 58 | for (u_long i=0; i < n; ++i) { 59 | F.col(i) = f( _x(i), _u.col(i) ); 60 | } 61 | 62 | arma::mat A = _u*D.t() - F; 63 | 64 | arma::vec RHS = arma::join_cols( A.as_col(), BC ); 65 | 66 | arma::sp_mat J((n+1)*dim,n*dim); 67 | for (u_long i=0; i < n; ++i) { 68 | J.submat(i*dim, i*dim, (i+1)*dim-1, (i+1)*dim-1) = -jacobian(_x(i), _u.col(i)); 69 | } 70 | J.rows(0,n*dim-1) += DD; 71 | 72 | col1 = 0; col2 = dim-1; 73 | row1 = n*dim; row2 = (n+1)*dim-1; 74 | J.submat(row1, col1, row2, col2) = bcJac_L; 75 | col1 = J.n_cols - dim; col2 = J.n_cols - 1; 76 | J.submat(row1, col1, row2, col2) = bcJac_R; 77 | 78 | RHS = J.t()*RHS; 79 | J = J.t()*J; 80 | 81 | // solve 82 | arma::superlu_opts opts; 83 | opts.symmetric = true; 84 | bool solve_success = arma::spsolve(du, J, -RHS, "superlu", opts); 85 | if (not solve_success) { 86 | _flag = 3; 87 | break; 88 | } 89 | if (du.has_nan() || du.has_inf()) { 90 | _flag = 2; 91 | break; 92 | } 93 | _u += arma::reshape(du,dim,n); 94 | j++; 95 | } while (arma::abs(du).max() > _tol); 96 | 97 | _num_iter = j; 98 | 99 | for (u_int i=0; i < n; ++i) { 100 | F.col(i) = f(_x(i), _u.col(i)); 101 | } 102 | _du = std::move(F); 103 | 104 | for (u_int i=0; i < dim; ++i) { 105 | _sol.push_back(hermite_cubic_spline(_x,_u.row(i).as_col(),_du.row(i).as_col(),"boundary")); 106 | } 107 | } -------------------------------------------------------------------------------- /src/data_science/bin_data.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::BinData::_set_bins(u_long n_obs) { 4 | if (_n == 0) { 5 | if (n_obs > 1000) _n = 500; // almost always sufficient 6 | if (n_obs > 30) _n = n_obs / 10; // somewhat arbitrary... we want _n << x.n_elem 7 | else _n = n_obs / 5; 8 | if (_n <= 5) _n = n_obs; 9 | } 10 | if (n_obs <= _n) { 11 | throw std::invalid_argument("require number of observations in x (x.n_rows=" + std::to_string(n_obs) + ") > number of bins (=" + std::to_string(_n) + ")."); 12 | } 13 | } 14 | 15 | void numerics::BinData::fit(const arma::mat& x) { 16 | _set_bins(x.n_rows); 17 | 18 | if (x.n_cols > 1) { 19 | throw std::logic_error("multi-dimensional binning not yet supported (x.n_cols=" + std::to_string(x.n_cols) + " > 1)."); 20 | } else { 21 | arma::vec xx = arma::sort(x); 22 | double xmin = xx.front(); 23 | double xmax = xx.back(); 24 | double eps = 1e-3 * (xmax - xmin); 25 | _bins = arma::linspace(xmin - eps, xmax + eps, _n); 26 | double _bin_width = _bins(1) - _bins(0); 27 | _counts = arma::zeros(_n); 28 | 29 | int j = 0; 30 | for (int i=0; i < xx.n_elem; ++i) { 31 | bool counted = false; 32 | while (!counted) { 33 | if (_bins(j) <= xx(i) && xx(i) < _bins(j+1)) { // xx(i) is in bin j 34 | double w = (_bins(j+1) - xx(i)) / _bin_width; 35 | _counts(j) += w; 36 | _counts(j+1) += 1-w; 37 | counted = true; 38 | } else j++; 39 | } 40 | } 41 | } 42 | } 43 | 44 | void numerics::BinData::fit(const arma::mat& x, const arma::vec& y) { 45 | if (x.n_rows != y.n_rows) { 46 | throw std::invalid_argument("data vectors must have the same length (x.n_rows = " + std::to_string(x.n_rows) + " != " + std::to_string(y.n_rows) + " = y.n_rows)"); 47 | } 48 | _set_bins(x.n_rows); 49 | 50 | if (bins.n_cols > 1) { 51 | throw std::logic_error("multi-dimensional binning not yet supported (x.n_cols=" + std::to_string(x.n_cols) + " > 1)."); 52 | } else { 53 | arma::uvec I = arma::sort_index(x); 54 | arma::vec xx = x(I); 55 | arma::vec yy = y(I); 56 | 57 | double xmin = xx.front(); 58 | double xmax = xx.back(); 59 | double eps = 1e-3 * (xmax - xmin); 60 | _bins = arma::linspace(xmin - eps, xmax + eps, _n); 61 | double _bin_width = _bins(1) - _bins(0); 62 | _counts = arma::zeros(_n); 63 | 64 | int i=0; 65 | double w=0, sum_w=0, sum_yw=0; 66 | while (i < x.n_elem && xx(i) < bins(1)) { 67 | w = 1 - std::abs(xx(i)-_bins(1))/_bin_width; 68 | sum_w += w; 69 | sum_yw += yy(i)*w; 70 | i++; 71 | } 72 | if (sum_w != 0) _counts(0) = sum_yw / sum_w; 73 | 74 | int i1; 75 | for (int j=1; j < _n-1; ++j) { 76 | sum_w=0; 77 | sum_yw=0; 78 | if (j==_n-2) i1 = i; 79 | while (i < x.n_elem && _bins(j-1) <= xx(i) && xx(i) < _bins(j+1)) { 80 | w = 1 - std::abs(xx(i)-_bins(j))/_bin_width; 81 | sum_w += w; 82 | sum_yw += yy(i)*w; 83 | i++; 84 | } 85 | if (sum_w != 0) _counts(j) = sum_yw/sum_w; 86 | } 87 | 88 | sum_w=0; 89 | sum_yw=0; 90 | i = i1; 91 | while (i < x.n_elem) { 92 | w = 1 - std::abs(xx(i)-_bins(_n-1))/_bin_width; 93 | sum_w += w; 94 | sum_yw += yy(i)*w; 95 | i++; 96 | } 97 | if (sum_w != 0) _counts(_n-1) = sum_yw/sum_w; 98 | } 99 | } -------------------------------------------------------------------------------- /src/derivatives/approx_jacobian.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | arma::mat _J1p(const std::function& f, const arma::vec& x, double h) { 4 | arma::mat J = arma::repmat(-f(x), 1, x.n_elem); 5 | arma::vec y = x; 6 | for (uint i=0; i < x.n_elem; ++i) { 7 | y(i) += h; 8 | J.col(i) += f(y); 9 | y(i) = x(i); 10 | } 11 | J /= h; 12 | return J; 13 | } 14 | 15 | arma::mat _J2p(const std::function& f, const arma::vec& x, double h) { 16 | arma::vec y = x; 17 | y(0) += h; // perform first step outside of loop for inferring size of f. 18 | arma::vec ff = f(y); 19 | y(0) = x(0) - h; 20 | ff -= f(y); 21 | y(0) = x(0); 22 | uint m = ff.n_elem; 23 | arma::mat J(m,x.n_elem); 24 | J.col(0) = ff; 25 | for (uint i=1; i < x.n_elem; ++i) { 26 | y(i) += h; 27 | J.col(i) = f(y); 28 | y(i) = x(i) - h; 29 | J.col(i) -= f(y); 30 | y(i) = x(i); 31 | } 32 | J /= 2*h; 33 | return J; 34 | } 35 | 36 | arma::mat _J4p(const std::function& f, const arma::vec& x, double h) { 37 | arma::vec y = x; 38 | y(0) += 2*h; 39 | arma::vec ff = -f(y); 40 | y(0) = x(0) + h; 41 | ff += 8*f(y); 42 | y(0) = x(0) - h; 43 | ff -= 8*f(y); 44 | y(0) = x(0) - 2*h; 45 | ff += f(y); 46 | y(0) = x(0); 47 | 48 | arma::mat J(ff.n_elem, x.n_elem); 49 | for (uint i=1; i < x.n_elem; ++i) { 50 | y(i) += 2*h; 51 | J.col(i) = -f(y); 52 | y(i) = x(i) + h; 53 | J.col(i) += 8*f(y); 54 | y(i) = x(i) - h; 55 | J.col(i) -= 8*f(y); 56 | y(i) = x(i) - 2*h; 57 | J.col(i) += f(y); 58 | y(i) = x(i); 59 | } 60 | J /= 12*h; 61 | return J; 62 | } 63 | 64 | /* approx_jacobian(f, x, h, catch_zero) : computes the jacobian of a system of nonlinear equations. 65 | * --- f : f(x) whose jacobian to approximate. 66 | * --- x : vector to evaluate jacobian at. 67 | * --- h : finite difference step size. method is O(h^4) 68 | * --- catch_zero: rounds near zero elements to zero. */ 69 | arma::mat numerics::approx_jacobian(const std::function& f, const arma::vec& x, double h, bool catch_zero, short npt) { 70 | uint n = x.n_elem; // num variables -> num cols 71 | if (x.n_elem < 1) throw std::invalid_argument("when computing the jacobian, require x.n_elem (=" + std::to_string(x.n_elem) + ") >= 1."); 72 | 73 | arma::mat J; 74 | if (npt == 1) J = _J1p(f, x, h); 75 | else if (npt == 2) J = _J2p(f, x, h); 76 | else if (npt == 4) J = _J4p(f, x, h); 77 | else { 78 | throw std::invalid_argument("only 1, 2, and 4 point derivatives supported (not " + std::to_string(npt) + ")."); 79 | } 80 | 81 | if (catch_zero) J(arma::find(arma::abs(J) < h/2)).zeros(); 82 | 83 | return J; 84 | } 85 | 86 | /* jacobian_diag(f, x, h) : computes only the diagonal of a system of nonlinear equations. 87 | * --- f : f(x) system to approximate jacobian of. 88 | * --- x : vector to evaluate jacobian at. 89 | * --- h : finite difference step size. method is O(h^4) */ 90 | arma::vec numerics::jacobian_diag(const std::function& f, const arma::vec& x, double h, bool catch_zero, short npt) { 91 | int m = x.n_elem; 92 | arma::vec J = arma::zeros(m); 93 | for (int i=0; i < m; ++i) { 94 | auto ff = [&f,&x,i](double z) -> double { 95 | arma::vec u = x; 96 | u(i) = z; 97 | u = f(u); 98 | return u(i); 99 | }; 100 | J(i) = deriv(ff, x(i), h, catch_zero, npt); 101 | } 102 | return J; 103 | } -------------------------------------------------------------------------------- /src/ode/bvp_cheb.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::ode::BVPCheb::solve_bvp(const odefunc& f, const boundary_conditions& bc, const arma::vec& x0, const arma::mat& U0) { 4 | auto jacobian = [&](double x, const arma::vec& u) -> arma::mat { 5 | return numerics::approx_jacobian( 6 | [&](const arma::vec& v) -> arma::vec { 7 | return f(x,v); 8 | }, 9 | u 10 | ); 11 | }; 12 | solve_bvp(f, jacobian, bc, x0, U0); 13 | } 14 | 15 | void numerics::ode::BVPCheb::solve_bvp(const odefunc& f, const odejacobian& jacobian, const boundary_conditions& bc, const arma::vec& x0, const arma::mat& U0) { 16 | u_long dim = _check_dim(x0, U0); 17 | 18 | _check_x(x0); 19 | 20 | arma::mat D; 21 | cheb(D, _x, x0.front(), x0.back(), num_pts); 22 | arma::mat DD = arma::kron(D,arma::eye(dim,dim)); 23 | 24 | _u.set_size(dim, num_pts); 25 | for (u_long i=0; i < dim; ++i) { 26 | arma::vec uu; 27 | if (U0.n_rows == num_pts) arma::interp1(x0, U0.col(i), _x, uu, "*linear"); 28 | else arma::interp1(x0, U0.row(i).as_col(), _x, uu, "*linear"); 29 | _u.row(i) = uu.as_row(); 30 | } 31 | 32 | u_long row1, row2, col1, col2; // for accessing submatrix views 33 | 34 | arma::mat F(dim,num_pts), du; 35 | arma::mat J((num_pts+1)*dim, num_pts*dim); 36 | u_long j = 0; 37 | do { 38 | if (j >= _max_iter) { 39 | _flag = 1; 40 | break; 41 | } 42 | 43 | // set up system and jacobian 44 | arma::vec BC = bc(_u.col(0), _u.col(num_pts-1)); 45 | if (BC.n_elem != dim) { 46 | throw std::runtime_error("number of boundary conditions (=" + std::to_string(BC.n_elem) + ") does not match the system dimensions (=" + std::to_string(dim) + ")"); 47 | } 48 | arma::mat bcJac_L = numerics::approx_jacobian( 49 | [&](const arma::vec& v) -> arma::vec { 50 | return bc(v,_u.col(num_pts-1)); 51 | }, 52 | _u.col(0) 53 | ); 54 | arma::mat bcJac_R = numerics::approx_jacobian( 55 | [&](const arma::vec& v) -> arma::vec { 56 | return bc(_u.col(0),v); 57 | }, 58 | _u.col(num_pts-1) 59 | ); 60 | 61 | for (int i=0; i < num_pts; ++i) { 62 | F.col(i) = f( _x(i), _u.col(i) ); 63 | } 64 | 65 | arma::mat A = _u*D.t() - F; 66 | 67 | arma::vec RHS = arma::join_cols( A.as_col(), BC ); 68 | 69 | J.zeros(); 70 | for (int i=0; i < num_pts; ++i) { 71 | J.submat(i*dim, i*dim, (i+1)*dim-1, (i+1)*dim-1) = -jacobian(_x(i),_u.col(i)); 72 | } 73 | J.rows(0, num_pts*dim-1) += DD; 74 | 75 | col1 = 0; col2 = dim-1; 76 | row1 = num_pts*dim; row2 = (num_pts+1)*dim-1; 77 | J.submat(row1, col1, row2, col2) = bcJac_L; 78 | col1 = J.n_cols - dim; col2 = J.n_cols - 1; 79 | J.submat(row1, col1, row2, col2) = bcJac_R; 80 | 81 | // solve 82 | bool solve_success = arma::solve(du, J, -RHS); 83 | if (not solve_success) { 84 | _flag = 3; 85 | break; 86 | } 87 | if (du.has_nan() || du.has_inf()) { 88 | _flag = 2; 89 | break; 90 | } 91 | _u += arma::reshape(du,dim,num_pts); 92 | j++; 93 | } while (arma::abs(du).max() > _tol); 94 | _num_iter = j; 95 | 96 | for (u_int i=0; i < num_pts; ++i) { 97 | F.col(i) = f(x(i), _u.col(i)); 98 | } 99 | _du = std::move(F); 100 | 101 | for (u_long i=0; i < dim; ++i) { 102 | _sol.push_back(Polynomial(_x, _u.row(i).as_col())); 103 | } 104 | } -------------------------------------------------------------------------------- /examples/optimization/lmlsqr_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o lmlsqr lmlsqr_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector dvec; 7 | 8 | arma::vec f_true(const arma::vec& t) { 9 | return arma::exp(-t%t); 10 | } 11 | 12 | int main() { 13 | std::cout << std::fixed << std::setprecision(3); 14 | std::cout << "LmLSQR uses the Levenberg-Marquardt algorithm to find least squares solutions of root finding problems f(x) = 0" << std::endl; 15 | std::cout << "We will try to perform a nonlinear fit of the form:" << std::endl 16 | << "\ty = f(x) = b(0) + b(1) / (b(2) + b(3)*x^2)" << std::endl; 17 | 18 | arma::arma_rng::set_seed_random(); 19 | double lower = -5; double upper = 5; 20 | arma::vec x = (upper-lower)*arma::randu(100) + lower; 21 | arma::vec t = arma::linspace(lower,upper); 22 | arma::vec y_true = f_true(t); 23 | arma::vec y = f_true(x) + 0.1*arma::randn(arma::size(x)); 24 | 25 | arma::vec b = {-0.13217, 0.66178, 0.56353, 0.78157}; 26 | arma::vec b_hat = {0,1,1,1}; b_hat += 0.1*arma::randn(4); 27 | std::cout << "we will use initial guess b_hat = [" 28 | << b_hat(0) << ", " << b_hat(1) << ", " << b_hat(2) << ", " << b_hat(3) 29 | << "] because it is close to our actual parameters" << std::endl << std::endl; 30 | 31 | auto f = [&x,&y](const arma::vec& b) -> arma::vec { 32 | return b(0) + b(1) / (b(2) + b(3)*x%x) - y; 33 | }; 34 | 35 | // providing a jacobian function provides a substantial performance boost, you can try it with or without it. 36 | auto J = [&x](const arma::vec& b) -> arma::mat { 37 | arma::mat A(x.n_elem,4,arma::fill::zeros); 38 | A.col(0) = arma::ones(arma::size(x)); 39 | A.col(1) = 1 / (b(2) + b(3)*x%x); 40 | A.col(2) = -b(1) / arma::pow(b(2) + b(3)*x%x, 2); 41 | A.col(3) = -b(1)*x%x / arma::pow(b(2) + b(3)*x%x, 2); 42 | return A; 43 | }; 44 | 45 | numerics::optimization::LmLSQR lm(1e-3,100,true); 46 | lm.use_lu(); 47 | // lm.fsolve(b_hat,f,J); // specify jacobian 48 | lm.fsolve(b_hat,f); // compute jacobian by finite differences and Broyden updates 49 | std::string flag = lm.get_exit_flag(); 50 | 51 | std::cout << "results after optimization : " << std::endl 52 | << "\tb_hat = " << b_hat.t() 53 | << "\tb_hat sum of squares = " << arma::norm(f(b_hat)) << std::endl << std::endl 54 | << "\ttheoretical b = " << b.t() 55 | << "\ttheoretical b sum of squares = " << arma::norm(f(b)) << std::endl 56 | << "\t||b - b_hat|| = " << arma::norm(b - b_hat,"inf") << std::endl << std::endl 57 | << "\tflag:" << flag << std::endl << std::endl 58 | << "resulting model:" << std::endl; 59 | std::cout << std::fixed << std::setprecision(3) 60 | << "\ty = " << b_hat(0) << " + " << b_hat(1) << " / (" << b_hat(2) << " + " << b_hat(3) << "x^2)" << std::endl; 61 | std::cout << "We can, ofcourse, use lmlsqr to compute roots of an algebraic system, which is explored in newton_ex" << std::endl; 62 | 63 | arma::vec y_hat = b_hat(0) + b_hat(1) / (b_hat(2) + b_hat(3)*t%t); 64 | 65 | dvec xx = arma::conv_to::from(x); 66 | dvec yy = arma::conv_to::from(y); 67 | dvec tt = arma::conv_to::from(t); 68 | dvec yyt = arma::conv_to::from(y_true); 69 | dvec yyh = arma::conv_to::from(y_hat); 70 | 71 | matplotlibcpp::named_plot("data",xx,yy,"og"); 72 | matplotlibcpp::named_plot("exact model", tt, yyt, "--k"); 73 | matplotlibcpp::named_plot("least squares model", tt, yyh, "-r"); 74 | matplotlibcpp::legend(); 75 | matplotlibcpp::show(); 76 | 77 | return 0; 78 | } -------------------------------------------------------------------------------- /src/interpolation/cubic_interp.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | numerics::PieceWisePoly numerics::natural_cubic_spline(const arma::vec& x, const arma::vec&y, const std::string& extrapolation, double val) { 4 | PieceWisePoly out(extrapolation,val); 5 | 6 | out._check_xy(x, y); 7 | 8 | u_long n = x.n_elem - 1; 9 | 10 | arma::vec h = arma::zeros(n); 11 | arma::sp_mat A(n+1,n+1); 12 | arma::vec RHS = arma::zeros(n+1); 13 | arma::vec b = arma::zeros(n); 14 | arma::vec d = arma::zeros(n); 15 | arma::uvec I = arma::sort_index(x); 16 | out._x = x(I); 17 | out._check_x(out._x); 18 | arma::vec _y = y(I); 19 | 20 | for (u_long i=1; i < n+1; ++i) { 21 | h(i-1) = out._x(i) - out._x(i-1); 22 | } 23 | 24 | arma::vec subD = h; 25 | arma::vec supD(arma::size(subD),arma::fill::zeros); 26 | arma::vec mainD(n+1,arma::fill::zeros); 27 | 28 | subD(n-1) = 0; 29 | mainD(0) = 1; 30 | mainD(n) = 1; 31 | supD(0) = 0; 32 | 33 | for (u_long i=1; i < n; ++i) { 34 | mainD(i) = 2 * (h(i) + h(i-1)); 35 | supD(i) = h(i); 36 | 37 | RHS(i) = 3 * (_y(i+1) - _y(i))/h(i) - 3 * (_y(i) - _y(i-1))/h(i-1); 38 | } 39 | 40 | A.diag(-1) = subD; 41 | A.diag() = mainD; 42 | A.diag(1) = supD; 43 | 44 | arma::vec c = spsolve(A,RHS); 45 | 46 | for (u_long i=0; i < n; ++i) { 47 | b(i) = (_y(i+1) - _y(i))/h(i) - h(i)*(2*c(i) + c(i+1))/3; 48 | d(i) = (c(i+1) - c(i))/(3*h(i)); 49 | } 50 | c = c.rows(0,n-1); 51 | 52 | for (u_long i=0; i < n; ++i) { 53 | arma::vec p = { 54 | d(i), 55 | c(i) - 3*d(i)*out._x(i), 56 | b(i) - 2*c(i)*out._x(i) + 3*d(i)*std::pow(out._x(i),2), 57 | _y(i) - b(i)*out._x(i) + c(i)*std::pow(out._x(i),2) - d(i)*std::pow(out._x(i),3) 58 | }; // convert spline to polynomial 59 | out._P.push_back(Polynomial(p)); 60 | } 61 | return out; 62 | } 63 | 64 | // numerics::CubicInterp::CubicInterp(const arma::vec& x, const arma::vec& y, const std::string& extrapolation, double val) : PieceWisePoly(extrapolation,val) { 65 | // _check_xy(x, y); 66 | 67 | // u_long n = x.n_elem - 1; 68 | 69 | // arma::vec h = arma::zeros(n); 70 | // arma::sp_mat A(n+1,n+1); 71 | // arma::vec RHS = arma::zeros(n+1); 72 | // arma::vec b = arma::zeros(n); 73 | // arma::vec d = arma::zeros(n); 74 | // arma::uvec I = arma::sort_index(x); 75 | // _x = x(I); 76 | // _check_x(_x); 77 | // arma::vec _y = y(I); 78 | 79 | // for (u_long i=1; i < n+1; ++i) { 80 | // h(i-1) = _x(i) - _x(i-1); 81 | // } 82 | 83 | // arma::vec subD = h; 84 | // arma::vec supD(arma::size(subD),arma::fill::zeros); 85 | // arma::vec mainD(n+1,arma::fill::zeros); 86 | 87 | // subD(n-1) = 0; 88 | // mainD(0) = 1; 89 | // mainD(n) = 1; 90 | // supD(0) = 0; 91 | 92 | // for (u_long i=1; i < n; ++i) { 93 | // mainD(i) = 2 * (h(i) + h(i-1)); 94 | // supD(i) = h(i); 95 | 96 | // RHS(i) = 3 * (_y(i+1) - _y(i))/h(i) - 3 * (_y(i) - _y(i-1))/h(i-1); 97 | // } 98 | 99 | // A.diag(-1) = subD; 100 | // A.diag() = mainD; 101 | // A.diag(1) = supD; 102 | 103 | // arma::vec c = spsolve(A,RHS); 104 | 105 | // for (u_long i=0; i < n; ++i) { 106 | // b(i) = (_y(i+1) - _y(i))/h(i) - h(i)*(2*c(i) + c(i+1))/3; 107 | // d(i) = (c(i+1) - c(i))/(3*h(i)); 108 | // } 109 | // c = c.rows(0,n-1); 110 | 111 | // for (u_long i=0; i < n; ++i) { 112 | // arma::vec p = { 113 | // d(i), 114 | // c(i) - 3*d(i)*_x(i), 115 | // b(i) - 2*c(i)*_x(i) + 3*d(i)*std::pow(_x(i),2), 116 | // _y(i) - b(i)*_x(i) + c(i)*std::pow(_x(i),2) - d(i)*std::pow(_x(i),3) 117 | // }; // convert spline to polynomial 118 | // _P.push_back(Polynomial(p)); 119 | // } 120 | // } -------------------------------------------------------------------------------- /examples/ode/ode_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o diffeq ode_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | using namespace numerics::ode; 7 | typedef std::vector dvec; 8 | 9 | const std::string methods[] = {"am2","rk4","rk5i","rk34i","rk45"}; 10 | bool in_methods(const std::string& s) { 11 | return (std::count(methods, methods+5, s) > 0); 12 | } 13 | 14 | const double mu = 100; // bigger value --> more stiff 15 | 16 | arma::vec f(double t, const arma::vec& u) { 17 | arma::vec v(2); 18 | v(0) = u(1); 19 | v(1) = mu*(1-u(0)*u(0))*u(1) - u(0); // vanderpol with mu = 1 20 | return v; 21 | } 22 | 23 | arma::mat J(double t, const arma::vec& u) { 24 | double x = u(0); 25 | double y = u(1); 26 | arma::mat M = {{ 0, 1}, 27 | {-2*mu*x*y - 1, mu*(1-x*x)}}; 28 | return M; 29 | } 30 | 31 | double evnt(double t, const arma::vec& u) { 32 | return u(0) + 1.6; 33 | } 34 | 35 | int main() { 36 | const double t0 = 0; 37 | const double tf = std::max(20.0,2*(3-2*std::log(2))*mu); 38 | const arma::vec& U0 = {2,0}; 39 | const bool add_event = false; 40 | 41 | std::cout << "Let's solve the vanderpol equation with mu=" << mu << std::endl 42 | << "We will assume the true solution is the output of rk45 with error < 1e-6." << std::endl 43 | << "The solvers are:" << std::endl 44 | << "\t'am2' : trapezoid rule" << std::endl 45 | << "\t'rk4' : fourth order 5-stage explicit Runge-Kutta" << std::endl 46 | << "\t'rk5i' : diag-implicit fifth order Runge-Kutta" << std::endl 47 | << "\t'rk34i' : adaptive diag-implicit fourth order Runge-Kutta" << std::endl 48 | << "\t'rk45' : adaptive fourth order Dormand-Prince method." << std::endl 49 | << "solver: "; 50 | 51 | std::string choice; 52 | do { 53 | std::cin >> choice; 54 | if (in_methods(choice)) break; 55 | else { 56 | std::cout << "solver must be one of {"; 57 | for (std::string m : methods) std::cout << m << ","; 58 | std::cout << "}, try again.\nsolver: "; 59 | } 60 | } while (true); 61 | 62 | ivpOpts opts; opts.atol = 1e-8; opts.rtol = 1e-6; 63 | rk45 RK45(opts); 64 | 65 | if (add_event) RK45.add_stopping_event(evnt, "inc"); // enable event 66 | RK45.solve_ivp(f, t0, tf, U0); // we will use our rk45() approximation as the exact solution 67 | 68 | arma::mat U; arma::vec t; 69 | RK45.as_mat(t,U); 70 | 71 | InitialValueProblem* dsolver; 72 | ivpOpts opts1; 73 | 74 | if (choice == "am2") {opts1.cstep = 1/(4*mu); dsolver = new am2(opts1);} 75 | else if (choice == "rk4") {opts1.cstep = 1/(2*mu); dsolver = new rk4(opts1);} 76 | else if (choice == "rk5i") {opts1.cstep = 1/mu; dsolver = new rk5i(opts1);} 77 | else if (choice == "rk34i") dsolver = new rk34i(opts1); 78 | else dsolver = new rk45(opts1); 79 | 80 | if (add_event) dsolver->add_stopping_event(evnt, "inc"); 81 | dsolver->solve_ivp(f,t0,tf,U0); 82 | // dsolver->solve_ivp(f,J,t0,tf,U0); 83 | 84 | arma::mat V; arma::vec s; 85 | dsolver->as_mat(s,V); 86 | 87 | matplotlibcpp::subplot(1,2,1); 88 | dvec uu = arma::conv_to::from(U.row(0)); 89 | matplotlibcpp::named_plot("U1 - exact", RK45.t, uu, "-r"); 90 | uu = arma::conv_to::from(V.row(0)); 91 | matplotlibcpp::named_plot("U1 - test", dsolver->t, uu, "*k"); 92 | 93 | 94 | matplotlibcpp::subplot(1,2,2); 95 | uu = arma::conv_to::from(U.row(1)); 96 | matplotlibcpp::named_plot("U2 - exact", RK45.t, uu, "-b"); 97 | uu = arma::conv_to::from(V.row(1)); 98 | matplotlibcpp::named_plot("U2 - test", dsolver->t, uu, "*k"); 99 | matplotlibcpp::legend(); 100 | matplotlibcpp::show(); 101 | 102 | return 0; 103 | } -------------------------------------------------------------------------------- /include/numerics/utility.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NUMERICS_UTILITY_HPP 2 | #define NUMERICS_UTILITY_HPP 3 | 4 | // --- utitility -------------- // 5 | namespace constants { 6 | // --- integral constants 7 | const long double lobatto_4pt_nodes[4] = {-1, -0.447213595499958, 0.447213595499958, 1}; 8 | const long double lobatto_4pt_weights[4] = {0.166666666666667, 0.833333333333333, 0.833333333333333, 0.166666666666667}; 9 | 10 | const long double lobatto_7pt_nodes[7] = {-1, -0.468848793470714, -0.830223896278567, 0, 0.830223896278567, 0.468848793470714, 1}; 11 | const long double lobatto_7pt_weights[7] = {0.047619047619048, 0.431745381209863, 0.276826047361566, 0.487619047619048, 0.276826047361566, 0.431745381209863, 0.047619047619048}; 12 | } 13 | 14 | inline int mod(int a, int b) { 15 | return (a%b + b)%b; 16 | } 17 | 18 | template class CycleQueue { 19 | protected: 20 | u_long _max_elem; 21 | u_long _size; 22 | u_long _head; 23 | std::vector _data; 24 | 25 | public: 26 | // const std::vector& data; 27 | explicit CycleQueue(u_long size) /* : data(_data) */ { 28 | if (size < 1) throw std::runtime_error("cannot initialize CycleQueue to empty size"); 29 | _max_elem = size; 30 | _size = 0; 31 | _head = 0; 32 | } 33 | 34 | void push(const eT& x) { 35 | if (_size < _max_elem) { 36 | _data.push_back(x); 37 | _size++; 38 | } else { 39 | _data.at(_head) = x; 40 | _head = (_head + 1) % _size; 41 | } 42 | } 43 | 44 | void push(eT&& x) { 45 | if (_size < _max_elem) { 46 | _data.push_back(x); 47 | _size++; 48 | } else { 49 | _data.at(_head) = x; 50 | _head = (_head + 1) % _size; 51 | } 52 | } 53 | 54 | eT& at(u_long i) { 55 | if (i >= _size) { 56 | throw std::range_error("index (=" + std::to_string(i) + ") out of range of CycleQueue of size=" + std::to_string(_size) + ", with maximum size=" + std::to_string(_max_elem)); 57 | } 58 | u_long ind = (i + _head) % _size; 59 | return _data.at(ind); 60 | } 61 | 62 | const eT& at(u_long i) const { 63 | if (i >= _size) { 64 | throw std::range_error("index (=" + std::to_string(i) + ") out of range of CycleQueue of size=" + std::to_string(_size) + ", with maximum size=" + std::to_string(_max_elem)); 65 | } 66 | u_long ind = (i + _head) % _size; 67 | return _data.at(ind); 68 | } 69 | 70 | eT& back() { 71 | if (_size == 0) { 72 | throw std::range_error("cannot access back of empty queue."); 73 | } 74 | return _data.at(mod(_head - 1, _size)); 75 | } 76 | 77 | const eT& back() const { 78 | if (_size == 0) { 79 | throw std::range_error("cannot access back of empty queue."); 80 | } 81 | return _data.at(mod(_head - 1, _size)); 82 | } 83 | 84 | eT& front() { 85 | if (_size == 0) { 86 | throw std::range_error("cannot access back of empty queue."); 87 | } 88 | return _data.at(mod(_head-_size,_size)); 89 | } 90 | 91 | const eT& front() const { 92 | if (_size == 0) { 93 | throw std::range_error("cannot access back of empty queue."); 94 | } 95 | return _data.at(mod(_head-_size,_size)); 96 | } 97 | 98 | u_long size() const { 99 | return _size; 100 | } 101 | 102 | void clear() { 103 | _data.clear(); 104 | _size = 0; 105 | _head = 0; 106 | } 107 | }; 108 | 109 | void meshgrid(arma::mat&, arma::mat&, const arma::vec&, const arma::vec&); 110 | void meshgrid(arma::mat&, const arma::vec&); 111 | 112 | arma::uvec sample_from(int, const arma::vec&, const arma::uvec& labels = arma::uvec()); 113 | int sample_from(const arma::vec&, const arma::uvec& labels = arma::uvec()); 114 | 115 | uint index_median(const arma::vec& x); 116 | 117 | #endif -------------------------------------------------------------------------------- /include/numerics/interpolation.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NUMERICS_INTERPOLATION_HPP 2 | #define NUMERICS_INTERPOLATION_HPP 3 | 4 | /* polyder(p, k) : return the k^th derivative of a polynomial. 5 | * --- p : polynomial to differentiate. 6 | * --- k : the derivative order (k = 1 by default, i.e. first derivative). */ 7 | arma::vec polyder(const arma::vec& p, uint k = 1); 8 | 9 | /* polyint(p, c) : return the integral of a polynomial. 10 | * --- p : polynomial to integrate. 11 | * --- c : integration constant. */ 12 | arma::vec polyint(const arma::vec& p, double c = 0); 13 | 14 | class Polynomial { 15 | protected: 16 | arma::vec _p; 17 | u_int _deg; 18 | 19 | void _set_degree(); 20 | 21 | public: 22 | const u_int& degree; 23 | const arma::vec& coefficients; 24 | 25 | // default constructor --> constant 26 | explicit Polynomial(double s=0); 27 | // initialize with polynomial coefficients taken column-wise 28 | explicit Polynomial(const arma::vec& p); 29 | explicit Polynomial(arma::vec&& p); 30 | // initialize with interpolation problem 31 | explicit Polynomial(const arma::vec& x, const arma::vec& y, u_int deg); 32 | explicit Polynomial(const arma::vec& x, const arma::vec& y); 33 | 34 | Polynomial(const Polynomial& P); 35 | void operator=(const Polynomial& P); 36 | 37 | double operator()(double x) const; 38 | arma::vec operator()(const arma::vec& x) const; 39 | 40 | 41 | Polynomial derivative(u_int k=1) const; 42 | Polynomial integral(double c=0) const; 43 | Polynomial operator+(const Polynomial& P) const; 44 | Polynomial operator+(double c) const; 45 | 46 | Polynomial operator-() const; 47 | Polynomial operator-(const Polynomial& P) const; 48 | Polynomial operator-(double c) const; 49 | 50 | 51 | Polynomial operator*(const Polynomial& P) const; 52 | Polynomial operator*(double c) const; 53 | 54 | Polynomial& operator+=(const Polynomial& P); 55 | Polynomial& operator+=(double c); 56 | Polynomial& operator-=(const Polynomial& P); 57 | Polynomial& operator-=(double c); 58 | Polynomial& operator*=(const Polynomial& P); 59 | Polynomial& operator*=(double c); 60 | }; 61 | 62 | std::ostream& operator<<(std::ostream& out, const Polynomial& p); 63 | 64 | class PieceWisePoly { 65 | friend PieceWisePoly natural_cubic_spline(const arma::vec& x, const arma::vec&y, const std::string& extrapolation, double val); 66 | friend PieceWisePoly hermite_cubic_spline(const arma::vec& x, const arma::vec& y, const std::string& extrapolation, double val); 67 | friend PieceWisePoly hermite_cubic_spline(const arma::vec& x, const arma::vec& y, const arma::vec& yp, const std::string& extrapolation, double val); 68 | 69 | protected: 70 | double _lb, _ub; 71 | short _extrap; 72 | double _extrap_val; 73 | std::vector _P; 74 | arma::vec _x; 75 | 76 | void _check_xy(const arma::vec& x, const arma::vec& y); 77 | void _check_x(const arma::vec& x); 78 | 79 | double _periodic(double t) const; 80 | double _flat_past_boundary(double t) const; 81 | 82 | public: 83 | PieceWisePoly(const std::string& extrapolation="const", double val=0); 84 | 85 | double operator()(double t) const; 86 | 87 | arma::vec operator()(const arma::vec& t) const; 88 | 89 | PieceWisePoly derivative(int k=1) const; 90 | PieceWisePoly integral(double c=0) const; 91 | }; 92 | 93 | PieceWisePoly natural_cubic_spline(const arma::vec& x, const arma::vec&y, const std::string& extrapolation="boundary", double val=0); 94 | PieceWisePoly hermite_cubic_spline(const arma::vec& x, const arma::vec& y, const std::string& extrapolation="linear", double val=0); 95 | PieceWisePoly hermite_cubic_spline(const arma::vec& x, const arma::vec& y, const arma::vec& yp, const std::string& extrapolation="linear", double val=0); 96 | 97 | arma::mat lagrange_interp(const arma::vec&, const arma::mat&, const arma::vec&); 98 | arma::mat sinc_interp(const arma::vec&, const arma::mat&, const arma::vec&); 99 | 100 | #endif -------------------------------------------------------------------------------- /src/optimization/nelder_mead.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /* init_simplex(x) : initialize a simplex from an intial guess using randomized orthonormal directions. 4 | * --- x : initial guess of optimal point */ 5 | arma::mat numerics::optimization::NelderMead::_init_simplex(const arma::vec& x) { 6 | int n = x.n_elem; 7 | arma::mat dx = arma::randn(n,1); 8 | dx = _side_length*arma::join_rows( arma::orth(dx), arma::null(dx.t()) ); // spanning orthonormal matrix 9 | dx.each_col() += x; 10 | return arma::join_rows(x,dx); 11 | } 12 | 13 | void numerics::optimization::NelderMead::minimize(arma::vec& x, const dFunc& f) { 14 | int m=x.n_elem; 15 | arma::mat xx = _init_simplex(x); 16 | arma::vec yy = arma::zeros(xx.n_cols); 17 | for (int i=0; i < m+1; ++i) { 18 | yy(i) = f(xx.col(i)); 19 | } 20 | 21 | int worst, scndw, best; 22 | _n_iter = 0; 23 | VerboseTracker T(_max_iter); 24 | if (_v) T.header(); 25 | while (true) { 26 | arma::uvec ind = arma::sort_index(yy); 27 | worst = ind(m); 28 | scndw = ind(m-1); 29 | best = ind(0); 30 | 31 | // reflect x(worst) accross center 32 | arma::vec c = arma::mean(xx.cols(ind.rows(0,m-1)),1); 33 | arma::vec xr = c + _step * (c - xx.col(worst)); 34 | double yr = f(xr); 35 | if (yy(best) < yr && yr < yy(scndw)) { 36 | xx.col(worst) = xr; 37 | yy(worst) = yr; 38 | } else if (yr < yy(best)) { // new point is very good, attempt further search in this direction 39 | arma::vec xe = c + _expand * (xr - c); 40 | double ye = f(xe); 41 | if (ye < yr) { 42 | xx.col(worst) = xe; 43 | yy(worst) = ye; 44 | } else { 45 | xx.col(worst) = xr; 46 | yy(worst) = yr; 47 | } 48 | } else if (yy(scndw) < yr) { // potential over shoot 49 | if (yy(scndw) < yr && yr < yy(worst)) { // contraction outside simplex 50 | arma::vec xc = c + _contract * (xr - c); 51 | double yc = f(xc); 52 | if (yc < yr) { 53 | xx.col(worst) = xc; 54 | yy(worst) = yc; 55 | } else { // shrink simplex 56 | for (int i=0; i < m+1; ++i) { 57 | if (i==best) continue; 58 | xx.col(i) = xx.col(best) + _shrink * (xx.col(i) - xx.col(best)); 59 | yy(i) = f(xx.col(i)); 60 | } 61 | } 62 | } else if (yr > yy(worst)) { // contraction inside simplex 63 | arma::vec xc = c + _contract * (xx.col(worst) - c); 64 | double yc = f(xc); 65 | if (yc < yy(worst)) { 66 | xx.col(worst) = xc; 67 | yy(worst) = yc; 68 | } else { // shrink simplex 69 | for (int i=0; i < m+1; ++i) { 70 | if (i==best) continue; 71 | xx.col(i) = xx.col(best) + _shrink * (xx.col(i) - xx.col(best)); 72 | yy(i) = f(xx.col(i)); 73 | } 74 | } 75 | } 76 | } 77 | if (_v) T.iter(_n_iter, yy(best)); 78 | _n_iter++; 79 | 80 | double ftol = _ftol*std::max(1.0, std::abs(yy(best))); 81 | if (std::abs(yy(best) - yy(worst)) < ftol) { 82 | _exit_flag = 0; 83 | if (_v) T.success_flag(); 84 | break; 85 | } 86 | 87 | double xtol = _xtol*std::max(1.0, arma::norm(xx.col(best))); 88 | if (arma::norm(xx.col(worst) - xx.col(best),"inf") > xtol) { 89 | _exit_flag = 1; 90 | if (_v) T.success_flag(); 91 | break; 92 | } 93 | 94 | if (_n_iter >= _max_iter) { 95 | _exit_flag = 2; 96 | if (_v) T.max_iter_flag(); 97 | break; 98 | } 99 | 100 | 101 | } 102 | u_int i = yy.index_min(); 103 | x = xx.col(i); 104 | } -------------------------------------------------------------------------------- /src/neural_network/layer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::neuralnet::Layer::_initialize() { 4 | double limit = std::sqrt(6.0 / (_shape[0] + _shape[1])); 5 | if (_weights.is_empty()) _weights = (2*limit)*arma::randu(input_shape, units) - limit; 6 | if (_bias.is_empty()) _bias = arma::zeros(1, _shape[1]); 7 | } 8 | 9 | arma::mat numerics::neuralnet::Layer::_evaluate(arma::mat& x) const { 10 | arma::mat z = x * weights; 11 | z.each_row() += bias; 12 | return _activation->evaluate(z); 13 | } 14 | 15 | void numerics::neuralnet::Layer::_compute_delta(const Layer& L_i1p) { 16 | _delta = L_i1p._delta * L_i1p.weights.t(); 17 | _delta = _delta % _activation->derivative(_Z); 18 | } 19 | 20 | void numerics::neuralnet::Layer::_compute_derivatives(const arma::mat& A_i1m) { 21 | _dW = A_i1m.t() * _delta; 22 | _db = arma::sum(_delta, 0); 23 | } 24 | 25 | void numerics::neuralnet::Layer::_compute_layer_outputs(const arma::mat& A_i1m) { 26 | _Z = A_i1m * weights; 27 | _Z.each_row() += bias; 28 | _A = _activation->evaluate(_Z); 29 | } 30 | 31 | void numerics::neuralnet::Layer::set_activation(const std::string& activation) { 32 | if (activation == "relu") _activation = std::make_unique(); 33 | else if (activation == "tanh") _activation = std::make_unique(); 34 | else if (activation == "logexp") _activation = std::make_unique(); 35 | else if (activation == "logistic") _activation = std::make_unique(); 36 | else if (activation == "softmax") _activation = std::make_unique(); 37 | else if (activation == "linear") _activation = std::make_unique(); 38 | else if (activation == "trig") _activation = std::make_unique(); 39 | else if (activation == "cubic") _activation = std::make_unique(); 40 | else if (activation == "sqexp") _activation = std::make_unique(); 41 | else { 42 | std::vector ACTIVATIONS = {"linear","relu","tanh","trig","cubic","sqexp","logexp","logistic","softmax"}; 43 | std::string err = "activation (=" + activation + ") must belong to {"; 44 | for (const std::string& a : ACTIVATIONS) { 45 | err += ", " + a; 46 | } 47 | err += "}"; 48 | throw std::invalid_argument(err); 49 | } 50 | } 51 | 52 | void numerics::neuralnet::Layer::set_activation(const Activation& activation) { 53 | _activation = activation.clone(); 54 | } 55 | 56 | void numerics::neuralnet::Layer::set_weights(const arma::mat& w) { 57 | if (input_shape == 0) { 58 | throw std::runtime_error("cannot set weights without explicitly declaring the input shape, or before compiling model."); 59 | } 60 | if ((w.n_rows != input_shape) or (w.n_cols != units)) { 61 | throw std::invalid_argument("weight shape (=[" + std::to_string(w.n_rows) + ", " + std::to_string(w.n_cols) + "]) does not equal layer shape (=[" + std::to_string(input_shape) + ", " + std::to_string(units) + "])."); 62 | } 63 | if (not w.is_finite()) { 64 | throw std::runtime_error("weights contain NaN or infinite values."); 65 | } 66 | _weights = w; 67 | } 68 | 69 | void numerics::neuralnet::Layer::set_bias(const arma::mat& b) { 70 | if (input_shape == 0) { 71 | throw std::runtime_error("cannot set weights without explicitly declaring the input shape, or before compiling model."); 72 | } 73 | if (b.n_elem != units) { 74 | throw std::invalid_argument("bias size (=" + std::to_string(b.n_elem) + ") does not equal the layer output shape (=" + std::to_string(units) + ")."); 75 | } 76 | if (not b.is_finite()) { 77 | throw std::runtime_error("bias contains NaN or infinite values."); 78 | } 79 | _bias = b.as_row(); 80 | } 81 | 82 | void numerics::neuralnet::Layer::disable_training_weights() { 83 | _trainable_weights = false; 84 | } 85 | 86 | void numerics::neuralnet::Layer::enable_training_weight() { 87 | _trainable_weights = true; 88 | } 89 | 90 | void numerics::neuralnet::Layer::disable_training_bias() { 91 | _trainable_bias = false; 92 | } 93 | 94 | void numerics::neuralnet::Layer::enable_training_bias() { 95 | _trainable_bias = true; 96 | } -------------------------------------------------------------------------------- /src/data_science/k_fold.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /* k_folds(x, y, k, dim) : split data into k many groups, i.e. folds. 4 | * --- X : independent variable data matrix. 5 | * --- Y : dependent variable data matrix. 6 | * --- k : number of folds. 7 | * --- dim : dimension to split along. default dim=0, i.e. split the colomns where each row is treated as a data point. dim=1 implies each colomn is a data point and we split the rows. */ 8 | numerics::k_folds::k_folds(const arma::mat& x, const arma::mat& y, uint k, uint dim) { 9 | int m = x.n_rows, n = x.n_cols; 10 | direction = dim; 11 | num_folds = k; 12 | 13 | X = x; 14 | Y = y; 15 | if (direction==0) { 16 | range = arma::shuffle(arma::regspace(0,m-1)); 17 | I = arma::reshape(range, m/k, k); 18 | } else { 19 | range = arma::shuffle(arma::regspace(0,n-1)); 20 | I = arma::reshape(range, n/k, k); 21 | } 22 | range = arma::regspace(0,k-1); 23 | } 24 | 25 | /* fold_X(j) : return the x-values of the j^th fold */ 26 | arma::mat numerics::k_folds::test_set_X(uint j) { 27 | if (j >= num_folds) { 28 | std::cerr << "k_folds element access index (=" << j << ") is out of range of the size - 1 (=" << num_folds << ")." << std::endl; 29 | return arma::mat(); 30 | } 31 | if (direction==0) return X.rows( I.col(j) ); 32 | else return X.cols( I.col(j) ); 33 | } 34 | 35 | /* fold_Y(j) : return the y-values of the j^th fold */ 36 | arma::mat numerics::k_folds::test_set_Y(uint j) { 37 | if (j >= num_folds) { 38 | std::cerr << "k_folds element access index (=" << j << ") is out of range of the size - 1 (=" << num_folds << ")." << std::endl; 39 | return arma::mat(); 40 | } 41 | if (direction==0) return Y.rows( I.col(j) ); 42 | else return Y.cols( I.col(j) ); 43 | } 44 | 45 | /* not_fold_X(j) : return the x-values of all but j^th fold */ 46 | arma::mat numerics::k_folds::train_set_X(uint j) { 47 | if (j >= num_folds) { 48 | std::cerr << "k_folds element access index (=" << j << ") is out of range of the size - 1 (=" << num_folds << ")." << std::endl; 49 | return arma::mat(); 50 | } 51 | arma::umat ii = I.cols(arma::find(range != j)); 52 | ii = arma::vectorise(ii); 53 | if (direction==0) return X.rows(ii); 54 | else return X.cols(ii); 55 | } 56 | 57 | /* not_fold_Y(j) : return the y-values of all but j^th fold */ 58 | arma::mat numerics::k_folds::train_set_Y(uint j) { 59 | if (j >= num_folds) { 60 | std::cerr << "k_folds element access index (=" << j << ") is out of range of the size - 1 (=" << num_folds << ")." << std::endl; 61 | return arma::mat(); 62 | } 63 | arma::umat ii = I.cols(arma::find(range != j)); 64 | ii = arma::vectorise(ii); 65 | if (direction==0) return Y.rows(ii); 66 | else return Y.cols(ii); 67 | } 68 | 69 | numerics::k_folds_1d::k_folds_1d(const arma::mat& x, uint k, uint dim) { 70 | int m = x.n_rows, n = x.n_cols; 71 | direction = dim; 72 | num_folds = k; 73 | 74 | X = x; 75 | if (direction==0) { 76 | range = arma::shuffle(arma::regspace(0,m-1)); 77 | I = arma::reshape(range, m/k, k); 78 | } else { 79 | range = arma::shuffle(arma::regspace(0,n-1)); 80 | I = arma::reshape(range, n/k, k); 81 | } 82 | range = arma::regspace(0,k-1); 83 | } 84 | 85 | arma::mat numerics::k_folds_1d::test_set(uint j) { 86 | if (j >= num_folds) { 87 | std::cerr << "k_folds element access index (=" << j << ") is out of range of the size - 1 (=" << num_folds << ")." << std::endl; 88 | return arma::mat(); 89 | } 90 | if (direction==0) return X.rows( I.col(j) ); 91 | else return X.cols( I.col(j) ); 92 | } 93 | 94 | arma::mat numerics::k_folds_1d::train_set(uint j) { 95 | if (j >= num_folds) { 96 | std::cerr << "k_folds element access index (=" << j << ") is out of range of the size - 1 (=" << num_folds << ")." << std::endl; 97 | return arma::mat(); 98 | } 99 | arma::umat ii = I.cols(arma::find(range != j)); 100 | ii = arma::vectorise(ii); 101 | if (direction==0) return X.rows(ii); 102 | else return X.cols(ii); 103 | } -------------------------------------------------------------------------------- /src/data_science/logistic_regression.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void mult_coefs_serial(arma::mat& yh, const arma::vec& coefs, const arma::rowvec& b, const arma::mat& w, const arma::mat& x) { 4 | yh = arma::repmat(coefs.subvec(0,b.n_elem-1).as_row(), x.n_rows, 1); 5 | arma::uword start = b.n_elem; 6 | for (u_long i=0; i < w.n_rows; ++i) { 7 | for (u_long j=0; j < w.n_cols; ++j) { 8 | arma::uword ij = start + arma::sub2ind(arma::size(w), i, j); 9 | yh.col(j) += x.col(i) * coefs(ij); 10 | } 11 | } 12 | numerics::softmax_inplace(yh); 13 | } 14 | 15 | arma::vec grad_serial(const arma::mat& yh, const arma::vec& coefs, const arma::rowvec& b, const arma::mat& w, const arma::mat& x, const arma::mat& y, double L) { 16 | u_long size = x.n_cols + 1; 17 | size *= y.n_cols; 18 | 19 | arma::vec out(size); 20 | 21 | arma::mat res = y - yh; 22 | 23 | for (u_long i=0; i < b.n_elem; ++i) { 24 | out(i) = -arma::sum(res.col(i)); 25 | } 26 | 27 | arma::uword start = b.n_elem; 28 | for (u_long i=0; i < w.n_rows; ++i) { 29 | for (u_long j=0; j < w.n_cols; ++j) { 30 | arma::uword ij = start + arma::sub2ind(arma::size(w), i, j); 31 | out(ij) = -arma::dot(x.col(i), res.col(j)); 32 | out(ij) += L * coefs(ij); 33 | } 34 | } 35 | return out; 36 | } 37 | 38 | void fit_logreg(arma::mat& w, const arma::sp_mat& I, const arma::mat& P, const arma::mat& y, double L) { 39 | u_long n = P.n_cols-1; 40 | 41 | auto f = [&](const arma::vec& coefs) -> double { 42 | arma::mat W = arma::reshape(coefs, n+1, y.n_cols); 43 | 44 | arma::mat yh = P * W; 45 | numerics::softmax_inplace(yh); 46 | 47 | return 0.5*L*arma::trace(W.t()*I*W) - arma::accu(yh % y); 48 | }; 49 | 50 | auto grad_f = [&](const arma::vec& coefs) -> arma::vec { 51 | arma::mat W = arma::reshape(coefs, n+1, y.n_cols); 52 | 53 | arma::mat yh = P * W; 54 | numerics::softmax_inplace(yh); 55 | 56 | arma::mat df = L*I*W - P.t()*(y - yh); 57 | 58 | return df.as_col(); 59 | }; 60 | 61 | arma::vec coefs = w.as_col(); 62 | 63 | numerics::optimization::TrustMin fmin; 64 | fmin.minimize(coefs, f, grad_f); 65 | 66 | w = arma::reshape(coefs, n+1, y.n_cols); 67 | } 68 | 69 | arma::mat pred_logreg(const arma::mat& w, const arma::mat& P) { 70 | arma::mat yh = P*w; 71 | numerics::softmax_inplace(yh); 72 | return yh; 73 | } 74 | 75 | void numerics::LogisticRegression::fit(const arma::mat& x, const arma::uvec& y) { 76 | _check_xy(x, y); 77 | _dim = x.n_cols; 78 | u_long nobs = x.n_rows; 79 | 80 | _encoder.fit(y); 81 | arma::mat onehot = _encoder.encode(y); 82 | 83 | u_long n = x.n_cols; 84 | arma::mat P(nobs, n+1); 85 | arma::sp_mat I = arma::speye(n+1,n+1); I(0,0) = 0; 86 | P.col(0).ones(); 87 | P.cols(1,n) = x; 88 | 89 | arma::mat W = arma::zeros(n+1, onehot.n_cols); 90 | 91 | if (_lambda < 0) { 92 | u_short nfolds = 5; 93 | if (nobs / nfolds < 50) nfolds = 3; 94 | KFolds2Arr split(nfolds); 95 | split.fit(P, onehot); 96 | 97 | auto cv = [&](double L) -> double { 98 | L = std::pow(10.0, L); 99 | double score = 0; 100 | for (u_short i=0; i < nfolds; ++i) { 101 | fit_logreg(W, I, split.trainX(i), split.trainY(i), L); 102 | arma::mat p = pred_logreg(W, split.testX(i)); 103 | score -= accuracy_score(_encoder.decode(split.testY(i)), _encoder.decode(p)); 104 | } 105 | return score; 106 | }; 107 | _lambda = optimization::fminbnd(cv, -5, 4, 0.1); 108 | _lambda = std::pow(10.0, _lambda); 109 | } 110 | 111 | fit_logreg(W, I, P, onehot, _lambda); 112 | _b = W.row(0); 113 | _w = W.rows(1,n); 114 | } 115 | 116 | arma::mat numerics::LogisticRegression::predict_proba(const arma::mat& x) const { 117 | _check_x(x); 118 | arma::mat yh = x * _w; 119 | yh.each_row() += _b; 120 | softmax_inplace(yh); 121 | return yh; 122 | } 123 | 124 | arma::uvec numerics::LogisticRegression::predict(const arma::mat& x) const { 125 | return _encoder.decode(predict_proba(x)); 126 | } 127 | 128 | double numerics::LogisticRegression::score(const arma::mat& x, const arma::uvec& y) const { 129 | _check_xy(x,y); 130 | return accuracy_score(y, predict(x)); 131 | } -------------------------------------------------------------------------------- /src/optimization/fmin.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | double numerics::optimization::fminbnd(const std::function& f, double a, double b, double tol) { 4 | if (b <= a) { 5 | throw std::invalid_argument("invalid interval; a must be less than b but (a=" + std::to_string(a) + ", b=" + std::to_string(b) + ")."); 6 | } 7 | if (tol <= 0) throw std::invalid_argument("fminbnd(): require tol (=" + std::to_string(tol) + ") > 0"); 8 | double x,u,v,w,fu,fv,fw,fx; 9 | double c,d,e,m,p,q,r; 10 | 11 | c = (3 - std::sqrt(5))/2; 12 | x = a+c*(b-a); e=0; v = x; w = x; 13 | fx = f(x); fv = fx; fw = fx; 14 | 15 | while (std::max(x-a,b-x) > 2*tol) { 16 | m = (a+b)/2; 17 | p=0;q=0;r=0; 18 | bool pinterp_succ = false; 19 | if (std::abs(e) > tol) { // attempt parabolic interpolation step 20 | r = (x - w) * (fx - fv); 21 | q = (x - v) * (fx - fw); 22 | p = (x - v)*q - (x-w)*r; 23 | q = 2*(q-r); 24 | if (q >= 0) p = -p; 25 | else q = -q; 26 | r = e; 27 | e = d; 28 | if ((std::abs(p)q*(a-x)) && (p=m) ? (tol) : (-tol); 34 | } 35 | } 36 | } 37 | if (!pinterp_succ) { // use golden section step 38 | if (x < m) e = b-x; 39 | else e = a-x; 40 | d = c*e; 41 | } 42 | 43 | 44 | if (std::abs(d) >= tol) u = x+d; // we do not want to evaluate the function again within tol of x. 45 | else if (d > 0) u = x+tol; 46 | else u = x-tol; 47 | 48 | fu = f(u); 49 | 50 | if (fu < fx) { // update points of interest 51 | if (u < x) b = x; 52 | else a = x; 53 | v=w; fv=fw; 54 | w=x; fw=fx; 55 | x=u; fx=fu; 56 | } else { 57 | if (u < x) a = u; 58 | else b = u; 59 | if ((fu <= fw)||(w==x)) { 60 | v=w; fv=fw; 61 | w=u; fw=fu; 62 | } 63 | else if ((fu <= fv)||(v==x)||(v==w)) { 64 | v=u; fv=fu; 65 | } 66 | } 67 | } 68 | return x; 69 | } 70 | 71 | double numerics::optimization::fminsearch(const std::function& f, double x0, double alpha) { 72 | uint best, worst; 73 | double tol = (std::abs(x0)<2e-8) ? (1e-8) : ((1e-8)*std::abs(x0)); 74 | double R=1.0, E=2.0, Co=0.5, Ci=0.5; 75 | double xr, fr, xe, fe, xc, fc; 76 | double x[2], fx[2]; 77 | 78 | if (alpha <= 0) alpha = 5*tol; 79 | 80 | x[0]=x0; 81 | x[1]=x0+alpha; 82 | fx[0]=f(x[0]); 83 | fx[1]=f(x[1]); 84 | 85 | while (std::abs(x[1] - x[0]) > 2*tol) { 86 | best = (fx[0] < fx[1]) ? (0) : (1); 87 | worst = not best; 88 | // attempt reflection step 89 | xr = (1+R)*x[best] - R*x[worst]; 90 | fr = f(xr); 91 | if (fx[best] < fr && fr < fx[worst]) { // the reflection is better that worst but not as good as best, so we replace worst with the reflection 92 | x[worst] = xr; 93 | fx[worst] = fr; 94 | } else if (fr < fx[best]) { // the reflection is better than the best, so we try an even bigger step size 95 | xe = (1+E)*x[best] - E*x[worst]; 96 | fe = f(xe); 97 | if (fe < fr) { // the expansion was successful 98 | x[worst] = xe; 99 | fx[worst] = fe; 100 | } else { // the expansion was not successful 101 | x[worst] = xr; 102 | fx[worst] = fr; 103 | } 104 | } else { // the reflection step was worse than the worst, so we take a smaller step size 105 | xc = (1+Co)*x[best] - Co*x[worst]; 106 | fc = f(xc); 107 | if (fc < fr) { // contraction is better than the reflection so we keep it. 108 | x[worst] = xc; 109 | fx[worst] = fc; 110 | } else { // the contraction is worse so we replace worst with a point closer to best 111 | x[worst] = (1-Ci)*x[best] + Ci*x[worst]; 112 | fx[worst] = f(x[worst]); 113 | } 114 | } 115 | } 116 | if (fx[0] < fx[1]) return x[0]; 117 | else return x[1]; 118 | } -------------------------------------------------------------------------------- /src/interpolation/hspline_interp.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | numerics::PieceWisePoly numerics::hermite_cubic_spline(const arma::vec& x, const arma::vec& y, const std::string& extrapolation, double val) { 4 | PieceWisePoly out(extrapolation,val); 5 | 6 | out._check_xy(x,y); 7 | 8 | u_long n = x.n_elem; 9 | arma::uvec I = arma::sort_index(x); 10 | out._x = x(I); 11 | out._check_x(out._x); 12 | arma::vec _y = y(I); 13 | 14 | arma::sp_mat D; 15 | ode::diffmat(D,out._x); 16 | arma::vec _dy = D*_y; 17 | 18 | arma::vec h = out._x.rows(1,n-1) - out._x.rows(0,n-2); 19 | arma::vec d = (2*(_y.rows(0,n-2) - _y.rows(1,n-1)) + (_dy.rows(0,n-2) + _dy.rows(1,n-1))%h) / arma::pow(h,3); 20 | arma::vec c = -(3*(_y.rows(0,n-2) - _y.rows(1,n-1)) + (2*_dy.rows(0,n-2) + _dy.rows(1,n-1))%h) / arma::pow(h,2); 21 | 22 | for (u_int i=0; i < n-1; ++i) { 23 | arma::vec p = { 24 | d(i), 25 | c(i) - 3*d(i)*out._x(i), 26 | _dy(i) - 2*c(i)*out._x(i) + 3*d(i)*std::pow(out._x(i),2), 27 | _y(i) - _dy(i)*out._x(i) + c(i)*std::pow(out._x(i),2) - d(i)*std::pow(out._x(i),3) 28 | }; 29 | out._P.push_back(Polynomial(p)); 30 | } 31 | return out; 32 | } 33 | 34 | numerics::PieceWisePoly numerics::hermite_cubic_spline(const arma::vec& x, const arma::vec& y, const arma::vec& yp, const std::string& extrapolation, double val) { 35 | PieceWisePoly out(extrapolation,val); 36 | out._check_xy(x,y); 37 | out._check_xy(x,yp); 38 | 39 | u_long n = x.n_elem; 40 | arma::uvec I = arma::sort_index(x); 41 | out._x = x(I); 42 | out._check_x(out._x); 43 | arma::vec _y = y(I); 44 | arma::vec _dy = yp(I); 45 | 46 | arma::vec h = out._x.rows(1,n-1) - out._x.rows(0,n-2); 47 | arma::vec d = (2*(_y.rows(0,n-2) - _y.rows(1,n-1)) + (_dy.rows(0,n-2) + _dy.rows(1,n-1))%h) / arma::pow(h,3); 48 | arma::vec c = -(3*(_y.rows(0,n-2) - _y.rows(1,n-1)) + (2*_dy.rows(0,n-2) + _dy.rows(1,n-1))%h) / arma::pow(h,2); 49 | 50 | for (u_int i=0; i < n-1; ++i) { 51 | arma::vec p = { 52 | d(i), 53 | c(i) - 3*d(i)*out._x(i), 54 | _dy(i) - 2*c(i)*out._x(i) + 3*d(i)*std::pow(out._x(i),2), 55 | _y(i) - _dy(i)*out._x(i) + c(i)*std::pow(out._x(i),2) - d(i)*std::pow(out._x(i),3) 56 | }; 57 | out._P.push_back(Polynomial(p)); 58 | } 59 | return out; 60 | } 61 | 62 | // numerics::HSplineInterp::HSplineInterp(const arma::vec& x, const arma::vec& y, const arma::vec& yp, const std::string& extrapolation, double val) : PieceWisePoly(extrapolation,val) { 63 | // _check_xy(x,y); 64 | // _check_xy(x,yp); 65 | 66 | // u_long n = x.n_elem; 67 | // arma::uvec I = arma::sort_index(x); 68 | // _x = x(I); 69 | // _check_x(_x); 70 | // arma::vec _y = y(I); 71 | // arma::vec _dy = yp(I); 72 | 73 | // arma::vec h = _x.rows(1,n-1) - _x.rows(0,n-2); 74 | // arma::vec d = (2*(_y.rows(0,n-2) - _y.rows(1,n-1)) + (_dy.rows(0,n-2) + _dy.rows(1,n-1))%h) / arma::pow(h,3); 75 | // arma::vec c = -(3*(_y.rows(0,n-2) - _y.rows(1,n-1)) + (2*_dy.rows(0,n-2) + _dy.rows(1,n-1))%h) / arma::pow(h,2); 76 | 77 | // for (u_int i=0; i < n-1; ++i) { 78 | // arma::vec p = { 79 | // d(i), 80 | // c(i) - 3*d(i)*_x(i), 81 | // _dy(i) - 2*c(i)*_x(i) + 3*d(i)*std::pow(x(i),2), 82 | // _y(i) - _dy(i)*_x(i) + c(i)*std::pow(x(i),2) - d(i)*std::pow(x(i),3) 83 | // }; 84 | // _P.push_back(Polynomial(p)); 85 | // } 86 | // } 87 | 88 | // numerics::HSplineInterp::HSplineInterp(const arma::vec& x, const arma::vec& y, const std::string& extrapolation, double val) : PieceWisePoly(extrapolation,val) { 89 | // _check_xy(x,y); 90 | 91 | // u_long n = x.n_elem; 92 | // arma::uvec I = arma::sort_index(x); 93 | // _x = x(I); 94 | // _check_x(_x); 95 | // arma::vec _y = y(I); 96 | 97 | // arma::sp_mat D; 98 | // ode::diffmat(D,_x); 99 | // arma::vec _dy = D*_y; 100 | 101 | // arma::vec h = _x.rows(1,n-1) - _x.rows(0,n-2); 102 | // arma::vec d = (2*(_y.rows(0,n-2) - _y.rows(1,n-1)) + (_dy.rows(0,n-2) + _dy.rows(1,n-1))%h) / arma::pow(h,3); 103 | // arma::vec c = -(3*(_y.rows(0,n-2) - _y.rows(1,n-1)) + (2*_dy.rows(0,n-2) + _dy.rows(1,n-1))%h) / arma::pow(h,2); 104 | 105 | // for (u_int i=0; i < n-1; ++i) { 106 | // arma::vec p = { 107 | // d(i), 108 | // c(i) - 3*d(i)*_x(i), 109 | // _dy(i) - 2*c(i)*_x(i) + 3*d(i)*std::pow(x(i),2), 110 | // _y(i) - _dy(i)*_x(i) + c(i)*std::pow(x(i),2) - d(i)*std::pow(x(i),3) 111 | // }; 112 | // _P.push_back(Polynomial(p)); 113 | // } 114 | // } 115 | -------------------------------------------------------------------------------- /examples/ode/bvp_ex.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | #include "matplotlibcpp.h" 3 | 4 | // g++ -g -Wall -o bvp bvp_ex.cpp -O3 -lnumerics -larmadillo -I/usr/include/python3.8 -lpython3.8 5 | 6 | typedef std::vector dvec; 7 | 8 | int main() { 9 | double a = -M_PI, b = M_PI; 10 | 11 | std::cout << "Now we will solve the boundary value problem:\n" 12 | << "\tu''(x) = 2(1 - x^2)*u(x)\n" 13 | << "\t-pi < x < pi\n" 14 | << "\tu(-pi) = 1\tu(pi) = 1\n" 15 | << "----------------------------------------------------\n" 16 | << "first we re-write the problem as a system of first order ODEs:\n" 17 | << "\tu'(x) = v(x)\n\tv'(x) = 2(1-x^2)*u(x)\n" 18 | << "\twith the same set of initial conditions.\n" 19 | << "----------------------------------------------------\n" 20 | << "we need to provide the solver an initial guess, so we will simply use\n\tu(x) = 1\n\tv(x) = 0\n" 21 | << "----------------------------------------------------\n" 22 | << "Using method:" << std::endl 23 | << "\t('cheb' w/o quotes) Chebyshev spectral method." << std::endl 24 | << "\t('lobatto' w/o quotes) Lobatto IIIa two point explicit method." << std::endl 25 | << "\t(int > 1) Finite difference with user specified order." << std::endl; 26 | 27 | std::string choice; 28 | int method = -1; 29 | while (true) { 30 | std::cout << "your choice: "; 31 | std::cin >> choice; 32 | if (choice == "cheb") { 33 | method = -1; 34 | break; 35 | } else if (choice == "lobatto") { 36 | method = -2; 37 | break; 38 | } else { 39 | try { 40 | method = std::stoi(choice); 41 | if (method < 2) std::cout << "\nrequire FD integer order method > 1.\n"; 42 | else break; 43 | } catch(const std::exception& e) { 44 | std::cout << "\ninvalid choice.\n"; 45 | } 46 | } 47 | } 48 | 49 | int N; 50 | while (true) { 51 | std::cout << "initial number of grid points: "; 52 | std::cin >> N; 53 | if (N > 2) break; 54 | else std::cout << "\nnumber of points must be >= 3, and practically speaking should be large, consider 32, 64, 100 (powers of two are most efficient for cheb but not necessary).\n"; 55 | } 56 | 57 | auto f = [](double x, const arma::vec& u) -> arma::vec { 58 | arma::vec up(2); 59 | up(0) = u(1); 60 | up(1) = 2*(1 - x*x)*u(0); 61 | return up; 62 | }; 63 | 64 | auto bc = [](const arma::vec& uL, const arma::vec& uR) -> arma::vec { 65 | arma::vec v(2); 66 | v(0) = uL(0) - 1; // solution fixed as 1 at end points 67 | v(1) = uR(0) - 1; 68 | return v; 69 | }; 70 | 71 | auto guess = [](double x) -> arma::vec { 72 | arma::vec y(2); 73 | y(0) = 1; 74 | y(1) = 0; 75 | return y; 76 | }; 77 | 78 | arma::vec x = arma::linspace(a,b,N); 79 | arma::mat U(2,N); 80 | for (int i=0; i < N; ++i) U.col(i) = guess(x(i)); // set U to the initial guess 81 | 82 | arma::vec x_sol; 83 | arma::mat U_sol; 84 | arma::vec xx = arma::linspace(a,b,500); 85 | arma::mat uu; 86 | int nit; 87 | std::string flag; 88 | 89 | if (method == -1) { 90 | numerics::ode::BVPCheb sol(N); 91 | sol.ode_solve(f,bc,x,U); 92 | x_sol = sol.x; 93 | U_sol = sol.u; 94 | uu = sol(xx); 95 | nit = sol.num_iter; 96 | flag = sol.get_exit_flag(); 97 | } else if (method == -2) { 98 | numerics::ode::BVP3a sol; 99 | sol.ode_solve(f,bc,x,U); 100 | x_sol = sol.x; 101 | U_sol = sol.u; 102 | uu = sol(xx); 103 | nit = sol.num_iter; 104 | flag = sol.get_exit_flag(); 105 | } else { 106 | numerics::ode::BVPk sol(method); 107 | sol.ode_solve(f,bc,x,U); 108 | x_sol = sol.x; 109 | U_sol = sol.u; 110 | uu = sol(xx); 111 | nit = sol.num_iter; 112 | flag = sol.get_exit_flag(); 113 | } 114 | 115 | std::cout << "Number of nonlinear iterations needed by solver: " << nit << "\n" 116 | << "exit flag: " << flag << "\n"; 117 | dvec x1 = arma::conv_to::from(x_sol); 118 | dvec u1 = arma::conv_to::from(U_sol.row(0)); 119 | dvec v1 = arma::conv_to::from(U_sol.row(1)); 120 | dvec x2 = arma::conv_to::from(xx); 121 | dvec u2 = arma::conv_to::from(uu.row(0)); 122 | dvec v2 = arma::conv_to::from(uu.row(1)); 123 | 124 | matplotlibcpp::subplot(2,1,1); 125 | std::map ls = {{"marker","o"},{"label","u(x)"},{"ls","none"},{"color","purple"}}; 126 | matplotlibcpp::plot(x1,u1,ls); 127 | matplotlibcpp::plot(x2,u2,"-b"); 128 | matplotlibcpp::legend(); 129 | matplotlibcpp::subplot(2,1,2); 130 | ls["label"] = "v(x)"; ls["color"] = "orange"; ls["marker"] = "o"; ls["ls"] = "none"; 131 | matplotlibcpp::plot(x1,v1,ls); 132 | matplotlibcpp::plot(x2,v2,"-r"); 133 | matplotlibcpp::legend(); 134 | matplotlibcpp::show(); 135 | 136 | return 0; 137 | } -------------------------------------------------------------------------------- /src/optimization/trust_newton.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::optimization::TrustNewton::_initialize(const arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 4 | _F = f(x); 5 | _delta = 0.1*std::max(1.0, arma::norm(x,"inf")); 6 | } 7 | 8 | bool numerics::optimization::TrustNewton::_step(arma::vec& dx, arma::vec& F1, const arma::vec& x, const VecFunc& f, const MatFunc* jacobian) { 9 | arma::vec g, p, Jg; 10 | bool success; 11 | 12 | double delta_max = 10.0; 13 | 14 | if (jacobian == nullptr) { 15 | auto fnorm = [&f](const arma::vec& z) -> double {arma::vec fz = f(z); return 0.5*arma::dot(fz,fz);}; 16 | g = grad(fnorm, x); 17 | 18 | double h = 1e-6*std::max(1.0, arma::norm(x)); 19 | VecFunc JacMult = [this, h, &F1, &f, &x](const arma::vec& v) -> arma::vec { 20 | double C = std::max(1.0, arma::norm(v)) / h; 21 | return C * (f(x + v/C) - _F); 22 | }; 23 | success = gmres(p, JacMult, _F, _xtol); 24 | Jg = JacMult(g); 25 | } else { 26 | _J = (*jacobian)(x); 27 | g = _J.t() * _F; 28 | success = arma::solve(p, _J, _F); 29 | Jg = _J * g; 30 | } 31 | 32 | if (not success) return false; 33 | 34 | double ff = arma::dot(_F,_F); 35 | double gg = arma::dot(g,g); 36 | double Jg2 = arma::dot(Jg,Jg); 37 | double pp = arma::dot(p,p); 38 | 39 | arma::mat A = { 40 | {ff,gg}, 41 | {gg,Jg2} 42 | }; 43 | arma::mat B = { 44 | {pp,ff}, 45 | {ff,gg} 46 | }; 47 | 48 | arma::vec r = {ff, gg}; 49 | 50 | while (true) { 51 | if (arma::norm(p) < _delta) { 52 | dx = -p; 53 | _fh = -0.5*arma::dot(g,p); 54 | } 55 | else { 56 | arma::vec u; 57 | auto phi = [this,&u,&A,&B,&r,&p,&g](double l) -> double { 58 | u = arma::solve(A + (l*l)*B, -r); 59 | double unorm = arma::norm(u(0)*p + u(1)*g); 60 | return 1/unorm - 1/_delta; 61 | }; 62 | double l = newton_1d(phi, 1.0, 1e-4); 63 | l *= l; 64 | dx = u(0)*p + u(1)*g; 65 | _fh = arma::dot(u, 0.5*A*u + r); 66 | } 67 | 68 | F1 = f(x + dx); 69 | double f1 = arma::dot(F1,F1); 70 | 71 | if (f1 < ff) { 72 | double rho = 0.5*std::abs((ff - f1) / _fh); 73 | if (rho < 0.25) { 74 | _delta = 0.25*arma::norm(dx); 75 | } else if ((rho > 0.75) and (std::abs(arma::norm(dx)-_delta) < 1e-8)) { 76 | // approximation is good and dx is on the boundary of the trust region 77 | _delta = std::min(2*_delta, delta_max*arma::norm(x)); 78 | } 79 | break; 80 | } else { 81 | _delta /= 2.0; 82 | } 83 | if (_delta < _xtol) break; 84 | } 85 | return success; 86 | } 87 | 88 | void numerics::optimization::TrustMin::_initialize(const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 89 | _g = df(x); 90 | _f0 = f(x); 91 | _delta = 0.1*std::max(1.0, arma::norm(x,"inf")); 92 | } 93 | 94 | bool numerics::optimization::TrustMin::_step(arma::vec& dx, arma::vec& g1, const arma::vec& x, const dFunc& f, const VecFunc& df, const MatFunc* hessian) { 95 | bool success; 96 | double delta_max = 10.0; 97 | 98 | arma::vec p, Hg; 99 | 100 | if (hessian == nullptr) { 101 | double h = 1e-6*std::max(1.0, arma::norm(x)); 102 | VecFunc HessMult = [this,&df,&x,h](const arma::vec& v) -> arma::vec { 103 | double C = std::max(1.0, arma::norm(v)) / h; 104 | return C * (df(x + v/C) - _g); 105 | }; 106 | Hg = HessMult(_g); 107 | success = pcg(p, HessMult, -_g, _xtol); 108 | if (not success) success = gmres(p, HessMult, _g, _xtol); 109 | } else { 110 | arma::mat H = (*hessian)(x); 111 | Hg = H*_g; 112 | success = arma::solve(p, H, _g); 113 | } 114 | 115 | if (not success) return false; 116 | 117 | double gp = arma::dot(_g,p); 118 | double gg = arma::dot(_g,_g); 119 | double pp = arma::dot(p,p); 120 | double gHg = arma::dot(_g,Hg); 121 | 122 | arma::mat A = { 123 | {gp, gg}, 124 | {gg, gHg} 125 | }; 126 | 127 | arma::mat B = { 128 | {pp, gp}, 129 | {gp, gg} 130 | }; 131 | 132 | arma::vec r = {gp, gg}; 133 | 134 | while (true) { 135 | if (arma::norm(p) < _delta) { 136 | dx = -p; 137 | _fh = -0.5*arma::dot(_g,p); 138 | } 139 | else { 140 | arma::vec u; 141 | auto phi = [this,&u,&A,&B,&r,&p](double l) -> double { 142 | l = l*l; 143 | u = arma::solve(A + l*B, -r); 144 | double unorm = arma::norm(u(0)*p + u(1)*_g); 145 | return 1/unorm - 1/_delta; 146 | }; 147 | double l = newton_1d(phi, 1.0, 1e-4); 148 | l = l*l; 149 | dx = u(0)*p + u(1)*_g; 150 | _fh = arma::dot(u, 0.5*A*u + r); 151 | } 152 | 153 | double f1 = f(x+dx); 154 | 155 | if (f1 < _f0) { 156 | double rho = std::abs((_f0 - f1) / _fh); 157 | if (rho < 0.25) { 158 | _delta = 0.25*arma::norm(dx); 159 | } else if ((rho > 0.75) and (std::abs(arma::norm(dx)-_delta) < 1e-8)) { 160 | // approximation is good and dx is on the boundary of the trust region 161 | _delta = std::min(2*_delta, delta_max*arma::norm(x)); 162 | } 163 | _f0 = f1; 164 | g1 = df(x+dx); 165 | break; 166 | } else { 167 | _delta /= 2.0; 168 | } 169 | 170 | if (_delta < _xtol) { 171 | dx = arma::zeros(arma::size(x)); 172 | break; 173 | } 174 | } 175 | return true; 176 | } -------------------------------------------------------------------------------- /src/ode/ivp.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | double numerics::ode::InitialValueProblem::_event_handle(double t1, const arma::vec& u1) { 4 | int num_events = events.size(); 5 | 6 | const arma::vec& u = _prev_u.back(); 7 | const double& t = _prev_t.back(); 8 | double alpha = 1.0; 9 | 10 | for (int i=0; i < num_events; ++i) { 11 | const std::function& event = events.at(i); 12 | double g = event(t, u); 13 | double g1 = event(t1, u1); 14 | if (arma::sign(g) != arma::sign(g1)) { // event occurred 15 | if (std::abs(g1) < _event_tol) { // stopping event possibly reached 16 | if ((g1 < 0) and _dec.at(i)) { // {+} -> {-} 17 | _stopping_event = i; 18 | return 0.0; 19 | } else if ((g1 > 0) and _inc.at(i)) { // {-} -> {+} 20 | _stopping_event = i; 21 | return 0.0; 22 | } // if neither, then false alarm 23 | } else { // secant method for reducing step size. 24 | alpha = std::min(alpha, 0.95*std::abs(g1) / (std::abs(g1) + std::abs(g))); 25 | } 26 | } 27 | } 28 | return alpha; 29 | } 30 | 31 | void numerics::ode::InitialValueProblem::_check_range(double t0, arma::vec& T) { 32 | if (T.is_empty()) throw std::runtime_error("T array is empty, must include atleast one value."); 33 | T = arma::sort(T); 34 | if (T(0) <= t0) throw std::runtime_error("T[0] (=" + std::to_string(T(0)) + ") >= t0 (=" + std::to_string(t0) + ")."); 35 | } 36 | 37 | void numerics::ode::InitialValueProblem::_update_solution(double& t1, arma::vec& u1, arma::vec& f1, bool full, bool is_grid_val) { 38 | _prev_t.push(t1); _prev_u.push(u1); _prev_f.push(f1); 39 | if (full or is_grid_val) { 40 | _t.push_back(std::move(t1)); 41 | _U.push_back(std::move(u1)); 42 | } 43 | } 44 | 45 | void numerics::ode::InitialValueProblem::_solve(const odefunc& f, const odejacobian* J, double t0, arma::vec T, const arma::vec& U0, bool full) { 46 | _check_range(t0,T); 47 | double k = _initial_step_size(); 48 | 49 | _t.clear(); _U.clear(); 50 | _t.push_back(t0); 51 | _U.push_back(U0); 52 | 53 | _prev_t.push(t0); 54 | _prev_u.push(U0); 55 | _prev_f.push( f(t0,U0) ); 56 | double t1; arma::vec u1, f1; 57 | 58 | for (const double& tf : T) { 59 | while (_prev_t.back() < tf) { 60 | double tt = _prev_t.back(); 61 | 62 | if (std::abs((tf - tt) - k) < 1e-10) k = tf - tt; 63 | else k = std::min(k, tf - tt); 64 | 65 | while (true) { 66 | double k1 = _step(k, t1, u1, f1, f, J); 67 | double alpha = _event_handle(t1, u1); 68 | if ((0 < alpha) and (alpha < 1)) k *= alpha; 69 | else { 70 | bool is_grid_val = (std::abs(tf - tt) < 1e-10); 71 | _update_solution(t1, u1, f1, full, is_grid_val); 72 | k = k1; 73 | 74 | if (alpha == 0) return; 75 | break; 76 | } 77 | } 78 | } 79 | } 80 | } 81 | 82 | numerics::ode::InitialValueProblem::InitialValueProblem(double event_tol, int n_step) : _prev_u(n_step), _prev_f(n_step), _prev_t(n_step), stopping_event(_stopping_event), t(_t), U(_U) { 83 | _stopping_event = -1; 84 | if (event_tol <= 0) throw std::invalid_argument("event_tol (=" + std::to_string(event_tol) + ") must be positive."); 85 | _event_tol = event_tol; 86 | } 87 | 88 | void numerics::ode::InitialValueProblem::add_stopping_event(const std::function& event, const std::string& dir) { 89 | if (dir == "all") { 90 | events.push_back(event); 91 | _inc.push_back(true); 92 | _dec.push_back(true); 93 | } else if (dir == "inc") { 94 | events.push_back(event); 95 | _inc.push_back(true); 96 | _dec.push_back(false); 97 | } else if (dir == "dec") { 98 | events.push_back(event); 99 | _inc.push_back(false); 100 | _dec.push_back(true); 101 | } else throw std::invalid_argument("dir (='" + dir + "') must be one of {'all','inc','dec'}."); 102 | } 103 | 104 | void numerics::ode::InitialValueProblem::solve_ivp(const odefunc& f, double t0, double tf, const arma::vec& U0, bool full) { 105 | if (U0.n_elem < 200) solver = std::make_unique(_solver_xtol, _solver_ftol, _solver_miter, false); 106 | else solver = std::make_unique(_solver_xtol, _solver_ftol, _solver_miter, false); 107 | _solve(f, nullptr, t0, arma::vec({tf}), U0, full); 108 | } 109 | 110 | void numerics::ode::InitialValueProblem::solve_ivp(const odefunc& f, double t0, arma::vec T, const arma::vec& U0, bool full) { 111 | if (U0.n_elem < 200) solver = std::make_unique(_solver_xtol, _solver_ftol, _solver_miter, false); 112 | else solver = std::make_unique(_solver_xtol, _solver_ftol, _solver_miter, false); 113 | _solve(f, nullptr, t0, T, U0, full); 114 | } 115 | 116 | void numerics::ode::InitialValueProblem::solve_ivp(const odefunc& f, const odejacobian& J, double t0, double tf, const arma::vec& U0, bool full) { 117 | solver = std::make_unique(_solver_xtol, _solver_ftol, _solver_miter, false); 118 | _solve(f, &J, t0, arma::vec({tf}), U0, full); 119 | } 120 | 121 | void numerics::ode::InitialValueProblem::solve_ivp(const odefunc& f, const odejacobian& J, double t0, arma::vec T, const arma::vec& U0, bool full) { 122 | solver = std::make_unique(_solver_xtol, _solver_ftol, _solver_miter, false); 123 | _solve(f, &J, t0, T, U0, full); 124 | } 125 | 126 | void numerics::ode::InitialValueProblem::as_mat(arma::vec& tt, arma::mat& uu) { 127 | uint n = U.size(); 128 | uint m = U.front().n_elem; 129 | uu.set_size(m,n); 130 | tt.set_size(n); 131 | for (uint i=0; i < n; ++i) { 132 | tt(i) = t.at(i); 133 | uu.col(i) = U.at(i); 134 | } 135 | } -------------------------------------------------------------------------------- /src/neural_network/optimize.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void numerics::neuralnet::SGD::_check_alpha(const double a) { 4 | if (a <= 0) { 5 | throw std::domain_error("adam requires: alpha (=" + std::to_string(a) + ") > 0."); 6 | } 7 | } 8 | 9 | void numerics::neuralnet::SGD::_initialize_averages(const std::vector& layers) { 10 | _avg_weights.clear(); 11 | _avg_bias.clear(); 12 | for (const Layer& L : layers) { 13 | if (L._trainable_weights) _avg_weights.push_back(L.weights); 14 | else _avg_weights.push_back(arma::mat()); 15 | 16 | if (L._trainable_bias) _avg_bias.push_back(L.bias); 17 | else _avg_bias.push_back(arma::mat()); 18 | } 19 | _initialized_averages = true; 20 | } 21 | 22 | void numerics::neuralnet::SGD::_update_averages(const std::vector& layers) { 23 | if (_t >= _avg_start) { 24 | if (not _initialized_averages) _initialize_averages(layers); 25 | else { 26 | u_long N = _t - _avg_start; 27 | for (u_long i=0; i < layers.size(); ++i) { 28 | if (layers.at(i)._trainable_weights) _avg_weights.at(i) = (_avg_weights.at(i) * (N-1) + layers.at(i).weights) / N; 29 | if (layers.at(i)._trainable_bias) _avg_bias.at(i) = (_avg_bias.at(i) * (N-1) + layers.at(i).bias) / N; 30 | } 31 | } 32 | } 33 | } 34 | 35 | void numerics::neuralnet::SGD::set_alpha(const double a) { 36 | _check_alpha(a); 37 | _alpha = a; 38 | } 39 | 40 | std::unique_ptr numerics::neuralnet::SGD::clone() const { 41 | return std::make_unique(*this); 42 | } 43 | 44 | void numerics::neuralnet::SGD::set_averaging(u_long average) { 45 | if (average == 0) _averaging = false; 46 | else { 47 | _averaging = true; 48 | _avg_start = average; 49 | } 50 | } 51 | 52 | void numerics::neuralnet::SGD::restart() { 53 | _t = 0; 54 | if (_averaging) { 55 | _initialized_averages = false; 56 | } 57 | } 58 | 59 | void numerics::neuralnet::SGD::step(std::vector& layers) { 60 | _t++; 61 | for (Layer& L : layers) { 62 | if (L._trainable_weights) L._weights += _alpha * L._dW; 63 | if (L._trainable_bias) L._bias += _alpha * L._db; 64 | } 65 | if (_averaging) _update_averages(layers); 66 | } 67 | 68 | void numerics::neuralnet::SGD::finalize(std::vector& layers) { 69 | if (_averaging) { 70 | for (u_long i=0; i < layers.size(); ++i) { 71 | if (layers.at(i)._trainable_weights) layers.at(i)._weights = _avg_weights.at(i); 72 | if (layers.at(i)._trainable_bias) layers.at(i)._bias = _avg_bias.at(i); 73 | } 74 | } 75 | } 76 | 77 | void numerics::neuralnet::Adam::_check_beta1(double b1) { 78 | if ((b1 <= 0) or (b1 >= 1)) { 79 | throw std::domain_error("adam requires: 1 > beta1 (=" + std::to_string(b1) + ") > 0"); 80 | } 81 | } 82 | 83 | void numerics::neuralnet::Adam::_check_beta2(double b2) { 84 | if ((b2 <= 0) or (b2 >= 1)) { 85 | throw std::domain_error("adam requires: 1 > beta2 (=" + std::to_string(b2) + ") > 0"); 86 | } 87 | } 88 | 89 | void numerics::neuralnet::Adam::_check_epsilon(const double eps) { 90 | if (eps < 0) { 91 | throw std::domain_error("adam requires: epsilon (=" + std::to_string(eps) + ") > 0"); 92 | } 93 | } 94 | 95 | void numerics::neuralnet::Adam::_initialize_moments(const std::vector& layers) { 96 | _mW.clear(); 97 | _mb.clear(); 98 | _vW.clear(); 99 | _vb.clear(); 100 | for (const Layer& L : layers) { 101 | if (L._trainable_weights) { 102 | _mW.push_back(arma::zeros(arma::size(L.weights))); 103 | _vW.push_back(arma::zeros(arma::size(L.weights))); 104 | } else { 105 | _mW.push_back(arma::mat()); // push empty 106 | _vW.push_back(arma::mat()); 107 | } 108 | 109 | if (L._trainable_bias) { 110 | _mb.push_back(arma::zeros(arma::size(L.bias))); 111 | _vb.push_back(arma::zeros(arma::size(L.bias))); 112 | } else { 113 | _mb.push_back(arma::mat()); 114 | _vb.push_back(arma::mat()); 115 | } 116 | } 117 | _initialized_moments = true; 118 | } 119 | 120 | std::unique_ptr numerics::neuralnet::Adam::clone() const { 121 | return std::make_unique(*this); 122 | } 123 | 124 | void numerics::neuralnet::Adam::set_beta1(const double b1) { 125 | _check_beta1(b1); 126 | _beta1 = b1; 127 | } 128 | 129 | void numerics::neuralnet::Adam::set_beta2(const double b2) { 130 | _check_beta2(b2); 131 | _beta2 = b2; 132 | } 133 | 134 | void numerics::neuralnet::Adam::set_epsilon(const double eps) { 135 | _check_epsilon(eps); 136 | _epsilon = eps; 137 | } 138 | 139 | void numerics::neuralnet::Adam::restart() { 140 | SGD::restart(); 141 | _initialized_moments = false; 142 | } 143 | 144 | void numerics::neuralnet::Adam::step(std::vector& layers) { 145 | if (not _initialized_moments) _initialize_moments(layers); 146 | _t++; 147 | for (u_long i=0; i < layers.size(); ++i) { 148 | if (layers.at(i)._trainable_weights) { 149 | _mW.at(i) = _beta1*_mW.at(i) + (1 - _beta1)*layers.at(i)._dW; 150 | _vW.at(i) = _beta2*_vW.at(i) + (1 - _beta2)*arma::square(layers.at(i)._dW); 151 | 152 | arma::mat mWhat = _mW.at(i) / (1 - std::pow(_beta1, _t)); 153 | arma::mat vWhat = _vW.at(i) / (1 - std::pow(_beta2, _t)); 154 | 155 | layers.at(i)._weights += _alpha * mWhat / (arma::sqrt(vWhat) + _epsilon); 156 | } 157 | if (layers.at(i)._trainable_bias) { 158 | _mb.at(i) = _beta1*_mb.at(i) + (1 - _beta1)*layers.at(i)._db; 159 | _vb.at(i) = _beta2*_vb.at(i) + (1 - _beta2)*arma::square(layers.at(i)._db); 160 | 161 | arma::mat mbhat = _mb.at(i) / (1 - std::pow(_beta1, _t)); 162 | arma::mat vbhat = _vb.at(i) / (1 - std::pow(_beta2, _t)); 163 | 164 | layers.at(i)._bias += _alpha * mbhat / (arma::sqrt(vbhat) + _epsilon); 165 | } 166 | } 167 | if (_averaging) _update_averages(layers); 168 | } -------------------------------------------------------------------------------- /src/ode/diffmat.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | arma::rowvec _diffvec_unsafe(const arma::vec& x, double x0, u_int k) { 4 | int n = x.n_elem; 5 | 6 | double c1 = 1, c2, c3, c4 = x(0) - x0, c5; 7 | arma::mat C = arma::zeros(n,k+1); 8 | C(0,0) = 1; 9 | for (int i=1; i < n; ++i) { 10 | int mn = std::min(i,(int)k); 11 | c2 = 1; 12 | c5 = c4; 13 | c4 = x(i) - x0; 14 | for (int j=0; j < i; ++j) { 15 | c3 = x(i) - x(j); 16 | c2 *= c3; 17 | if (j==i-1) { 18 | for (int l=mn; l >= 1; --l) C(i,l) = c1*(l * C(i-1,l-1) - c5*C(i-1,l))/c2; 19 | C(i,0) = -c1*c5*C(i-1,0)/c2; 20 | } 21 | for (int l=mn; l >= 1; --l) C(j,l) = (c4*C(j,l) - l*C(j,l-1))/c3; 22 | C(j,0) = c4*C(j,0)/c3; 23 | } 24 | c1 = c2; 25 | } 26 | return C.col(k).t(); 27 | } 28 | 29 | /* diffvec(x, x0, k) : returns rowvec w such the w * f(x) produces an approximation of the k^th-derivative f^{k} at x0. 30 | * source : Fornberg, Bengt "Calculation of weights in finite difference formulas". SIAM, 1998 31 | * --- x : points (near) x0 to form the interpolation over. 32 | * --- x0 : the point to produce the approximation for. 33 | * --- k : the order of the derivative (k < x.n_elem), e.g. k=1 is the first derivative */ 34 | arma::rowvec numerics::ode::diffvec(const arma::vec& x, double x0, uint k) { 35 | int n = x.n_elem; 36 | if (k >= n) { 37 | throw std::invalid_argument( 38 | "diffvec() error: in order to approximate a " + std::to_string(k) + "-order derivative, at least " 39 | + std::to_string(k+1) + " x-values are needed but only " + std::to_string(n) 40 | + " were provided." 41 | ); 42 | } 43 | 44 | if (not x.is_sorted()) { 45 | throw std::runtime_error("diffvec() error: require sorted x values"); 46 | } 47 | 48 | return _diffvec_unsafe(x,x0,k); 49 | } 50 | 51 | 52 | /* diffmat(x, k, bdw) : produces the differentiation matrix of nonuniformly spaced data. 53 | * --- x : values to evaluate the operator for. 54 | * --- k : the order of the derivative (k < x.n_elem), e.g. k=1 is the first derivative. 55 | * --- npt : number of points to use in approximation, bdw > 1, if npt = even then a symmetric differencing will be used, but when npt = odd then a forward biased differencing will be used for npt > 3. The truncation error should be ~ O(h^{npt-1}), where h is the maximum spacing between consecutive x-values. */ 56 | void numerics::ode::diffmat(arma::mat& D, const arma::vec& x, uint k, uint npt) { 57 | int n = x.n_elem; 58 | if (k >= n) { 59 | throw std::invalid_argument( 60 | "diffmat() error: in order to approximate a " + std::to_string(k) + "-order derivative, at least " 61 | + std::to_string(k+1) + " x-values are needed but only " + std::to_string(n) 62 | + " were provided." 63 | ); 64 | } 65 | 66 | if (npt < 2) { 67 | throw std::invalid_argument("diffmat() error: require npt (=" + std::to_string(npt) + ") > 1"); 68 | } 69 | 70 | arma::uvec ind = arma::sort_index(x); 71 | arma::vec t = x(ind); 72 | 73 | bool center = (npt%2 != 0); 74 | D = arma::zeros(n,n); 75 | for (int i=0; i < n; ++i) { 76 | int j = (center) ? (i - npt/2) : (i - 1); 77 | if (j < 0) j = 0; 78 | if (j + npt >= n-1) j = (n-1) - npt; 79 | D.row(i).cols(j,j+npt-1) = diffvec(t.rows(j,j+npt-1), t(i), k); 80 | } 81 | D = D.cols(ind); 82 | D = D.rows(ind); 83 | } 84 | 85 | /* diffmat(x, k, bdw) : produces the differentiation matrix of nonuniformly spaced data. 86 | * --- x : values to evaluate the operator for. (x must be sorted) 87 | * --- k : the order of the derivative (k < x.n_elem), e.g. k=1 is the first derivative. 88 | * --- bdw : number of points -1 to use in approximation, bdw > 1, if bdw = even then a symmetric differencing will be prefered when possible and if bdw = odd then a backwards differencing will be prefered. The truncation error should be ~ O(h^bdw), where h is the maximum spacing between consecutive x-values. */ 89 | void numerics::ode::diffmat(arma::sp_mat& D, const arma::vec& x, uint k, uint bdw) { 90 | int n = x.n_elem; 91 | if (k >= n) { 92 | std::cerr << "diffmat() error: in order to approximate a " << k << "-order derivative, at least " 93 | << k+1 << " x-values are needed but only " << n << " were provided." << std::endl; 94 | return; 95 | } 96 | 97 | bool center = (bdw%2 == 0); 98 | D = arma::zeros(n,n); 99 | for (int i=0; i < n; ++i) { 100 | int j = i - bdw/2; 101 | if (!center) j--; 102 | if (j < 0) j = 0; 103 | if (j + bdw + 1 >= n) j = (n-1) - bdw; 104 | D.row(i).cols(j,j+bdw) = diffvec(x.rows(j,j+bdw), x(i), k); 105 | } 106 | } 107 | 108 | /* diffmat4(D, x, L, R, m) : returns the general 4th order differentiation matrix of uniformly spaced data. 109 | * --- D : to store mat in. 110 | * --- x : x values overwhich to calc diff mat 111 | * --- L,R : limits on x. 112 | * --- m : number of points. */ 113 | void numerics::ode::diffmat4(arma::mat& D, arma::vec& x, double L, double R, uint m) { 114 | m = m-1; 115 | x = arma::regspace(0,m)/m; // regspace on [0,1] with m points 116 | double h = (R - L)/m; // spacing 117 | x = (R - L) * x + L; // transformation from [0,1] -> [L,R] 118 | 119 | D = arma::zeros(m+1,m+1); 120 | D.diag(-2) += 1; 121 | D.diag(-1) += -8; 122 | D.diag(1) += 8; 123 | D.diag(2) += -1; 124 | 125 | D.row(0).head(5) = arma::rowvec({-25, 48, -36, 16, -3}); 126 | D.row(1).head(5) = arma::rowvec({-3, -10, 18, -6, 1}); 127 | D.row(m-1).tail(5) = arma::rowvec({-1, 6, -18, 10, 3}); 128 | D.row(m).tail(5) = arma::rowvec({3, -16, 36, -48, 25}); 129 | D /= 12*h; 130 | } 131 | 132 | /* diffmat2(D, x, L, R, m) : returns the general 2nd order differentiation matrix. 133 | * --- D : to store mat in. 134 | * --- x : x values overwhich to calc diff mat 135 | * --- L,R : limits on x. 136 | * --- m : number of points. */ 137 | void numerics::ode::diffmat2(arma::mat& D, arma::vec& x, double L, double R, uint m) { 138 | m = m-1; 139 | x = arma::regspace(0,m)/m; // regspace on [0,1] with m+1 points 140 | double h = (R - L)/m; // spacing 141 | x = (R - L) * x + L; // transformation from [0,1] -> [L,R] 142 | 143 | D = arma::zeros(m+1,m+1); 144 | D.diag(-1) += -1; 145 | D.diag(1) += 1; 146 | 147 | D.row(0).head(3) = arma::rowvec({-3, 4, -1}); 148 | D.row(m).tail(3) = arma::rowvec({1, -4, 3}); 149 | D /= 2*h; 150 | } -------------------------------------------------------------------------------- /src/optimization/fzero.cpp: -------------------------------------------------------------------------------- 1 | #include "numerics.hpp" 2 | 3 | double numerics::optimization::newton_1d(const std::function& f, const std::function& df, double x, double tol) { 4 | if (tol <= 0) throw std::invalid_argument("error bound should be strictly positive, but tol=" + std::to_string(tol)); 5 | int max_iter = 100; 6 | 7 | double s=tol/2; 8 | u_long k = 0; 9 | double fx, fp; 10 | do { 11 | if (k >= max_iter) { // too many iterations 12 | std::cerr << "newton_1d() failed: too many iterations needed to converge." << std::endl 13 | << "returing current best estimate." 14 | << "!!!---not necessarily a good estimate---!!!" << std::endl 15 | << "|f(x)| = " << std::abs(f(x)) << " > tolerance" << std::endl << std::endl; 16 | return x; 17 | } 18 | fx = f(x); 19 | fp = df(x); 20 | if (std::abs(fp) < tol/2) s *= -fx/(fx - f(x-s)); 21 | else s = -fx/fp; 22 | x += s; 23 | k++; 24 | } while ((std::abs(fx) > tol) && (std::abs(s) > tol)); 25 | return x; 26 | } 27 | 28 | double numerics::optimization::newton_1d(const std::function& f, double x, double tol) { 29 | auto df = [&](double u) -> double { 30 | return deriv(f, u, tol/2, true, 2); 31 | }; 32 | return newton_1d(f, df, x, tol); 33 | } 34 | 35 | double numerics::optimization::secant(const std::function& f, double a, double b, double tol) { 36 | int max_iter = 100; 37 | 38 | double fa = f(a), fb = f(b); 39 | int k = 2; 40 | if (std::abs(fa) < tol) return a; 41 | if (std::abs(fb) < tol) return b; 42 | 43 | while (true) { 44 | if (k >= max_iter) { // too many iterations 45 | std::cerr << "secant() error: could not converge within " << max_iter << " function evaluations." << std::endl 46 | << "\treturing current best estimate." 47 | << "!!!---not necessarily a good estimate---!!!" << std::endl 48 | << "|f(x)| = " << std::min(std::abs(fa), std::abs(fb)) << " > tolerance" << std::endl << std::endl; 49 | return (std::abs(fa) < std::abs(fb)) ? a : b; 50 | } 51 | double c; 52 | 53 | double rel = std::max( std::max(std::abs(fb), std::abs(fa)), 1.0); 54 | if (std::abs(fa - fb) < 1e-10*rel) c = (a + b) / 2; 55 | else c = b - fb*(b-a)/(fb-fa); 56 | 57 | double fc = f(c); k++; 58 | 59 | if (std::abs(fc) < tol) return c; 60 | 61 | if (fa*fb < 0) { 62 | if (fb*fc < 0) { 63 | a = c; fa = fc; 64 | } else { 65 | b = c; fb = fc; 66 | } 67 | } else { 68 | if (std::abs(fa) < std::abs(fb)) { 69 | b = c; fb = fc; 70 | } else { 71 | a = c; fa = fc; 72 | } 73 | } 74 | 75 | rel = std::max( std::max(std::abs(a), std::abs(b)), 1.0); 76 | if (std::abs(a - b) < rel*tol) return (a+b)/2; 77 | } 78 | } 79 | 80 | double numerics::optimization::bisect(const std::function& f, double a, double b, double tol) { 81 | tol = std::abs(tol); 82 | long long k = 2; 83 | 84 | double fa = f(a), fb = f(b); 85 | 86 | if (std::abs(fa) < tol ) { 87 | return a; 88 | } else if (std::abs(fb) < tol ) { 89 | return b; 90 | } 91 | 92 | if (fa*fb > 0) { // bracketing error 93 | std::cerr << "bisect() error: provided points do not bracket a simple root." << std::endl; 94 | return NAN; 95 | } 96 | 97 | double c, fc; 98 | 99 | do { 100 | c = (a+b)/2; 101 | fc = f(c); k++; 102 | if (std::abs(fc) < tol) break; 103 | if (fc*fa < 0) { 104 | b = c; 105 | fb = fc; 106 | } else { 107 | a = c; 108 | fa = fc; 109 | } 110 | } while (std::abs(fc) > tol && std::abs(b-a) > tol); 111 | 112 | return (a + b)/2; 113 | } 114 | 115 | double numerics::optimization::fzero(const std::function& f, double a, double b, double tol) { 116 | int max_iter = std::min(std::pow(std::log2((b-a)/tol)+1,2), 1.0e2); // will nearly never happen 117 | 118 | double c, d, e, fa, fb, fc, m=0, s=0, p=0, q=0, r=0, t, eps = std::numeric_limits::epsilon(); 119 | int k=0; 120 | fa = f(a); k++; 121 | fb = f(b); k++; 122 | if (std::abs(fa) == 0) return a; 123 | if (std::abs(fb) == 0) return b; 124 | 125 | if (fa*fb > 0) { 126 | std::cerr << "fzero() error: provided points do not bracket a simple root." << std::endl; 127 | return NAN; 128 | } 129 | 130 | c = a; fc = fa; d = b-a; e = d; 131 | 132 | while (true) { 133 | if (std::abs(fc) < std::abs(fb)) { 134 | a = b; b = c; c = a; 135 | fa = fb; fb = fc; fc = fa; 136 | } 137 | m = (c-b)/2; 138 | t = 2*std::abs(b)*eps + tol; 139 | if (std::abs(m) < t || fb == 0) break; // convergence criteria 140 | if (k >= max_iter) { 141 | std::cerr << "fzero() error: could not converge within " << max_iter << " function evaluations (the estimated neccessary ammount).\n" 142 | << "returing current best estimate.\n" 143 | << "!!!---not necessarily a good estimate---!!!\n" 144 | << "|dx| = " << std::abs(m) << " > " << tol << "\n"; 145 | break; 146 | } 147 | 148 | if (std::abs(e) < t || std::abs(fa) < std::abs(fb)) { // bisection 149 | d = m; e = m; 150 | } else { 151 | s = fb/fa; 152 | if (a == c) { // secant 153 | p = 2*m*s; 154 | q = 1 - s; 155 | } else { // inverse quadratic 156 | q = fa/fc; 157 | r = fb/fc; 158 | p = s*(2*m*q*(q-r)-(b-a)*(r-1)); 159 | q = (q-1)*(r-1)*(s-1); 160 | } 161 | 162 | if (p > 0) q = -q; 163 | else p = -p; 164 | 165 | s = e; e = d; 166 | 167 | if (2*p < 3*m*q - std::abs(t*q) && p < std::abs(0.5*s*q)) d = p/q; 168 | else { 169 | d = m; e = m; 170 | } 171 | } 172 | a = b; fa = fb; 173 | 174 | if (std::abs(d) > t) b += d; 175 | else if (m > 0) b += t; 176 | else b -= t; 177 | 178 | fb = f(b); k++; 179 | 180 | if (fb*fc > 0) { 181 | c = a; fc = fa; 182 | e = b-a; d = e; 183 | } 184 | } 185 | return b; 186 | } --------------------------------------------------------------------------------