├── .appveyor.yml ├── .github ├── ISSUE_TEMPLATE │ ├── 0-bug.md │ ├── 1-documentation.md │ └── 2-question.md └── workflows │ ├── auto-approve.yml │ ├── stale.yml │ └── stickers.yml ├── .gitignore ├── .travis.yml ├── CMake ├── ARMA_FindACML.cmake ├── ARMA_FindACMLMP.cmake ├── ARMA_FindARPACK.cmake ├── ARMA_FindBLAS.cmake ├── ARMA_FindCBLAS.cmake ├── ARMA_FindCLAPACK.cmake ├── ARMA_FindLAPACK.cmake ├── ARMA_FindMKL.cmake ├── ARMA_FindOpenBLAS.cmake ├── FindArmadillo.cmake └── ensmallen-config.cmake.in ├── CMakeLists.txt ├── CONTRIBUTING.md ├── COPYRIGHT.txt ├── HISTORY.md ├── LICENSE.txt ├── README.md ├── UPDATING.txt ├── doc ├── README.md ├── callbacks.md ├── function_types.md └── optimizers.md ├── example.cpp ├── include ├── ensmallen.hpp └── ensmallen_bits │ ├── ada_belief │ ├── ada_belief.hpp │ ├── ada_belief_impl.hpp │ └── ada_belief_update.hpp │ ├── ada_bound │ ├── ada_bound.hpp │ ├── ada_bound_impl.hpp │ ├── ada_bound_update.hpp │ └── ams_bound_update.hpp │ ├── ada_delta │ ├── ada_delta.hpp │ ├── ada_delta_impl.hpp │ └── ada_delta_update.hpp │ ├── ada_grad │ ├── ada_grad.hpp │ ├── ada_grad_impl.hpp │ └── ada_grad_update.hpp │ ├── ada_sqrt │ ├── ada_sqrt.hpp │ ├── ada_sqrt_impl.hpp │ └── ada_sqrt_update.hpp │ ├── adam │ ├── adam.hpp │ ├── adam_impl.hpp │ ├── adam_update.hpp │ ├── adamax_update.hpp │ ├── amsgrad_update.hpp │ ├── nadam_update.hpp │ ├── nadamax_update.hpp │ └── optimisticadam_update.hpp │ ├── agemoea │ ├── agemoea.hpp │ └── agemoea_impl.hpp │ ├── aug_lagrangian │ ├── aug_lagrangian.hpp │ ├── aug_lagrangian_function.hpp │ ├── aug_lagrangian_function_impl.hpp │ └── aug_lagrangian_impl.hpp │ ├── bigbatch_sgd │ ├── adaptive_stepsize.hpp │ ├── backtracking_line_search.hpp │ ├── bigbatch_sgd.hpp │ └── bigbatch_sgd_impl.hpp │ ├── callbacks │ ├── callbacks.hpp │ ├── early_stop_at_min_loss.hpp │ ├── grad_clip_by_norm.hpp │ ├── grad_clip_by_value.hpp │ ├── print_loss.hpp │ ├── progress_bar.hpp │ ├── query_front.hpp │ ├── report.hpp │ ├── store_best_coordinates.hpp │ ├── timer_stop.hpp │ └── traits.hpp │ ├── cd │ ├── cd.hpp │ ├── cd_impl.hpp │ └── descent_policies │ │ ├── cyclic_descent.hpp │ │ ├── greedy_descent.hpp │ │ └── random_descent.hpp │ ├── cmaes │ ├── active_cmaes.hpp │ ├── active_cmaes_impl.hpp │ ├── cmaes.hpp │ ├── cmaes_impl.hpp │ ├── full_selection.hpp │ ├── not_empty_transformation.hpp │ ├── pop_cmaes.hpp │ ├── pop_cmaes_impl.hpp │ ├── random_selection.hpp │ └── transformation_policies │ │ ├── boundary_box_constraint.hpp │ │ └── empty_transformation.hpp │ ├── cne │ ├── cne.hpp │ └── cne_impl.hpp │ ├── config.hpp │ ├── de │ ├── de.hpp │ └── de_impl.hpp │ ├── demon_adam │ ├── demon_adam.hpp │ └── demon_adam_update.hpp │ ├── demon_sgd │ ├── demon_sgd.hpp │ └── demon_sgd_update.hpp │ ├── ens_version.hpp │ ├── eve │ ├── eve.hpp │ └── eve_impl.hpp │ ├── ftml │ ├── ftml.hpp │ ├── ftml_impl.hpp │ └── ftml_update.hpp │ ├── function.hpp │ ├── function │ ├── add_evaluate.hpp │ ├── add_evaluate_with_gradient.hpp │ ├── add_gradient.hpp │ ├── add_separable_evaluate.hpp │ ├── add_separable_evaluate_with_gradient.hpp │ ├── add_separable_gradient.hpp │ ├── arma_traits.hpp │ ├── sfinae_utility.hpp │ ├── static_checks.hpp │ └── traits.hpp │ ├── fw │ ├── atoms.hpp │ ├── constr_lpball.hpp │ ├── constr_structure_group.hpp │ ├── frank_wolfe.hpp │ ├── frank_wolfe_impl.hpp │ ├── func_sq.hpp │ ├── line_search │ │ ├── line_search.hpp │ │ └── line_search_impl.hpp │ ├── proximal │ │ ├── proximal.hpp │ │ └── proximal_impl.hpp │ ├── update_classic.hpp │ ├── update_full_correction.hpp │ ├── update_linesearch.hpp │ └── update_span.hpp │ ├── gradient_descent │ ├── gradient_descent.hpp │ └── gradient_descent_impl.hpp │ ├── grid_search │ ├── grid_search.hpp │ └── grid_search_impl.hpp │ ├── iqn │ ├── iqn.hpp │ └── iqn_impl.hpp │ ├── katyusha │ ├── katyusha.hpp │ └── katyusha_impl.hpp │ ├── lbfgs │ ├── lbfgs.hpp │ └── lbfgs_impl.hpp │ ├── log.hpp │ ├── lookahead │ ├── lookahead.hpp │ └── lookahead_impl.hpp │ ├── moead │ ├── decomposition_policies │ │ ├── pbi_decomposition.hpp │ │ ├── tchebycheff_decomposition.hpp │ │ └── weighted_decomposition.hpp │ ├── moead.hpp │ ├── moead_impl.hpp │ └── weight_init_policies │ │ ├── bbs_init.hpp │ │ ├── dirichlet_init.hpp │ │ └── uniform_init.hpp │ ├── nsga2 │ ├── nsga2.hpp │ └── nsga2_impl.hpp │ ├── padam │ ├── padam.hpp │ └── padam_update.hpp │ ├── parallel_sgd │ ├── decay_policies │ │ ├── constant_step.hpp │ │ └── exponential_backoff.hpp │ ├── parallel_sgd.hpp │ └── parallel_sgd_impl.hpp │ ├── problems │ ├── ackley_function.hpp │ ├── ackley_function_impl.hpp │ ├── aug_lagrangian_test_functions.hpp │ ├── aug_lagrangian_test_functions_impl.hpp │ ├── beale_function.hpp │ ├── beale_function_impl.hpp │ ├── booth_function.hpp │ ├── booth_function_impl.hpp │ ├── bukin_function.hpp │ ├── bukin_function_impl.hpp │ ├── colville_function.hpp │ ├── colville_function_impl.hpp │ ├── cross_in_tray_function.hpp │ ├── cross_in_tray_function_impl.hpp │ ├── drop_wave_function.hpp │ ├── drop_wave_function_impl.hpp │ ├── dtlz │ │ ├── dtlz1_function.hpp │ │ ├── dtlz2_function.hpp │ │ ├── dtlz3_function.hpp │ │ ├── dtlz4_function.hpp │ │ ├── dtlz5_function.hpp │ │ ├── dtlz6_function.hpp │ │ └── dtlz7_function.hpp │ ├── easom_function.hpp │ ├── easom_function_impl.hpp │ ├── eggholder_function.hpp │ ├── eggholder_function_impl.hpp │ ├── fonseca_fleming_function.hpp │ ├── fw_test_function.hpp │ ├── generalized_rosenbrock_function.hpp │ ├── generalized_rosenbrock_function_impl.hpp │ ├── goldstein_price_function.hpp │ ├── goldstein_price_function_impl.hpp │ ├── gradient_descent_test_function.hpp │ ├── gradient_descent_test_function_impl.hpp │ ├── himmelblau_function.hpp │ ├── himmelblau_function_impl.hpp │ ├── holder_table_function.hpp │ ├── holder_table_function_impl.hpp │ ├── levy_function_n13.hpp │ ├── levy_function_n13_impl.hpp │ ├── logistic_regression_function.hpp │ ├── logistic_regression_function_impl.hpp │ ├── maf │ │ ├── maf1_function.hpp │ │ ├── maf2_function.hpp │ │ ├── maf3_function.hpp │ │ ├── maf4_function.hpp │ │ ├── maf5_function.hpp │ │ └── maf6_function.hpp │ ├── matyas_function.hpp │ ├── matyas_function_impl.hpp │ ├── mc_cormick_function.hpp │ ├── mc_cormick_function_impl.hpp │ ├── problems.hpp │ ├── rastrigin_function.hpp │ ├── rastrigin_function_impl.hpp │ ├── rosenbrock_function.hpp │ ├── rosenbrock_function_impl.hpp │ ├── rosenbrock_wood_function.hpp │ ├── rosenbrock_wood_function_impl.hpp │ ├── schaffer_function_n1.hpp │ ├── schaffer_function_n2.hpp │ ├── schaffer_function_n2_impl.hpp │ ├── schaffer_function_n4.hpp │ ├── schaffer_function_n4_impl.hpp │ ├── schwefel_function.hpp │ ├── schwefel_function_impl.hpp │ ├── sgd_test_function.hpp │ ├── sgd_test_function_impl.hpp │ ├── softmax_regression_function.hpp │ ├── softmax_regression_function_impl.hpp │ ├── sparse_test_function.hpp │ ├── sparse_test_function_impl.hpp │ ├── sphere_function.hpp │ ├── sphere_function_impl.hpp │ ├── styblinski_tang_function.hpp │ ├── styblinski_tang_function_impl.hpp │ ├── three_hump_camel_function.hpp │ ├── three_hump_camel_function_impl.hpp │ ├── wood_function.hpp │ ├── wood_function_impl.hpp │ └── zdt │ │ ├── zdt1_function.hpp │ │ ├── zdt2_function.hpp │ │ ├── zdt3_function.hpp │ │ ├── zdt4_function.hpp │ │ └── zdt6_function.hpp │ ├── pso │ ├── init_policies │ │ └── default_init.hpp │ ├── pso.hpp │ ├── pso_impl.hpp │ └── update_policies │ │ └── lbest_update.hpp │ ├── qhadam │ ├── qhadam.hpp │ ├── qhadam_impl.hpp │ └── qhadam_update.hpp │ ├── rmsprop │ ├── rmsprop.hpp │ └── rmsprop_update.hpp │ ├── sa │ ├── exponential_schedule.hpp │ ├── sa.hpp │ └── sa_impl.hpp │ ├── sarah │ ├── sarah.hpp │ ├── sarah_impl.hpp │ ├── sarah_plus_update.hpp │ └── sarah_update.hpp │ ├── sdp │ ├── lin_alg.hpp │ ├── lrsdp.hpp │ ├── lrsdp_function.hpp │ ├── lrsdp_function_impl.hpp │ ├── lrsdp_impl.hpp │ ├── primal_dual.hpp │ ├── primal_dual_impl.hpp │ ├── sdp.hpp │ └── sdp_impl.hpp │ ├── sgd │ ├── decay_policies │ │ └── no_decay.hpp │ ├── sgd.hpp │ ├── sgd_impl.hpp │ └── update_policies │ │ ├── gradient_clipping.hpp │ │ ├── momentum_update.hpp │ │ ├── nesterov_momentum_update.hpp │ │ ├── quasi_hyperbolic_update.hpp │ │ └── vanilla_update.hpp │ ├── sgdr │ ├── cyclical_decay.hpp │ ├── sgdr.hpp │ ├── sgdr_impl.hpp │ ├── snapshot_ensembles.hpp │ ├── snapshot_sgdr.hpp │ └── snapshot_sgdr_impl.hpp │ ├── smorms3 │ ├── smorms3.hpp │ ├── smorms3_impl.hpp │ └── smorms3_update.hpp │ ├── spalera_sgd │ ├── spalera_sgd.hpp │ ├── spalera_sgd_impl.hpp │ └── spalera_stepsize.hpp │ ├── spsa │ ├── spsa.hpp │ └── spsa_impl.hpp │ ├── svrg │ ├── barzilai_borwein_decay.hpp │ ├── svrg.hpp │ ├── svrg_impl.hpp │ └── svrg_update.hpp │ ├── swats │ ├── swats.hpp │ ├── swats_impl.hpp │ └── swats_update.hpp │ ├── utility │ ├── any.hpp │ ├── arma_traits.hpp │ └── indicators │ │ ├── epsilon.hpp │ │ ├── igd.hpp │ │ └── igd_plus.hpp │ ├── wn_grad │ ├── wn_grad.hpp │ ├── wn_grad_impl.hpp │ └── wn_grad_update.hpp │ └── yogi │ ├── yogi.hpp │ ├── yogi_impl.hpp │ └── yogi_update.hpp ├── scripts ├── ensmallen-release.sh ├── history-update-check.sh └── update-website-after-release.sh └── tests ├── CMakeLists.txt ├── LICENSE_1_0.txt ├── active_cmaes_test.cpp ├── ada_belief_test.cpp ├── ada_bound_test.cpp ├── ada_delta_test.cpp ├── ada_grad_test.cpp ├── ada_sqrt_test.cpp ├── adam_test.cpp ├── agemoea_test.cpp ├── aug_lagrangian_test.cpp ├── bigbatch_sgd_test.cpp ├── callbacks_test.cpp ├── catch.hpp ├── cd_test.cpp ├── cmaes_test.cpp ├── cne_test.cpp ├── data ├── erdosrenyi-n100.csv ├── johnson8-4-4.csv ├── r10.txt ├── sensing_A.csv └── sensing_X.csv ├── de_test.cpp ├── demon_adam_test.cpp ├── demon_sgd_test.cpp ├── eve_test.cpp ├── frankwolfe_test.cpp ├── ftml_test.cpp ├── function_test.cpp ├── gradient_descent_test.cpp ├── grid_search_test.cpp ├── indicators_test.cpp ├── iqn_test.cpp ├── katyusha_test.cpp ├── lbfgs_test.cpp ├── line_search_test.cpp ├── lookahead_test.cpp ├── lrsdp_test.cpp ├── main.cpp ├── moead_test.cpp ├── momentum_sgd_test.cpp ├── nesterov_momentum_sgd_test.cpp ├── nsga2_test.cpp ├── parallel_sgd_test.cpp ├── pop_cmaes_test.cpp ├── proximal_test.cpp ├── pso_test.cpp ├── quasi_hyperbolic_momentum_sgd_test.cpp ├── rmsprop_test.cpp ├── sa_test.cpp ├── sarah_test.cpp ├── sdp_primal_dual_test.cpp ├── sgd_test.cpp ├── sgdr_test.cpp ├── smorms3_test.cpp ├── snapshot_ensembles.cpp ├── spalera_sgd_test.cpp ├── spsa_test.cpp ├── svrg_test.cpp ├── swats_test.cpp ├── test_function_tools.hpp ├── wn_grad_test.cpp └── yogi_test.cpp /.appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | ARMADILLO_DOWNLOAD: "https://sourceforge.net/projects/arma/files/armadillo-10.8.2.tar.xz" 3 | BLAS_LIBRARY: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" 4 | BLAS_LIBRARY_DLL: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll" 5 | 6 | matrix: 7 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 8 | VSVER: Visual Studio 15 2017 Win64 9 | MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe 10 | 11 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 12 | VSVER: Visual Studio 16 2019 13 | MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe 14 | 15 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 16 | VSVER: Visual Studio 17 2022 17 | MSBUILD: C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Current\Bin\MSBuild.exe 18 | 19 | configuration: Release 20 | 21 | install: 22 | - ps: nuget install OpenBLAS -o "${env:APPVEYOR_BUILD_FOLDER}" 23 | 24 | build_script: 25 | # First, download and build Armadillo. 26 | - cd .. 27 | - appveyor DownloadFile %ARMADILLO_DOWNLOAD% -FileName armadillo.tar.xz 28 | - 7z x armadillo.tar.xz -so -txz | 7z x -si -ttar > nul 29 | - cd armadillo-10.8.2 && mkdir build && cd build 30 | - > 31 | cmake -G "%VSVER%" 32 | -DBLAS_LIBRARY:FILEPATH=%BLAS_LIBRARY% 33 | -DLAPACK_LIBRARY:FILEPATH=%BLAS_LIBRARY% 34 | -DCMAKE_PREFIX:FILEPATH="%APPVEYOR_BUILD_FOLDER%/armadillo" 35 | -DBUILD_SHARED_LIBS=OFF 36 | -DCMAKE_BUILD_TYPE=Release .. 37 | - > 38 | "%MSBUILD%" "armadillo.sln" 39 | /m /verbosity:quiet /p:Configuration=Release;Platform=x64 40 | - cd ../.. 41 | 42 | # Now build ensmallen. 43 | - cd ensmallen && mkdir build && cd build 44 | - > 45 | cmake -G "%VSVER%" 46 | -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-10.8.2/include/ 47 | -DARMADILLO_LIBRARIES=%BLAS_LIBRARY% 48 | -DLAPACK_LIBRARY=%BLAS_LIBRARY% 49 | -DBLAS_LIBRARY=%BLAS_LIBRARY% 50 | -DCMAKE_BUILD_TYPE=Release .. 51 | - > 52 | "%MSBUILD%" "ensmallen.sln" 53 | /m /verbosity:minimal /nologo /p:BuildInParallel=false 54 | 55 | # Run tests after copying libraries. 56 | - ps: cp C:\projects\ensmallen\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* C:\projects\ensmallen\build\ 57 | - ctest -C Release -V --output-on-failure . 58 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/0-bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Use this template for reporting a bug that you have found in ensmallen. 4 | title: '' 5 | labels: 't: bug report, s: unanswered' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 16 | 17 | #### Issue description 18 | 19 | 20 | 21 | #### Your environment 22 | 23 | * version of ensmallen: 24 | * operating system: 25 | * compiler: 26 | * version of Armadillo: 27 | * any other environment information you think is relevant: 28 | 29 | #### Steps to reproduce 30 | 31 | 33 | 34 | #### Expected behavior 35 | 36 | 37 | 38 | #### Actual behavior 39 | 40 | 41 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/1-documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation issue 3 | about: Use this template to report an issue you've found with the documentation. 4 | title: '' 5 | labels: 't: bug report, c: documentation, s: unanswered' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 18 | 19 | #### Problem location 20 | 21 | 23 | 24 | #### Description of problem 25 | 26 | 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Use this template for other problems, requests, or questions. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 19 | -------------------------------------------------------------------------------- /.github/workflows/auto-approve.yml: -------------------------------------------------------------------------------- 1 | # Once a PR has been approved by one member of the mlpack organization, a second 2 | # approving review will automatically be added 24 hours later. This allows time 3 | # for other maintainers to take a look. 4 | name: Auto-approve pull requests 5 | on: 6 | schedule: 7 | # Run roughly every four hours. 8 | - cron: "15 0,4,8,12,16,20 * * *" 9 | 10 | jobs: 11 | auto-approve: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | pull-requests: write 15 | 16 | steps: 17 | - name: Auto-approve pull requests 18 | uses: rcurtin/actions/auto-approve@v1 19 | with: 20 | repo-token: ${{ secrets.GITHUB_TOKEN }} 21 | approval-message: 22 | 'Second approval provided automatically after 24 hours. :+1:' 23 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close inactive issues 2 | on: 3 | schedule: 4 | - cron: "30 1 * * *" 5 | 6 | jobs: 7 | close-issues: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | issues: write 11 | pull-requests: write 12 | steps: 13 | - uses: actions/stale@v9 14 | with: 15 | days-before-issues-stale: 30 16 | days-before-issue-close: 7 17 | stale-issue-label: "s: stale" 18 | stale-pr-label: "s: stale" 19 | stale-issue-message: "This issue has been automatically marked as stale because it has not had any recent activity. It will be closed in 7 days if no further activity occurs. Thank you for your contributions! :+1:" 20 | days-before-pr-stale: 30, 21 | days-before-pr-close: 14 22 | repo-token: ${{ secrets.GITHUB_TOKEN }} 23 | exempt-issue-labels: "s: keep open" 24 | exempt-pr-labels: "s: keep open" 25 | -------------------------------------------------------------------------------- /.github/workflows/stickers.yml: -------------------------------------------------------------------------------- 1 | # Post a message to new contributors that they can get some stickers mailed to 2 | # them. 3 | name: 'Stickers for new contributors' 4 | on: 5 | pull_request: 6 | types: [closed] 7 | 8 | jobs: 9 | sticker_comment: 10 | runs-on: ubuntu-latest 11 | if: github.event.pull_request.merged == true 12 | steps: 13 | # Forked version of first-interaction that runs only on first merged PR. 14 | - uses: rcurtin/actions/stickers@v1 15 | with: 16 | repo-token: ${{ secrets.GITHUB_TOKEN }} 17 | pr-message: "Hello there! Thanks for your contribution. Congratulations on your first contribution to mlpack! If you'd like to add your name to the list of contributors in `COPYRIGHT.txt` and you haven't already, please feel free to push a change to this PR---or, if it gets merged before you can, feel free to open another PR.\n\nIn addition, if you'd like some stickers to put on your laptop, we can get them in the mail for you. Just send an email with your physical mailing address to stickers@mlpack.org, and then one of the mlpack maintainers will put some stickers in an envelope for you. It may take a few weeks to get them, depending on your location. :+1:" 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | dist: focal 3 | language: cpp 4 | 5 | env: 6 | - ARMADILLO=latest SANITY_HISTORY=perform 7 | - ARMADILLO=minimum 8 | 9 | stages: 10 | - test 11 | - name: sanity 12 | if: type = pull_request AND env(SANITY_HISTORY) = "perform" 13 | 14 | jobs: 15 | include: 16 | - stage: sanity 17 | name: "HISTORY.md Check" 18 | script: sh ./scripts/history-update-check.sh 19 | 20 | script: 21 | - sudo apt-get update 22 | - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ xz-utils 23 | - if [ $ARMADILLO == "latest" ]; then 24 | curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.* 9 | #include 10 | #include 11 | 12 | 13 | class LinearRegressionFunction 14 | { 15 | public: 16 | 17 | LinearRegressionFunction(arma::mat& X, arma::vec& y) : X(X), y(y) { } 18 | 19 | double EvaluateWithGradient(const arma::mat& theta, arma::mat& gradient) 20 | { 21 | const arma::vec tmp = X.t() * theta - y; 22 | gradient = 2 * X * tmp; 23 | return arma::dot(tmp,tmp); 24 | } 25 | 26 | private: 27 | 28 | const arma::mat& X; 29 | const arma::vec& y; 30 | }; 31 | 32 | 33 | int main(int argc, char** argv) 34 | { 35 | if (argc < 3) 36 | { 37 | std::cout << "usage: " << argv[0] << " n_dims n_points" << std::endl; 38 | return -1; 39 | } 40 | 41 | int n_dims = atoi(argv[1]); 42 | int n_points = atoi(argv[2]); 43 | 44 | // generate noisy dataset with a slight linear pattern 45 | arma::mat X(n_dims, n_points, arma::fill::randu); 46 | arma::vec y( n_points, arma::fill::randu); 47 | 48 | for (size_t i = 0; i < n_points; ++i) 49 | { 50 | double a = arma::randu(); 51 | X(1, i) += a; 52 | y(i) += a; 53 | } 54 | 55 | LinearRegressionFunction lrf(X, y); 56 | 57 | // create a Limited-memory BFGS optimizer object with default parameters 58 | ens::L_BFGS opt; 59 | opt.MaxIterations() = 10; 60 | 61 | // initial point (uniform random) 62 | arma::vec theta(n_dims, arma::fill::randu); 63 | 64 | opt.Optimize(lrf, theta); 65 | 66 | // theta now contains the optimized parameters 67 | theta.print("theta:"); 68 | 69 | return 0; 70 | } 71 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_belief/ada_belief_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_belief_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of AdaBelief class wrapper. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP 13 | #define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "ada_belief.hpp" 17 | 18 | namespace ens { 19 | 20 | inline AdaBelief::AdaBelief( 21 | const double stepSize, 22 | const size_t batchSize, 23 | const double beta1, 24 | const double beta2, 25 | const double epsilon, 26 | const size_t maxIterations, 27 | const double tolerance, 28 | const bool shuffle, 29 | const bool resetPolicy, 30 | const bool exactObjective) : 31 | optimizer(stepSize, 32 | batchSize, 33 | maxIterations, 34 | tolerance, 35 | shuffle, 36 | AdaBeliefUpdate(epsilon, beta1, beta2), 37 | NoDecay(), 38 | resetPolicy, 39 | exactObjective) 40 | { /* Nothing to do. */ } 41 | 42 | } // namespace ens 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_bound/ada_bound_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file qhadam_impl.hpp 3 | * @author Niteya Shah 4 | * 5 | * Implementation of QHAdam class wrapper. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADA_BOUND_IMPL_HPP 13 | #define ENSMALLEN_ADA_BOUND_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "ada_bound.hpp" 17 | 18 | namespace ens { 19 | 20 | template 21 | AdaBoundType::AdaBoundType( 22 | const double stepSize, 23 | const size_t batchSize, 24 | const double finalLr, 25 | const double gamma, 26 | const double beta1, 27 | const double beta2, 28 | const double epsilon, 29 | const size_t maxIterations, 30 | const double tolerance, 31 | const bool shuffle, 32 | const bool resetPolicy, 33 | const bool exactObjective) : 34 | optimizer(stepSize, 35 | batchSize, 36 | maxIterations, 37 | tolerance, 38 | shuffle, 39 | UpdatePolicyType(finalLr, gamma, epsilon, beta1, beta2), 40 | NoDecay(), 41 | resetPolicy, 42 | exactObjective) 43 | { /* Nothing to do. */ } 44 | 45 | } // namespace ens 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_delta/ada_delta_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_delta_impl.hpp 3 | * @author Ryan Curtin 4 | * @author Vasanth Kalingeri 5 | * @author Abhinav Moudgil 6 | * 7 | * Implementation of the AdaDelta optimizer. 8 | * 9 | * ensmallen is free software; you may redistribute it and/or modify it under 10 | * the terms of the 3-clause BSD license. You should have received a copy of 11 | * the 3-clause BSD license along with ensmallen. If not, see 12 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 13 | */ 14 | #ifndef ENSMALLEN_ADA_DELTA_ADA_DELTA_IMPL_HPP 15 | #define ENSMALLEN_ADA_DELTA_ADA_DELTA_IMPL_HPP 16 | 17 | // In case it hasn't been included yet. 18 | #include "ada_delta.hpp" 19 | 20 | namespace ens { 21 | 22 | inline AdaDelta::AdaDelta(const double stepSize, 23 | const size_t batchSize, 24 | const double rho, 25 | const double epsilon, 26 | const size_t maxIterations, 27 | const double tolerance, 28 | const bool shuffle, 29 | const bool resetPolicy, 30 | const bool exactObjective) : 31 | optimizer(stepSize, 32 | batchSize, 33 | maxIterations, 34 | tolerance, 35 | shuffle, 36 | AdaDeltaUpdate(rho, epsilon), 37 | NoDecay(), 38 | resetPolicy, 39 | exactObjective) 40 | { /* Nothing to do. */ } 41 | 42 | } // namespace ens 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_grad/ada_grad_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_grad_impl.hpp 3 | * @author Abhinav Moudgil 4 | * 5 | * Implementation of AdaGrad optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADA_GRAD_ADA_GRAD_IMPL_HPP 13 | #define ENSMALLEN_ADA_GRAD_ADA_GRAD_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "ada_grad.hpp" 17 | 18 | namespace ens { 19 | 20 | inline AdaGrad::AdaGrad(const double stepSize, 21 | const size_t batchSize, 22 | const double epsilon, 23 | const size_t maxIterations, 24 | const double tolerance, 25 | const bool shuffle, 26 | const bool resetPolicy, 27 | const bool exactObjective) : 28 | optimizer(stepSize, 29 | batchSize, 30 | maxIterations, 31 | tolerance, 32 | shuffle, 33 | AdaGradUpdate(epsilon), 34 | NoDecay(), 35 | resetPolicy, 36 | exactObjective) 37 | { /* Nothing to do. */ } 38 | 39 | } // namespace ens 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_sqrt/ada_sqrt_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_sqrt_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of AdaSqrt optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADA_SQRT_ADA_SQRT_IMPL_HPP 13 | #define ENSMALLEN_ADA_SQRT_ADA_SQRT_IMPL_HPP 14 | 15 | namespace ens { 16 | 17 | inline AdaSqrt::AdaSqrt(const double stepSize, 18 | const size_t batchSize, 19 | const double epsilon, 20 | const size_t maxIterations, 21 | const double tolerance, 22 | const bool shuffle, 23 | const bool resetPolicy, 24 | const bool exactObjective) : 25 | optimizer(stepSize, 26 | batchSize, 27 | maxIterations, 28 | tolerance, 29 | shuffle, 30 | AdaSqrtUpdate(epsilon), 31 | NoDecay(), 32 | resetPolicy, 33 | exactObjective) 34 | { /* Nothing to do. */ } 35 | 36 | } // namespace ens 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /include/ensmallen_bits/adam/adam_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file adam_impl.hpp 3 | * @author Ryan Curtin 4 | * @author Vasanth Kalingeri 5 | * @author Marcus Edel 6 | * @author Vivek Pal 7 | * 8 | * Implementation of the Adam, AdaMax, AMSGrad, Nadam and NadaMax optimizer. 9 | * 10 | * ensmallen is free software; you may redistribute it and/or modify it under 11 | * the terms of the 3-clause BSD license. You should have received a copy of 12 | * the 3-clause BSD license along with ensmallen. If not, see 13 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 14 | */ 15 | #ifndef ENSMALLEN_ADAM_ADAM_IMPL_HPP 16 | #define ENSMALLEN_ADAM_ADAM_IMPL_HPP 17 | 18 | // In case it hasn't been included yet. 19 | #include "adam.hpp" 20 | 21 | namespace ens { 22 | 23 | template 24 | AdamType::AdamType( 25 | const double stepSize, 26 | const size_t batchSize, 27 | const double beta1, 28 | const double beta2, 29 | const double epsilon, 30 | const size_t maxIterations, 31 | const double tolerance, 32 | const bool shuffle, 33 | const bool resetPolicy, 34 | const bool exactObjective) : 35 | optimizer(stepSize, 36 | batchSize, 37 | maxIterations, 38 | tolerance, 39 | shuffle, 40 | UpdateRule(epsilon, beta1, beta2), 41 | NoDecay(), 42 | resetPolicy, 43 | exactObjective) 44 | { /* Nothing to do. */ } 45 | 46 | } // namespace ens 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/grad_clip_by_norm.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file grad_clip_by_norm.hpp 3 | * @author Marcus Edel 4 | * 5 | * Clip the gradients by multiplying the unit vector of the gradients with the 6 | * threshold. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP 14 | #define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP 15 | 16 | namespace ens { 17 | 18 | /** 19 | * Clip the gradients by multiplying the unit vector of the gradients with the 20 | * threshold. 21 | */ 22 | class GradClipByNorm 23 | { 24 | public: 25 | /** 26 | * Set up the gradient clip by norm callback class with the maximum clipping 27 | * value. 28 | * 29 | * @param maxNorm The maximum clipping value. 30 | */ 31 | GradClipByNorm(const double maxNorm) : maxNorm(maxNorm) 32 | { /* Nothing to do here. */ } 33 | 34 | /** 35 | * Callback function called at any call to Gradient(). 36 | * 37 | * @param optimizer The optimizer used to update the function. 38 | * @param function Function to optimize. 39 | * @param coordinates Starting point. 40 | * @param gradient Matrix that holds the gradient. 41 | */ 42 | template 43 | bool Gradient(OptimizerType& /* optimizer */, 44 | FunctionType& /* function */, 45 | const MatType& /* coordinates */, 46 | MatType& gradient) 47 | { 48 | const double gradientNorm = arma::norm(gradient); 49 | if (gradientNorm > maxNorm) 50 | gradient = maxNorm * gradient / gradientNorm; 51 | return false; 52 | } 53 | 54 | private: 55 | //! The maximum clipping value for gradient clipping. 56 | const double maxNorm; 57 | }; 58 | 59 | } // namespace ens 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/grad_clip_by_value.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file grad_clip_by_value.hpp 3 | * @author Marcus Edel 4 | * 5 | * Clips the gradient to a specified min and max. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP 13 | #define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Clip the gradient to a specified min and max. 19 | */ 20 | class GradClipByValue 21 | { 22 | public: 23 | /** 24 | * Set up the gradient clip by value callback class with the min and max 25 | * value. 26 | * 27 | * @param min The minimum value to clip to. 28 | * @param max The maximum value to clip to. 29 | */ 30 | GradClipByValue(const double min, const double max) : lower(min), upper(max) 31 | { /* Nothing to do here. */ } 32 | 33 | /** 34 | * Callback function called at any call to Gradient(). 35 | * 36 | * @param optimizer The optimizer used to update the function. 37 | * @param function Function to optimize. 38 | * @param coordinates Starting point. 39 | * @param gradient Matrix that holds the gradient. 40 | */ 41 | template 42 | bool Gradient(OptimizerType& /* optimizer */, 43 | FunctionType& /* function */, 44 | const MatType& /* coordinates */, 45 | MatType& gradient) 46 | { 47 | gradient = arma::clamp(gradient, lower, upper); 48 | return false; 49 | } 50 | 51 | private: 52 | //! The minimum value to clip to. 53 | const double lower; 54 | 55 | //! The maximum value to clip to. 56 | const double upper; 57 | }; 58 | 59 | } // namespace ens 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/print_loss.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file print_loss.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the print loss callback function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CALLBACKS_PRINT_LOSS_HPP 13 | #define ENSMALLEN_CALLBACKS_PRINT_LOSS_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Print loss function, based on the EndEpoch callback function. 19 | */ 20 | class PrintLoss 21 | { 22 | public: 23 | /** 24 | * Set up the print loss callback class with the width and output stream. 25 | * 26 | * @param ostream Ostream which receives output from this object. 27 | */ 28 | PrintLoss(std::ostream& output = arma::get_cout_stream()) : output(output) 29 | { /* Nothing to do here. */ } 30 | 31 | /** 32 | * Callback function called at the end of a pass over the data. 33 | * 34 | * @param optimizer The optimizer used to update the function. 35 | * @param function Function to optimize. 36 | * @param coordinates Starting point. 37 | * @param epoch The index of the current epoch. 38 | * @param objective Objective value of the current point. 39 | */ 40 | template 41 | bool EndEpoch(OptimizerType& /* optimizer */, 42 | FunctionType& /* function */, 43 | const MatType& /* coordinates */, 44 | const size_t /* epoch */, 45 | const double objective) 46 | { 47 | output << objective << std::endl; 48 | return false; 49 | } 50 | 51 | private: 52 | //! The output stream that all data is to be sent to; example: std::cout. 53 | std::ostream& output; 54 | }; 55 | 56 | } // namespace ens 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/store_best_coordinates.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file store_best_coordinates.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the store best coordinates callback function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CALLBACKS_STORE_BEST_COORDINATES_HPP 13 | #define ENSMALLEN_CALLBACKS_STORE_BEST_COORDINATES_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Store best coordinates function, based on the Evaluate callback function. 19 | * 20 | * @tparam MatType Type of the model coordinates (arma::colvec, arma::mat, 21 | * arma::sp_mat or arma::cube). 22 | */ 23 | template 24 | class StoreBestCoordinates 25 | { 26 | public: 27 | /** 28 | * Set up the store best model class, which keeps the best-performing 29 | * coordinates and objective. 30 | */ 31 | StoreBestCoordinates() : bestObjective(std::numeric_limits::max()) 32 | { /* Nothing to do here. */ } 33 | 34 | /** 35 | * Callback function called after any call to Evaluate(). 36 | * 37 | * @param optimizer The optimizer used to update the function. 38 | * @param function Function to optimize. 39 | * @param coordinates Starting point. 40 | * @param objective Objective value of the current point. 41 | */ 42 | template 43 | bool Evaluate(OptimizerType& /* optimizer */, 44 | FunctionType& /* function */, 45 | const MatType& coordinates, 46 | const double objective) 47 | { 48 | if (objective < bestObjective) 49 | { 50 | bestObjective = objective; 51 | bestCoordinates = coordinates; 52 | } 53 | return false; 54 | } 55 | 56 | //! Get the best coordinates. 57 | ModelMatType const& BestCoordinates() const { return bestCoordinates; } 58 | //! Modify the best coordinates. 59 | ModelMatType& BestCoordinates() { return bestCoordinates; } 60 | 61 | //! Get the best objective. 62 | double const& BestObjective() const { return bestObjective; } 63 | //! Modify the best objective. 64 | double& BestObjective() { return bestObjective; } 65 | 66 | private: 67 | //! Locally-stored best objective. 68 | double bestObjective; 69 | 70 | //! Locally-stored best model coordinates. 71 | ModelMatType bestCoordinates; 72 | }; 73 | 74 | } // namespace ens 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/timer_stop.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file timer_stop.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the timer stop callback function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CALLBACKS_TIMER_STOP_HPP 13 | #define ENSMALLEN_CALLBACKS_TIMER_STOP_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Timer stop function, is based on the BeginOptimization callback function to 19 | * start the timer and the EndEpoch callback function to update the timer. 20 | */ 21 | class TimerStop 22 | { 23 | public: 24 | /** 25 | * Set up the print loss callback class with the width and output stream. 26 | * 27 | * @param durationIn The duration of the timer in seconds. 28 | */ 29 | TimerStop(const double durationIn) : duration(durationIn) 30 | { /* Nothing to do here. */ } 31 | 32 | /** 33 | * Callback function called at the start of the optimization process. 34 | * 35 | * @param optimizer The optimizer used to update the function. 36 | * @param function Function to optimize. 37 | * @param coordinates Starting point. 38 | */ 39 | template 40 | void BeginOptimization(OptimizerType& /* optimizer */, 41 | FunctionType& /* function */, 42 | MatType& /* coordinates */) 43 | { 44 | // Start the timer. 45 | timer.tic(); 46 | } 47 | 48 | /** 49 | * Callback function called at the end of a pass over the data. 50 | * 51 | * @param optimizer The optimizer used to update the function. 52 | * @param function Function to optimize. 53 | * @param coordinates Starting point. 54 | * @param epoch The index of the current epoch. 55 | * @param objective Objective value of the current point. 56 | */ 57 | template 58 | bool EndEpoch(OptimizerType& /* optimizer */, 59 | FunctionType& /* function */, 60 | const MatType& /* coordinates */, 61 | const size_t /* epoch */, 62 | const double /* objective */) 63 | { 64 | if (timer.toc() > duration) 65 | { 66 | Info << "Timer timeout reached; terminate optimization." << std::endl; 67 | return true; 68 | } 69 | 70 | return false; 71 | } 72 | 73 | private: 74 | //! The duration in seconds. 75 | double duration; 76 | 77 | //! Locally-stored timer object. 78 | arma::wall_clock timer; 79 | }; 80 | 81 | } // namespace ens 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cd/descent_policies/cyclic_descent.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file cyclic_descent.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Cyclic descent policy for Stochastic Coordinate Descent (SCD). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SCD_DESCENT_POLICIES_CYCLIC_HPP 13 | #define ENSMALLEN_SCD_DESCENT_POLICIES_CYCLIC_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Cyclic descent policy for Stochastic Coordinate Descent(SCD). This 19 | * descent scheme picks a the co-ordinate for the descent in a cyclic manner 20 | * serially. 21 | * 22 | * For more information, see the following. 23 | * @code 24 | * @inproceedings{Shalev-Shwartz2009, 25 | * author = {Shalev-Shwartz, Shai and Tewari, Ambuj}, 26 | * title = {Stochastic Methods for L1 Regularized Loss Minimization}, 27 | * booktitle = {Proceedings of the 26th Annual International Conference on 28 | * Machine Learning}, 29 | * series = {ICML '09}, 30 | * year = {2009}, 31 | * isbn = {978-1-60558-516-1} 32 | * } 33 | * @endcode 34 | */ 35 | class CyclicDescent 36 | { 37 | public: 38 | /** 39 | * The DescentFeature method is used to get the descent coordinate for the 40 | * current iteration. 41 | * 42 | * @tparam ResolvableFunctionType The type of the function to be optimized. 43 | * @param iteration The iteration number for which the feature is to be 44 | * obtained. 45 | * @param iterate The current value of the decision variable. 46 | * @param function The function to be optimized. 47 | * @return The index of the coordinate to be descended. 48 | */ 49 | template 50 | static size_t DescentFeature(const size_t iteration, 51 | const MatType& /* iterate */, 52 | const ResolvableFunctionType& function) 53 | { 54 | return iteration % function.NumFeatures(); 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cd/descent_policies/greedy_descent.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file greedy_descent.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Greedy descent policy for Stochastic Coordinate Descent (SCD). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SCD_DESCENT_POLICIES_GREEDY_HPP 13 | #define ENSMALLEN_SCD_DESCENT_POLICIES_GREEDY_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Greedy descent policy for Stochastic Co-ordinate Descent(SCD). This 19 | * descent scheme picks a the co-ordinate for the descent with the maximum 20 | * guaranteed descent, according to the Gauss-Southwell rule. This is a 21 | * deterministic approach and is generally more expensive to calculate. 22 | * 23 | * For more information, refer to the following. 24 | * @code 25 | * @misc{Nutini2015, 26 | * author = {Julie Nutini and Mark Schmidt and Issam H. 27 | * Laradji and Michael Friedlander and Hoyt Koepke}, 28 | * title = {Coordinate Descent Converges Faster with the Gauss-Southwell Rule 29 | * Than Random Selection}, 30 | * year = {2015}, 31 | * eprint = {arXiv:1506.00552} 32 | * } 33 | * @endcode 34 | */ 35 | class GreedyDescent 36 | { 37 | public: 38 | /** 39 | * The DescentFeature method is used to get the descent coordinate for the 40 | * current iteration. 41 | * 42 | * @tparam ResolvableFunctionType The type of the function to be optimized. 43 | * @param iteration The iteration number for which the feature is to be 44 | * obtained. 45 | * @param iterate The current value of the decision variable. 46 | * @param function The function to be optimized. 47 | * @return The index of the coordinate to be descended. 48 | */ 49 | template 50 | static size_t DescentFeature(const size_t /* iteration */, 51 | const MatType& iterate, 52 | const ResolvableFunctionType& function) 53 | { 54 | typedef typename MatType::elem_type ElemType; 55 | 56 | size_t bestFeature = 0; 57 | ElemType bestDescent = 0; 58 | for (size_t i = 0; i < function.NumFeatures(); ++i) 59 | { 60 | GradType fGrad; 61 | 62 | function.PartialGradient(iterate, i, fGrad); 63 | 64 | ElemType descent = arma::accu(fGrad); 65 | if (descent > bestDescent) 66 | { 67 | bestFeature = i; 68 | bestDescent = descent; 69 | } 70 | } 71 | 72 | return bestFeature; 73 | } 74 | }; 75 | 76 | } // namespace ens 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cd/descent_policies/random_descent.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file random_descent.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Random descent policy for Stochastic Coordinate Descent (SCD). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SCD_DESCENT_POLICIES_RANDOM_HPP 13 | #define ENSMALLEN_SCD_DESCENT_POLICIES_RANDOM_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Random descent policy for Stochastic Coordinate Descent(SCD). This 19 | * descent scheme picks a the co-ordinate for the descent uniformly randomly. 20 | * 21 | * For more information, see the following. 22 | * @code 23 | * @inproceedings{ShalevShwartz2009, 24 | * author = {Shalev-Shwartz, Shai and Tewari, Ambuj}, 25 | * title = {Stochastic Methods for L1 Regularized Loss Minimization}, 26 | * booktitle = {Proceedings of the 26th Annual International Conference on 27 | * Machine Learning}, 28 | * series = {ICML '09}, 29 | * year = {2009}, 30 | * isbn = {978-1-60558-516-1} 31 | * } 32 | * @endcode 33 | */ 34 | class RandomDescent 35 | { 36 | public: 37 | /** 38 | * The DescentFeature method is used to get the descent coordinate for the 39 | * current iteration of the SCD optimizer. For more information regarding the 40 | * interface of this policy with the optimizer, have a look at the SCD 41 | * implementation. 42 | * 43 | * @tparam ResolvableFunctionType The type of the function to be optimized. 44 | * @param iteration The iteration number for which the feature is to be 45 | * obtained. 46 | * @param iterate The current value of the decision variable. 47 | * @param function The function to be optimized. 48 | * @return The index of the coordinate to be descended. 49 | */ 50 | template 51 | static size_t DescentFeature(const size_t /* iteration */, 52 | const MatType& /* iterate */, 53 | const ResolvableFunctionType& function) 54 | { 55 | return arma::as_scalar(arma::randi( 56 | 1, arma::distr_param(0, function.NumFeatures() - 1))); 57 | } 58 | }; 59 | 60 | } // namespace ens 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cmaes/full_selection.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file full_selection.hpp 3 | * @author Marcus Edel 4 | * 5 | * Select the full dataset for use in the Evaluation step. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CMAES_FULL_SELECTION_HPP 13 | #define ENSMALLEN_CMAES_FULL_SELECTION_HPP 14 | 15 | namespace ens { 16 | 17 | /* 18 | * Select the full dataset for use in the Evaluation step. 19 | */ 20 | class FullSelection 21 | { 22 | public: 23 | /** 24 | * Select the full dataset to calculate the objective function. 25 | * 26 | * @tparam SeparableFunctionType Type of the function to be evaluated. 27 | * @param function Function to optimize. 28 | * @param batchSize Batch size to use for each step. 29 | * @param terminate Whether optimization should be terminated after this call. 30 | * @param iterate starting point. 31 | */ 32 | template 35 | double Select(SeparableFunctionType& function, 36 | const size_t batchSize, 37 | const MatType& iterate, 38 | bool& terminate, 39 | CallbackTypes&... callbacks) 40 | { 41 | // Find the number of functions to use. 42 | const size_t numFunctions = function.NumFunctions(); 43 | 44 | typename MatType::elem_type objective = 0; 45 | for (size_t f = 0; f < numFunctions; f += batchSize) 46 | { 47 | const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f); 48 | objective += function.Evaluate(iterate, f, effectiveBatchSize); 49 | 50 | terminate |= Callback::Evaluate(*this, f, iterate, objective, 51 | callbacks...); 52 | } 53 | 54 | return objective; 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cmaes/not_empty_transformation.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file not_empty_transformation.hpp 3 | * @author Suvarsha Chennareddy 4 | * 5 | * Check whether TransformationPolicyType is EmptyTransformation. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CMAES_NOT_EMPTY_TRANSFORMATION 13 | #define ENSMALLEN_CMAES_NOT_EMPTY_TRANSFORMATION 14 | 15 | /** 16 | * This partial specialization is used to throw an exception when the 17 | * TransformationPolicyType is EmptyTransformation and call a constructor with 18 | * parameters 'lowerBound' and 'upperBound' otherwise. This shall be removed 19 | * when the deprecated constructor is removed in the next major version of 20 | * ensmallen. 21 | */ 22 | template 23 | struct NotEmptyTransformation : std::true_type 24 | { 25 | void Assign(T1& obj, double lowerBound, double upperBound) 26 | { 27 | obj = T1(lowerBound, upperBound); 28 | } 29 | }; 30 | 31 | template class T, typename... A, typename... B> 32 | struct NotEmptyTransformation, T> : std::false_type 33 | { 34 | void Assign(T& /* obj */, 35 | double /* lowerBound */, 36 | double /* upperBound */) 37 | { 38 | throw std::logic_error("TransformationPolicyType is EmptyTransformation"); 39 | } 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cmaes/random_selection.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file random_selection.hpp 3 | * @author Marcus Edel 4 | * 5 | * Randomly select dataset points for use in the Evaluation step. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CMAES_RANDOM_SELECTION_HPP 13 | #define ENSMALLEN_CMAES_RANDOM_SELECTION_HPP 14 | 15 | namespace ens { 16 | 17 | /* 18 | * Randomly select dataset points for use in the Evaluation step. 19 | */ 20 | class RandomSelection 21 | { 22 | public: 23 | /** 24 | * Constructor for the random selection strategy. 25 | * 26 | * @param fraction The dataset fraction used for the selection (Default 0.3). 27 | */ 28 | RandomSelection(const double fraction = 0.3) : fraction(fraction) 29 | { 30 | // Nothing to do here. 31 | } 32 | 33 | //! Get the dataset fraction. 34 | double Fraction() const { return fraction; } 35 | //! Modify the dataset fraction. 36 | double& Fraction() { return fraction; } 37 | 38 | /** 39 | * Randomly select dataset points to calculate the objective function. 40 | * 41 | * @tparam SeparableFunctionType Type of the function to be evaluated. 42 | * @param function Function to optimize. 43 | * @param batchSize Batch size to use for each step. 44 | * @param terminate Whether optimization should be terminated after this call. 45 | * @param iterate starting point. 46 | */ 47 | template 50 | double Select(SeparableFunctionType& function, 51 | const size_t batchSize, 52 | const MatType& iterate, 53 | bool& terminate, 54 | CallbackTypes&... callbacks) 55 | { 56 | // Find the number of functions to use. 57 | const size_t numFunctions = function.NumFunctions(); 58 | 59 | typename MatType::elem_type objective = 0; 60 | for (size_t f = 0; f < std::floor(numFunctions * fraction); f += batchSize) 61 | { 62 | const size_t selection = arma::as_scalar(arma::randi( 63 | 1, arma::distr_param(0, numFunctions - 1))); 64 | const size_t effectiveBatchSize = std::min(batchSize, 65 | numFunctions - selection); 66 | 67 | objective += function.Evaluate(iterate, selection, effectiveBatchSize); 68 | 69 | terminate |= Callback::Evaluate(*this, f, iterate, objective, 70 | callbacks...); 71 | } 72 | 73 | return objective; 74 | } 75 | 76 | private: 77 | //! Dataset fraction parameter. 78 | double fraction; 79 | }; 80 | 81 | } // namespace ens 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file empty_transformation.hpp 3 | * @author Suvarsha Chennareddy 4 | * 5 | * Empty Transformation, can also be called an Indentity Transformation. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP 13 | #define ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * This is an empty transformation. As the name indicates, it does 19 | * not do anything. It is essentially an identity 20 | * transformation and is meant to be used when there are no 21 | * sorts of constraints on the coordinates. 22 | * 23 | * @tparam MatType The matrix type of the coordinates. 24 | */ 25 | template 26 | class EmptyTransformation 27 | { 28 | public: 29 | /** 30 | * Transforms coordinates to themselves (effectively no transformation). 31 | * 32 | * @param x Input coordinates. 33 | * @return Transformed coordinates (the coordinates themselves). 34 | */ 35 | MatType Transform(const MatType& x) { return x; } 36 | 37 | /** 38 | * Return a suitable initial step size. 39 | * 40 | * @return initial step size. 41 | */ 42 | typename MatType::elem_type InitialStepSize() { return 1; } 43 | }; 44 | 45 | } // namespace ens 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /include/ensmallen_bits/config.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file config.hpp 3 | * @author Conrad Sanderson 4 | * @author Marcus Edel 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | 12 | 13 | #if !defined(ENS_PRINT_INFO) 14 | // #define ENS_PRINT_INFO 15 | #endif 16 | 17 | #if !defined(ENS_PRINT_WARN) 18 | // #define ENS_PRINT_WARN 19 | #endif 20 | 21 | #if defined(ARMA_USE_OPENMP) 22 | #undef ENS_USE_OPENMP 23 | #define ENS_USE_OPENMP 24 | #endif 25 | 26 | 27 | // 28 | 29 | 30 | #if defined(ENS_DONT_PRINT_INFO) 31 | #undef ENS_PRINT_INFO 32 | #endif 33 | 34 | #if defined(ENS_DONT_PRINT_WARN) 35 | #undef ENS_PRINT_WARN 36 | #endif 37 | 38 | #if defined(ENS_DONT_USE_OPENMP) 39 | #undef ENS_USE_OPENMP 40 | #endif 41 | 42 | 43 | // 44 | 45 | 46 | #if defined(ENS_USE_OPENMP) 47 | #define ENS_PRAGMA_OMP_PARALLEL _Pragma("omp parallel") 48 | #define ENS_PRAGMA_OMP_ATOMIC _Pragma("omp atomic") 49 | #define ENS_PRAGMA_OMP_CRITICAL _Pragma("omp critical") 50 | #define ENS_PRAGMA_OMP_CRITICAL_NAMED _Pragma("omp critical(section)") 51 | #else 52 | #define ENS_PRAGMA_OMP_PARALLEL 53 | #define ENS_PRAGMA_OMP_ATOMIC 54 | #define ENS_PRAGMA_OMP_CRITICAL 55 | #define ENS_PRAGMA_OMP_CRITICAL_NAMED 56 | #endif 57 | 58 | 59 | // undefine conflicting macros 60 | #if defined(As) 61 | #pragma message ("WARNING: undefined conflicting 'As' macro") 62 | #undef As 63 | #endif 64 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ens_version.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ens_version.hpp 3 | * @author Conrad Sanderson 4 | * @author Ryan Curtin 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | 12 | 13 | // This follows the Semantic Versioning pattern defined in https://semver.org/. 14 | 15 | #define ENS_VERSION_MAJOR 2 16 | // The minor version is two digits so regular numerical comparisons of versions 17 | // work right. The first minor version of a release is always 10. 18 | #define ENS_VERSION_MINOR 22 19 | #define ENS_VERSION_PATCH 2 20 | // If this is a release candidate, it will be reflected in the version name 21 | // (i.e. the version name will be "RC1", "RC2", etc.). Otherwise the version 22 | // name will typically be a seemingly arbitrary set of words that does not 23 | // contain the capitalized string "RC". 24 | #define ENS_VERSION_NAME "E-Bike Excitement" 25 | // Incorporate the date the version was released. 26 | #define ENS_VERSION_YEAR "2025" 27 | #define ENS_VERSION_MONTH "04" 28 | #define ENS_VERSION_DAY "30" 29 | 30 | namespace ens { 31 | 32 | struct version 33 | { 34 | static const unsigned int major = ENS_VERSION_MAJOR; 35 | static const unsigned int minor = ENS_VERSION_MINOR; 36 | static const unsigned int patch = ENS_VERSION_PATCH; 37 | 38 | static inline std::string as_string() 39 | { 40 | const char* nickname = ENS_VERSION_NAME; 41 | 42 | std::stringstream ss; 43 | ss << version::major << '.' << version::minor << '.' << version::patch 44 | << " (" << nickname << ')'; 45 | 46 | return ss.str(); 47 | } 48 | 49 | static inline std::string date() 50 | { 51 | std::stringstream ss; 52 | ss << ENS_VERSION_YEAR << '-' << ENS_VERSION_MONTH << '-' << ENS_VERSION_DAY; 53 | 54 | return ss.str(); 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ftml/ftml_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ftml_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Follow the Moving Leader (FTML) optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_FTML_FTML_IMPL_HPP 13 | #define ENSMALLEN_FTML_FTML_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "ftml.hpp" 17 | 18 | namespace ens { 19 | 20 | inline FTML::FTML(const double stepSize, 21 | const size_t batchSize, 22 | const double beta1, 23 | const double beta2, 24 | const double epsilon, 25 | const size_t maxIterations, 26 | const double tolerance, 27 | const bool shuffle, 28 | const bool resetPolicy, 29 | const bool exactObjective) : 30 | optimizer(stepSize, 31 | batchSize, 32 | maxIterations, 33 | tolerance, 34 | shuffle, 35 | FTMLUpdate(epsilon, beta1, beta2), 36 | NoDecay(), 37 | resetPolicy, 38 | exactObjective) 39 | { /* Nothing to do. */ } 40 | 41 | } // namespace ens 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fw/func_sq.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file func_sq.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * Square loss function: \f$ x-> 0.5 * || Ax - b ||_2^2 \f$. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_FW_FUNC_SQ_HPP 13 | #define ENSMALLEN_FW_FUNC_SQ_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Square loss function \f$ f(x) = 0.5 * ||Ax - b||_2^2 \f$. 19 | * 20 | * Contains matrix \f$ A \f$ and vector \f$ b \f$. 21 | */ 22 | class FuncSq 23 | { 24 | public: 25 | /** 26 | * Construct the square loss function. 27 | * 28 | * @param A matrix A. 29 | * @param b vector b. 30 | */ 31 | FuncSq(const arma::mat& A, const arma::vec& b) : A(A), b(b) 32 | {/* Nothing to do. */} 33 | 34 | /** 35 | * Evaluation of the function. 36 | * \f$ f(x) = 0.5 * ||Ax - b||_2^2 \f$ 37 | * 38 | * @param coords vector x. 39 | * @return \f$ f(x) \f$. 40 | */ 41 | double Evaluate(const arma::mat& coords) 42 | { 43 | arma::vec r = A * coords - b; 44 | return arma::dot(r, r) * 0.5; 45 | } 46 | 47 | /** 48 | * Gradient of square loss function. 49 | * \f$ \nabla f(x) = A^T(Ax - b) \f$ 50 | * 51 | * @param coords input vector x. 52 | * @param gradient output gradient vector. 53 | */ 54 | void Gradient(const arma::mat& coords, arma::mat& gradient) 55 | { 56 | arma::vec r = A * coords - b; 57 | gradient = A.t() * r; 58 | } 59 | 60 | //! Get the matrix A. 61 | arma::mat MatrixA() const {return A;} 62 | //! Modify the matrix A. 63 | arma::mat& MatrixA() {return A;} 64 | 65 | //! Get the vector b. 66 | arma::vec Vectorb() const { return b; } 67 | //! Modify the vector b. 68 | arma::vec& Vectorb() { return b; } 69 | 70 | private: 71 | //! Matrix A in square loss function. 72 | arma::mat A; 73 | 74 | //! Vector b in square loss function. 75 | arma::vec b; 76 | }; 77 | 78 | } // namespace ens 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fw/proximal/proximal.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file proximal.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * Approximate a vector with another vector on lp ball. Currently support l0 6 | * ball and l1 ball with specific norm. 7 | * It can be used in projected gradient method. 8 | * 9 | * ensmallen is free software; you may redistribute it and/or modify it under 10 | * the terms of the 3-clause BSD license. You should have received a copy of 11 | * the 3-clause BSD license along with ensmallen. If not, see 12 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 13 | */ 14 | #ifndef ENSMALLEN_PROXIMAL_PROXIMAL_HPP 15 | #define ENSMALLEN_PROXIMAL_PROXIMAL_HPP 16 | 17 | namespace ens { 18 | 19 | /** 20 | * Approximate a vector with another vector on lp ball. Currently support l0 21 | * ball and l1 ball with specific norm. 22 | * It can be used in projected gradient method. 23 | */ 24 | class Proximal 25 | { 26 | public: 27 | /** 28 | * Project the vector onto the l1 ball with norm tau. That is, we will solve 29 | * for: 30 | * \f[ 31 | * w = argmin_w ||w - v||_2, \qquad s.t. ~ ||w||_1 \leqslant tau 32 | * \f] 33 | * 34 | * @param v Input vector to be approxmated, the output optimal vector is 35 | * also saved in v. 36 | * @param tau Norm of l1 ball. 37 | */ 38 | template 39 | static void ProjectToL1Ball(MatType& v, double tau); 40 | 41 | /** 42 | * Project the vector onto the l0 ball with norm tau. That is, we try to 43 | * approximate v with sparse vector w: 44 | * \f[ 45 | * w = argmin_w ||w - v||_2, \qquad s.t. ~ ||w||_0 \leqslant tau 46 | * \f] 47 | * 48 | * @param v Input vector to be approxmated, the output optimal vector is 49 | * also saved in v. 50 | * @param tau Norm of l0 ball. 51 | */ 52 | template 53 | static void ProjectToL0Ball(MatType& v, int tau); 54 | }; // class Proximal 55 | 56 | } // namespace ens 57 | 58 | #include "proximal_impl.hpp" 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fw/proximal/proximal_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file proximal_impl.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #ifndef ENSMALLEN_PROXIMAL_PROXIMAL_IMPL_HPP 11 | #define ENSMALLEN_PROXIMAL_PROXIMAL_IMPL_HPP 12 | 13 | #include "proximal.hpp" 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Projection of the vector v onto l1 ball with norm tau. 19 | * See the paper: 20 | * @code 21 | * @inproceedings{DucShaSin2008Efficient, 22 | * author = {Duchi, John and Shalev-Shwartz, Shai and Singer, 23 | * Yoram and Chandra, Tushar}, 24 | * booktitle = {Proceedings of the 25th international conference on 25 | * Machine learning}, 26 | * organization = {ACM}, 27 | * pages = {272--279}, 28 | * title = {Efficient projections onto the l 1-ball for learning in 29 | * high dimensions}, 30 | * year = {2008}} 31 | * @endcode 32 | * 33 | * This is just a soft thresholding. 34 | */ 35 | template 36 | inline void Proximal::ProjectToL1Ball(MatType& v, double tau) 37 | { 38 | MatType simplexSol = arma::abs(v); 39 | 40 | // Already with L1 norm <= tau. 41 | if (arma::accu(simplexSol) <= tau) 42 | return; 43 | 44 | simplexSol = arma::sort(simplexSol, "descend"); 45 | MatType simplexSum = arma::cumsum(simplexSol); 46 | 47 | double nu = 0; 48 | size_t rho = simplexSol.n_rows - 1; 49 | for (size_t j = 1; j <= simplexSol.n_rows; j++) 50 | { 51 | rho = simplexSol.n_rows - j; 52 | nu = simplexSol(rho) - (simplexSum(rho) - tau) / (rho + 1); 53 | if (nu > 0) 54 | break; 55 | } 56 | const double theta = (simplexSum(rho) - tau) / rho; 57 | 58 | // Threshold on absolute value of v with theta. 59 | for (arma::uword j = 0; j < simplexSol.n_rows; j++) 60 | { 61 | if (v(j) >= 0.0) 62 | v(j) = std::max(v(j) - theta, 0.0); 63 | else 64 | v(j) = std::min(v(j) + theta, 0.0); 65 | } 66 | } 67 | 68 | /** 69 | * Approximate the vector v with a tau-sparse vector. 70 | * This is a hard-thresholding. 71 | */ 72 | template 73 | inline void Proximal::ProjectToL0Ball(MatType& v, int tau) 74 | { 75 | arma::uvec indices = arma::sort_index(arma::abs(v)); 76 | arma::uword numberToKill = v.n_elem - tau; 77 | 78 | for (arma::uword i = 0; i < numberToKill; i++) 79 | v(indices(i)) = 0.0; 80 | } 81 | 82 | } // namespace ens 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fw/update_classic.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file update_classic.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * Classic update method for FrankWolfe algorithm. Used as UpdateRuleType. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_FW_UPDATE_CLASSIC_HPP 13 | #define ENSMALLEN_FW_UPDATE_CLASSIC_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Use classic rule in the update step for FrankWolfe algorithm. That is, 19 | * take \f$ \gamma = \frac{2}{k+2} \f$, where \f$ k \f$ is the iteration 20 | * number. The update rule would be: 21 | * \f[ 22 | * x_{k+1} = (1-\gamma) x_k + \gamma s 23 | * \f] 24 | * 25 | */ 26 | class UpdateClassic 27 | { 28 | public: 29 | /** 30 | * Construct the classic update rule for FrankWolfe algorithm. 31 | */ 32 | UpdateClassic() { /* Do nothing. */ } 33 | 34 | /** 35 | * Classic update rule for FrankWolfe. 36 | * 37 | * \f$ x_{k+1} = (1-\gamma)x_k + \gamma s \f$, where \f$ \gamma = 2/(k+2) \f$ 38 | * 39 | * @param function Function to be optimized, not used in this update rule. 40 | * @param oldCoords Previous solution coords. 41 | * @param s Current linear_constr_solution result. 42 | * @param newCoords Output new solution coords. 43 | * @param numIter Current iteration number. 44 | */ 45 | template 46 | void Update(FunctionType& /* function */, 47 | const MatType& oldCoords, 48 | const MatType& s, 49 | MatType& newCoords, 50 | const size_t numIter) 51 | { 52 | typename MatType::elem_type gamma = 2.0 / (numIter + 2.0); 53 | newCoords = (1.0 - gamma) * oldCoords + gamma * s; 54 | } 55 | }; 56 | 57 | } // namespace ens 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /include/ensmallen_bits/grid_search/grid_search.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file grid_search.hpp 3 | * @author Kirill Mishchenko 4 | * 5 | * Grid-search optimization. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_GRID_SEARCH_GRID_SEARCH_HPP 13 | #define ENSMALLEN_GRID_SEARCH_GRID_SEARCH_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * An optimizer that finds the minimum of a given function by iterating through 19 | * points on a multidimensional grid. 20 | * 21 | * GridSearch can optimize categorical functions. For more details, see the 22 | * documentation on function types included with this distribution or on the 23 | * ensmallen website. 24 | */ 25 | class GridSearch 26 | { 27 | public: 28 | /** 29 | * Optimize (minimize) the given function by iterating through the all 30 | * possible combinations of values for the parameters specified in 31 | * datasetInfo. 32 | * 33 | * @tparam FunctionType Type of function to optimize. 34 | * @tparam MatType Type of matrix to optimize with. 35 | * @param function Function to optimize. 36 | * @param bestParameters Variable for storing results. 37 | * @param categoricalDimensions Set of dimension types. If a value is true, 38 | * then that dimension is a categorical dimension. 39 | * @param numCategories Number of categories in each categorical dimension. 40 | * @return Objective value of the final point. 41 | */ 42 | template 43 | typename MatType::elem_type Optimize( 44 | FunctionType& function, 45 | MatType& bestParameters, 46 | const std::vector& categoricalDimensions, 47 | const arma::Row& numCategories); 48 | 49 | private: 50 | /** 51 | * Iterate through the last (parameterValueCollections.size() - i) dimensions 52 | * of the grid and change the arguments bestObjective and bestParameters if 53 | * there is something better. The values for the first i dimensions 54 | * (parameters) are specified in the first i rows of the currentParameters 55 | * argument. 56 | */ 57 | template 58 | void Optimize( 59 | FunctionType& function, 60 | typename MatType::elem_type& bestObjective, 61 | MatType& bestParameters, 62 | MatType& currentParameters, 63 | const std::vector& categoricalDimensions, 64 | const arma::Row& numCategories, 65 | size_t i); 66 | }; 67 | 68 | } // namespace ens 69 | 70 | // Include implementation 71 | #include "grid_search_impl.hpp" 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /include/ensmallen_bits/log.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file log.hpp 3 | * @author Marcus Edel 4 | * 5 | * Definition of the Info and Warn log functions. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_LOG_HPP 13 | #define ENSMALLEN_LOG_HPP 14 | 15 | namespace ens { 16 | /** 17 | * This class does nothing and should be optimized out entirely by the compiler. 18 | */ 19 | class NullOutStream 20 | { 21 | public: 22 | /** 23 | * Does nothing. 24 | */ 25 | NullOutStream() { } 26 | 27 | /** 28 | * Does nothing. 29 | */ 30 | NullOutStream(const NullOutStream& /* other */) { } 31 | 32 | //! Does nothing. 33 | NullOutStream& operator<<(bool) { return *this; } 34 | //! Does nothing. 35 | NullOutStream& operator<<(short) { return *this; } 36 | //! Does nothing. 37 | NullOutStream& operator<<(unsigned short) { return *this; } 38 | //! Does nothing. 39 | NullOutStream& operator<<(int) { return *this; } 40 | //! Does nothing. 41 | NullOutStream& operator<<(unsigned int) { return *this; } 42 | //! Does nothing. 43 | NullOutStream& operator<<(long) { return *this; } 44 | //! Does nothing. 45 | NullOutStream& operator<<(unsigned long) { return *this; } 46 | //! Does nothing. 47 | NullOutStream& operator<<(float) { return *this; } 48 | //! Does nothing. 49 | NullOutStream& operator<<(double) { return *this; } 50 | //! Does nothing. 51 | NullOutStream& operator<<(long double) { return *this; } 52 | //! Does nothing. 53 | NullOutStream& operator<<(void*) { return *this; } 54 | //! Does nothing. 55 | NullOutStream& operator<<(const char*) { return *this; } 56 | //! Does nothing. 57 | NullOutStream& operator<<(std::string&) { return *this; } 58 | //! Does nothing. 59 | NullOutStream& operator<<(std::streambuf*) { return *this; } 60 | //! Does nothing. 61 | NullOutStream& operator<<(std::ostream& (*) (std::ostream&)) { return *this; } 62 | //! Does nothing. 63 | NullOutStream& operator<<(std::ios& (*) (std::ios&)) { return *this; } 64 | //! Does nothing. 65 | NullOutStream& operator<<(std::ios_base& (*) (std::ios_base&)) 66 | { return *this; } 67 | 68 | //! Does nothing. 69 | template 70 | NullOutStream& operator<<(const T&) { return *this; } 71 | }; 72 | 73 | #ifdef ENS_PRINT_INFO 74 | static std::ostream& Info = arma::get_cout_stream(); 75 | #else 76 | static NullOutStream Info; 77 | #endif 78 | 79 | #ifdef ENS_PRINT_WARN 80 | static std::ostream& Warn = arma::get_cerr_stream(); 81 | #else 82 | static NullOutStream Warn; 83 | #endif 84 | 85 | } // namespace ens 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /include/ensmallen_bits/moead/decomposition_policies/tchebycheff_decomposition.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file tchebycheff_decomposition.hpp 3 | * @author Nanubala Gnana Sai 4 | * 5 | * The Tchebycheff Weight decomposition policy. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP 13 | #define ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The Tchebycheff method works by taking the maximum of element-wise product 19 | * between reference direction and the line connecting objective vector and 20 | * ideal point. 21 | * 22 | * Under mild conditions, for each Pareto Optimal point there exists a reference 23 | * direction such that the given point is also the optimal solution 24 | * to this scalar objective. 25 | * 26 | * For more information, see the following: 27 | * @code 28 | * article{zhang2007moea, 29 | * title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition}, 30 | * author={Zhang, Qingfu and Li, Hui}, 31 | * journal={IEEE Transactions on evolutionary computation}, 32 | * pages={712--731}, 33 | * year={2007} 34 | * @endcode 35 | */ 36 | class Tchebycheff 37 | { 38 | public: 39 | /** 40 | * Constructor for Tchebycheff decomposition policy. 41 | */ 42 | Tchebycheff() 43 | { 44 | /* Nothing to do. */ 45 | } 46 | 47 | /** 48 | * Decompose the weight vectors. 49 | * 50 | * @tparam VecType The type of the vector used in the decommposition. 51 | * @param weight The weight vector corresponding to a subproblem. 52 | * @param idealPoint The reference point in the objective space. 53 | * @param candidateFitness The objective vector of the candidate. 54 | */ 55 | template 56 | typename VecType::elem_type Apply(const VecType& weight, 57 | const VecType& idealPoint, 58 | const VecType& candidateFitness) 59 | { 60 | return arma::max(weight % arma::abs(candidateFitness - idealPoint)); 61 | } 62 | }; 63 | 64 | } // namespace ens 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /include/ensmallen_bits/moead/decomposition_policies/weighted_decomposition.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file weighted_decomposition.hpp 3 | * @author Nanubala Gnana Sai 4 | * 5 | * The Weighted Average decomposition policy. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOEAD_WEIGHTED_HPP 13 | #define ENSMALLEN_MOEAD_WEIGHTED_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The Weighted average method of decomposition. The working principle is to 19 | * minimize the dot product between reference direction and the line connecting 20 | * objective vector and ideal point. 21 | * 22 | * For more information, see the following: 23 | * @code 24 | * article{zhang2007moea, 25 | * title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition}, 26 | * author={Zhang, Qingfu and Li, Hui}, 27 | * journal={IEEE Transactions on evolutionary computation}, 28 | * pages={712--731}, 29 | * year={2007} 30 | * @endcode 31 | */ 32 | class WeightedAverage 33 | { 34 | public: 35 | /** 36 | * Constructor for Weighted Average decomposition policy. 37 | */ 38 | WeightedAverage() 39 | { 40 | /* Nothing to do. */ 41 | } 42 | 43 | /** 44 | * Decompose the weight vectors. 45 | * 46 | * @tparam VecType The type of the vector used in the decommposition. 47 | * @param weight The weight vector corresponding to a subproblem. 48 | * @param idealPoint The reference point in the objective space. 49 | * @param candidateFitness The objective vector of the candidate. 50 | */ 51 | template 52 | typename VecType::elem_type Apply(const VecType& weight, 53 | const VecType& /* idealPoint */, 54 | const VecType& candidateFitness) 55 | { 56 | return arma::dot(weight, candidateFitness); 57 | } 58 | }; 59 | 60 | } // namespace ens 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /include/ensmallen_bits/moead/weight_init_policies/bbs_init.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file bbs_init.hpp 3 | * @author Nanubala Gnana Sai 4 | * 5 | * The Bayesian Bootstrap (BBS) method of Weight Initialization. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOEAD_BBS_HPP 13 | #define ENSMALLEN_MOEAD_BBS_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The Bayesian Bootstrap method for initializing weights. Samples are randomly picked from uniform 19 | * distribution followed by sorting and finding adjacent difference. This gives you a list of 20 | * numbers which is guaranteed to sum up to 1. 21 | * 22 | * @code 23 | * @article{rubin1981bayesian, 24 | * title={The bayesian bootstrap}, 25 | * author={Rubin, Donald B}, 26 | * journal={The annals of statistics}, 27 | * pages={130--134}, 28 | * year={1981}, 29 | * @endcode 30 | * 31 | */ 32 | class BayesianBootstrap 33 | { 34 | public: 35 | /** 36 | * Constructor for Bayesian Bootstrap policy. 37 | */ 38 | BayesianBootstrap() 39 | { 40 | /* Nothing to do. */ 41 | } 42 | 43 | /** 44 | * Generate the reference direction matrix. 45 | * 46 | * @tparam MatType The type of the matrix used for constructing weights. 47 | * @param numObjectives The dimensionality of objective space. 48 | * @param numPoints The number of reference directions requested. 49 | * @param epsilon Handle numerical stability after weight initialization. 50 | */ 51 | template 52 | MatType Generate(const size_t numObjectives, 53 | const size_t numPoints, 54 | const double epsilon) 55 | { 56 | typedef typename MatType::elem_type ElemType; 57 | typedef typename arma::Col VecType; 58 | 59 | MatType weights(numObjectives, numPoints); 60 | for (size_t pointIdx = 0; pointIdx < numPoints; ++pointIdx) 61 | { 62 | VecType referenceDirection(numObjectives + 1, arma::fill::randu); 63 | referenceDirection(0) = 0; 64 | referenceDirection(numObjectives) = 1; 65 | referenceDirection = arma::sort(referenceDirection); 66 | referenceDirection = arma::diff(referenceDirection); 67 | weights.col(pointIdx) = std::move(referenceDirection) + epsilon; 68 | } 69 | 70 | return weights; 71 | } 72 | }; 73 | 74 | } // namespace ens 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /include/ensmallen_bits/moead/weight_init_policies/dirichlet_init.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file dirichlet_init.hpp 3 | * @author Nanubala Gnana Sai 4 | * 5 | * The Dirichlet method of Weight Initialization. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOEAD_DIRICHLET_HPP 13 | #define ENSMALLEN_MOEAD_DIRICHLET_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The Dirichlet method for initializing weights. Sampling a 19 | * Dirichlet distribution with parameters set to one returns 20 | * point lying on unit simplex with uniform distribution. 21 | */ 22 | class Dirichlet 23 | { 24 | public: 25 | /** 26 | * Constructor for Dirichlet policy. 27 | */ 28 | Dirichlet() 29 | { 30 | /* Nothing to do. */ 31 | } 32 | 33 | /** 34 | * Generate the reference direction matrix. 35 | * 36 | * @tparam MatType The type of the matrix used for constructing weights. 37 | * @param numObjectives The dimensionality of objective space. 38 | * @param numPoints The number of reference directions requested. 39 | * @param epsilon Handle numerical stability after weight initialization. 40 | */ 41 | template 42 | MatType Generate(const size_t numObjectives, 43 | const size_t numPoints, 44 | const double epsilon) 45 | { 46 | MatType weights = arma::randg(numObjectives, numPoints, 47 | arma::distr_param(1.0, 1.0)) + epsilon; 48 | // Normalize each column. 49 | return arma::normalise(weights, 1, 0); 50 | } 51 | }; 52 | 53 | } // namespace ens 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /include/ensmallen_bits/parallel_sgd/decay_policies/constant_step.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file constant_step.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Constant step size policy for parallel Stochastic Gradient Descent. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PARALLEL_SGD_CONSTANT_STEP_HPP 13 | #define ENSMALLEN_PARALLEL_SGD_CONSTANT_STEP_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Implementation of the ConstantStep stepsize decay policy for parallel SGD. 19 | */ 20 | class ConstantStep 21 | { 22 | public: 23 | /** 24 | * Member initialization constructor. 25 | * 26 | * The defaults here are not necessarily good for the given problem, so it is 27 | * suggested that the values used be tailored to the task at hand. 28 | * 29 | * @param step The intial stepsize to use. 30 | */ 31 | ConstantStep(const double step = 0.01) : step(step) { /* Nothing to do */ } 32 | 33 | /** 34 | * This function is called in each iteration before the gradient update. 35 | * 36 | * @param numEpoch The iteration number for which the stepsize is to be 37 | * calculated. 38 | * @return The step size for the current iteration. 39 | */ 40 | double StepSize(const size_t /* numEpoch */) 41 | { 42 | return step; 43 | } 44 | 45 | private: 46 | //! The initial stepsize, which remains unchanged. 47 | double step; 48 | }; 49 | 50 | } // namespace ens 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /include/ensmallen_bits/parallel_sgd/decay_policies/exponential_backoff.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file exponential_backoff.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Exponential backoff step size decay policy for parallel Stochastic Gradient 6 | * Descent. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_PARALLEL_SGD_EXP_BACKOFF_HPP 14 | #define ENSMALLEN_PARALLEL_SGD_EXP_BACKOFF_HPP 15 | 16 | namespace ens { 17 | 18 | /** 19 | * Exponential backoff stepsize reduction policy for parallel SGD. 20 | * 21 | * For more information, see the following. 22 | * 23 | * @misc{1106.5730, 24 | * Author = {Feng Niu and Benjamin Recht and Christopher Re and Stephen J. 25 | * Wright}, 26 | * Title = {HOGWILD!: A Lock-Free Approach to Parallelizing Stochastic 27 | * Gradient Descent}, 28 | * Year = {2011}, 29 | * Eprint = {arXiv:1106.5730}, 30 | * } 31 | * 32 | * This stepsize update scheme gives robust 1/k convergence rates to the 33 | * implementation of parallel SGD. 34 | */ 35 | class ExponentialBackoff 36 | { 37 | public: 38 | /** 39 | * Member initializer constructor to construct the exponential backoff policy 40 | * with the required parameters. 41 | * 42 | * @param firstBackoffEpoch The number of updates to run before the first 43 | * stepsize backoff. 44 | * @param step The initial stepsize(gamma). 45 | * @param beta The reduction factor. This should be a value in range (0, 1). 46 | */ 47 | ExponentialBackoff(const size_t firstBackoffEpoch, 48 | const double step, 49 | const double beta) : 50 | firstBackoffEpoch(firstBackoffEpoch), 51 | cutoffEpoch(firstBackoffEpoch), 52 | step(step), 53 | beta(beta) 54 | { /* Nothing to do. */ } 55 | 56 | /** 57 | * Get the step size for the current gradient update. 58 | * 59 | * @param numEpoch The iteration number of the current update. 60 | * @return The stepsize for the current iteration. 61 | */ 62 | double StepSize(const size_t numEpoch) 63 | { 64 | if (numEpoch >= cutoffEpoch) 65 | { 66 | step *= beta; 67 | cutoffEpoch += firstBackoffEpoch / beta; 68 | } 69 | return step; 70 | } 71 | 72 | private: 73 | //! The first iteration at which the stepsize should be reduced. 74 | size_t firstBackoffEpoch; 75 | 76 | //! The iteration at which the next decay will be performed. 77 | size_t cutoffEpoch; 78 | 79 | //! The initial stepsize. 80 | double step; 81 | 82 | //! The reduction factor, should be in range (0, 1). 83 | double beta; 84 | }; 85 | 86 | } // namespace ens 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/booth_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file booth_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Booth function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_BOOTH_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_BOOTH_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "booth_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline BoothFunction::BoothFunction() { /* Nothing to do here */ } 22 | 23 | inline void BoothFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type BoothFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const ElemType objective = std::pow(x1 + 2 * x2 - 7, 2) + 39 | std::pow(2 * x1 + x2 - 5, 2); 40 | 41 | return objective; 42 | } 43 | 44 | template 45 | typename MatType::elem_type BoothFunction::Evaluate( 46 | const MatType& coordinates) const 47 | { 48 | return Evaluate(coordinates, 0, NumFunctions()); 49 | } 50 | 51 | template 52 | inline void BoothFunction::Gradient(const MatType& coordinates, 53 | const size_t /* begin */, 54 | GradType& gradient, 55 | const size_t /* batchSize */) const 56 | { 57 | // Convenience typedef. 58 | typedef typename MatType::elem_type ElemType; 59 | 60 | // For convenience; we assume these temporaries will be optimized out. 61 | const ElemType x1 = coordinates(0); 62 | const ElemType x2 = coordinates(1); 63 | 64 | gradient.set_size(2, 1); 65 | gradient(0) = 10 * x1 + 8 * x2 - 34; 66 | gradient(1) = 8 * x1 + 10 * x2 - 38; 67 | } 68 | 69 | template 70 | inline void BoothFunction::Gradient(const MatType& coordinates, 71 | GradType& gradient) 72 | { 73 | Gradient(coordinates, 0, gradient, 1); 74 | } 75 | 76 | } // namespace test 77 | } // namespace ens 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/cross_in_tray_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file cross_in_tray_function_impl.hpp 3 | * @author Suryoday Basak 4 | * 5 | * Implementation of the Cross-in-Tray function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_CROSS_IN_TRAY_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_CROSS_IN_TRAY_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "cross_in_tray_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline CrossInTrayFunction::CrossInTrayFunction() { /* Nothing to do here */ } 22 | 23 | inline void CrossInTrayFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type CrossInTrayFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const ElemType objective = -0.0001 * std::pow(std::abs(std::sin(x1) * 39 | std::sin(x2) * std::exp(std::abs(100 - (std::sqrt(std::pow(x1, 2) + 40 | std::pow(x2, 2)) / arma::datum::pi))) + 1), 0.1); 41 | return objective; 42 | } 43 | 44 | template 45 | typename MatType::elem_type CrossInTrayFunction::Evaluate( 46 | const MatType& coordinates) const 47 | { 48 | return Evaluate(coordinates, 0, NumFunctions()); 49 | } 50 | 51 | } // namespace test 52 | } // namespace ens 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/fw_test_function.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file fw_test_function.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * Simple test function for classic Frank Wolfe Algorithm: 6 | * 7 | * \f$ f(x) = (x1 - 0.1)^2 + (x2 - 0.2)^2 + (x3 - 0.3)^2 \f$ 8 | * 9 | * ensmallen is free software; you may redistribute it and/or modify it under 10 | * the terms of the 3-clause BSD license. You should have received a copy of 11 | * the 3-clause BSD license along with ensmallen. If not, see 12 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 13 | */ 14 | #ifndef ENSMALLEN_PROBLEMS_FW_TEST_FUNCTION_HPP 15 | #define ENSMALLEN_PROBLEMS_FW_TEST_FUNCTION_HPP 16 | 17 | namespace ens { 18 | 19 | /** 20 | * Simple test function for classic Frank Wolfe Algorithm: 21 | * 22 | * \f$ f(x) = (x1 - 0.1)^2 + (x2 - 0.2)^2 + (x3 - 0.3)^2 \f$. 23 | */ 24 | template 25 | class TestFuncFW 26 | { 27 | public: 28 | TestFuncFW() {/* Nothing to do. */} 29 | 30 | /** 31 | * Evaluation of the function. 32 | * 33 | * @param coords input vector x. 34 | */ 35 | typename MatType::elem_type Evaluate(const MatType& coords) 36 | { 37 | typename MatType::elem_type f = std::pow(coords[0] - 0.1, 2); 38 | f += std::pow(coords[1] - 0.2, 2); 39 | f += std::pow(coords[2] - 0.3, 2); 40 | return f; 41 | } 42 | 43 | /** 44 | * Gradient of the function. 45 | * 46 | * @param coords input vector x. 47 | * @param gradient output gradient vector. 48 | */ 49 | void Gradient(const MatType& coords, GradType& gradient) 50 | { 51 | gradient.set_size(3, 1); 52 | gradient[0] = coords[0] - 0.1; 53 | gradient[1] = coords[1] - 0.2; 54 | gradient[2] = coords[2] - 0.3; 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/gradient_descent_test_function.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file gradient_descent_test_function.hpp 3 | * @author Sumedh Ghaisas 4 | * 5 | * Very simple test function for SGD. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_GRADIENT_DESCENT_TEST_FUNCTION_HPP 13 | #define ENSMALLEN_PROBLEMS_GRADIENT_DESCENT_TEST_FUNCTION_HPP 14 | 15 | namespace ens { 16 | namespace test { 17 | 18 | //! Very, very simple test function which is the composite of three other 19 | //! functions. The gradient is not very steep far away from the optimum, so a 20 | //! larger step size may be required to optimize it in a reasonable number of 21 | //! iterations. 22 | class GDTestFunction 23 | { 24 | public: 25 | //! Nothing to do for the constructor. 26 | GDTestFunction() { } 27 | 28 | //! Evaluate a function. 29 | template 30 | typename MatType::elem_type Evaluate(const MatType& coordinates) const; 31 | 32 | //! Evaluate the gradient of a function. 33 | template 34 | void Gradient(const MatType& coordinates, GradType& gradient) const; 35 | 36 | // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not 37 | // required for using ensmallen to optimize this function! They are 38 | // specifically used as a convenience just for ensmallen's testing 39 | // infrastructure. 40 | 41 | //! Get the starting point. 42 | template 43 | MatType GetInitialPoint() const { return MatType("1; 3; 2"); } 44 | 45 | //! Get the final point. 46 | template 47 | MatType GetFinalPoint() const { return MatType("0; 0; 0"); } 48 | 49 | //! Get the final objective. 50 | double GetFinalObjective() const { return 0.0; } 51 | }; 52 | 53 | } // namespace test 54 | } // namespace ens 55 | 56 | #include "gradient_descent_test_function_impl.hpp" 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/gradient_descent_test_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file gradient_descent_test_function_impl.hpp 3 | * @author Sumedh Ghaisas 4 | * 5 | * Implementation of very simple test function for gradient descent. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_GRADIENT_DESCENT_TEST_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_GRADIENT_DESCENT_TEST_FUNCTION_IMPL_HPP 14 | 15 | #include "gradient_descent_test_function.hpp" 16 | 17 | namespace ens { 18 | namespace test { 19 | 20 | template 21 | inline typename MatType::elem_type GDTestFunction::Evaluate( 22 | const MatType& coordinates) const 23 | { 24 | MatType temp = arma::trans(coordinates) * coordinates; 25 | return temp(0, 0); 26 | } 27 | 28 | template 29 | inline void GDTestFunction::Gradient(const MatType& coordinates, 30 | GradType& gradient) const 31 | { 32 | gradient = 2 * coordinates; 33 | } 34 | 35 | } // namespace test 36 | } // namespace ens 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/holder_table_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file holder_table_function_impl.hpp 3 | * @author Suryoday Basak 4 | * 5 | * Implementation of the Holder table function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_HOLDER_TABLE_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_HOLDER_TABLE_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "holder_table_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline HolderTableFunction::HolderTableFunction() { /* Nothing to do here */ } 22 | 23 | inline void HolderTableFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type HolderTableFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const ElemType objective = -std::abs(std::sin(x1) * std::cos(x2) * 39 | std::exp(std::abs(1 - (std::sqrt(x1 * x1 + x2 * x2) / arma::datum::pi)))); 40 | 41 | return objective; 42 | } 43 | 44 | template 45 | typename MatType::elem_type HolderTableFunction::Evaluate( 46 | const MatType& coordinates) const 47 | { 48 | return Evaluate(coordinates, 0, NumFunctions()); 49 | } 50 | 51 | } // namespace test 52 | } // namespace ens 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/matyas_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file matyas_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Matyas function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_MATYAS_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_MATYAS_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "matyas_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline MatyasFunction::MatyasFunction() { /* Nothing to do here */ } 22 | 23 | inline void MatyasFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type MatyasFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const double objective = 0.26 * (pow(x1, 2) + std::pow(x2, 2)) - 39 | 0.48 * x1 * x2; 40 | 41 | return objective; 42 | } 43 | 44 | template 45 | typename MatType::elem_type MatyasFunction::Evaluate( 46 | const MatType& coordinates) const 47 | { 48 | return Evaluate(coordinates, 0, NumFunctions()); 49 | } 50 | 51 | template 52 | inline void MatyasFunction::Gradient(const MatType& coordinates, 53 | const size_t /* begin */, 54 | GradType& gradient, 55 | const size_t /* batchSize */) const 56 | { 57 | // Convenience typedef. 58 | typedef typename MatType::elem_type ElemType; 59 | 60 | // For convenience; we assume these temporaries will be optimized out. 61 | const ElemType x1 = coordinates(0); 62 | const ElemType x2 = coordinates(1); 63 | 64 | gradient.set_size(2, 1); 65 | gradient(0) = 0.52 * x1 - 48 * x2; 66 | gradient(1) = 0.52 * x2 - 0.48 * x1; 67 | } 68 | 69 | template 70 | inline void MatyasFunction::Gradient(const MatType& coordinates, 71 | GradType& gradient) 72 | { 73 | Gradient(coordinates, 0, gradient, NumFunctions()); 74 | } 75 | 76 | } // namespace test 77 | } // namespace ens 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/mc_cormick_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file mc_cormick_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the McCormick function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_MC_CORMICK_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_MC_CORMICK_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "mc_cormick_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline McCormickFunction::McCormickFunction() { /* Nothing to do here */ } 22 | 23 | inline void McCormickFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type McCormickFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // For convenience; we assume these temporaries will be optimized out. 32 | const typename MatType::elem_type x1 = coordinates(0); 33 | const typename MatType::elem_type x2 = coordinates(1); 34 | 35 | const typename MatType::elem_type objective = std::sin(x1 + x2) + 36 | std::pow(x1 - x2, 2) - 1.5 * x1 + 2.5 * x2 + 1; 37 | 38 | return objective; 39 | } 40 | 41 | template 42 | typename MatType::elem_type McCormickFunction::Evaluate( 43 | const MatType& coordinates) const 44 | { 45 | return Evaluate(coordinates, 0, NumFunctions()); 46 | } 47 | 48 | template 49 | inline void McCormickFunction::Gradient(const MatType& coordinates, 50 | const size_t /* begin */, 51 | GradType& gradient, 52 | const size_t /* batchSize */) const 53 | { 54 | // For convenience; we assume these temporaries will be optimized out. 55 | const typename MatType::elem_type x1 = coordinates(0); 56 | const typename MatType::elem_type x2 = coordinates(1); 57 | 58 | gradient.set_size(2, 1); 59 | gradient(0) = std::cos(x1 + x2) + 2 * x1 - 2 * x2 - 1.5; 60 | gradient(1) = std::cos(x1 + x2) - 2 * x1 + 2 * x2 + 2.5; 61 | } 62 | 63 | template 64 | inline void McCormickFunction::Gradient(const MatType& coordinates, 65 | GradType& gradient) 66 | { 67 | Gradient(coordinates, 0, gradient, NumFunctions()); 68 | } 69 | 70 | } // namespace test 71 | } // namespace ens 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/problems.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file problems.hpp 3 | * @author Ryan Curtin 4 | * 5 | * Include each of the test problems for ensmallen. 6 | */ 7 | #ifndef ENSMALLEN_PROBLEMS_PROBLEMS_HPP 8 | #define ENSMALLEN_PROBLEMS_PROBLEMS_HPP 9 | 10 | #include "ackley_function.hpp" 11 | #include "aug_lagrangian_test_functions.hpp" 12 | #include "beale_function.hpp" 13 | #include "booth_function.hpp" 14 | #include "bukin_function.hpp" 15 | #include "colville_function.hpp" 16 | #include "cross_in_tray_function.hpp" 17 | #include "drop_wave_function.hpp" 18 | #include "easom_function.hpp" 19 | #include "eggholder_function.hpp" 20 | #include "fonseca_fleming_function.hpp" 21 | #include "fw_test_function.hpp" 22 | #include "generalized_rosenbrock_function.hpp" 23 | #include "goldstein_price_function.hpp" 24 | #include "gradient_descent_test_function.hpp" 25 | #include "himmelblau_function.hpp" 26 | #include "holder_table_function.hpp" 27 | #include "levy_function_n13.hpp" 28 | #include "logistic_regression_function.hpp" 29 | #include "matyas_function.hpp" 30 | #include "mc_cormick_function.hpp" 31 | #include "rastrigin_function.hpp" 32 | #include "rosenbrock_function.hpp" 33 | #include "rosenbrock_wood_function.hpp" 34 | #include "schaffer_function_n1.hpp" 35 | #include "schaffer_function_n2.hpp" 36 | #include "schaffer_function_n4.hpp" 37 | #include "schwefel_function.hpp" 38 | #include "sgd_test_function.hpp" 39 | #include "softmax_regression_function.hpp" 40 | #include "sparse_test_function.hpp" 41 | #include "sphere_function.hpp" 42 | #include "styblinski_tang_function.hpp" 43 | #include "three_hump_camel_function.hpp" 44 | #include "wood_function.hpp" 45 | #include "zdt/zdt1_function.hpp" 46 | #include "zdt/zdt2_function.hpp" 47 | #include "zdt/zdt3_function.hpp" 48 | #include "zdt/zdt4_function.hpp" 49 | #include "zdt/zdt6_function.hpp" 50 | #include "dtlz/dtlz1_function.hpp" 51 | #include "dtlz/dtlz2_function.hpp" 52 | #include "dtlz/dtlz3_function.hpp" 53 | #include "dtlz/dtlz4_function.hpp" 54 | #include "dtlz/dtlz5_function.hpp" 55 | #include "dtlz/dtlz6_function.hpp" 56 | #include "dtlz/dtlz7_function.hpp" 57 | #include "maf/maf1_function.hpp" 58 | #include "maf/maf2_function.hpp" 59 | #include "maf/maf3_function.hpp" 60 | #include "maf/maf4_function.hpp" 61 | #include "maf/maf5_function.hpp" 62 | #include "maf/maf6_function.hpp" 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/rosenbrock_wood_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file rosenbrock_wood_function_impl.hpp 3 | * @author Ryan Curtin 4 | * @author Marcus Edel 5 | * 6 | * Implementation of the Rosenbrock-Wood function. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | 14 | #ifndef ENSMALLEN_PROBLEMS_ROSENBROCK_WOOD_FUNCTION_IMPL_HPP 15 | #define ENSMALLEN_PROBLEMS_ROSENBROCK_WOOD_FUNCTION_IMPL_HPP 16 | 17 | // In case it hasn't been included yet. 18 | #include "rosenbrock_wood_function.hpp" 19 | 20 | namespace ens { 21 | namespace test { 22 | 23 | inline RosenbrockWoodFunction::RosenbrockWoodFunction() : rf(4), wf() 24 | { 25 | initialPoint.set_size(4, 2); 26 | initialPoint.col(0) = rf.GetInitialPoint(); 27 | initialPoint.col(1) = wf.GetInitialPoint(); 28 | } 29 | 30 | inline void RosenbrockWoodFunction::Shuffle() { /* Nothing to do here */ } 31 | 32 | template 33 | typename MatType::elem_type RosenbrockWoodFunction::Evaluate( 34 | const MatType& coordinates, 35 | const size_t /* begin */, 36 | const size_t /* batchSize */) const 37 | { 38 | return rf.Evaluate(coordinates.col(0)) + wf.Evaluate(coordinates.col(1)); 39 | } 40 | 41 | template 42 | typename MatType::elem_type RosenbrockWoodFunction::Evaluate( 43 | const MatType& coordinates) const 44 | { 45 | return Evaluate(coordinates, 0, NumFunctions()); 46 | } 47 | 48 | template 49 | inline void RosenbrockWoodFunction::Gradient(const MatType& coordinates, 50 | const size_t /* begin */, 51 | GradType& gradient, 52 | const size_t /* batchSize */) const 53 | { 54 | // Convenience typedef. 55 | typedef typename MatType::elem_type ElemType; 56 | 57 | gradient.set_size(4, 2); 58 | 59 | arma::Col grf(4); 60 | arma::Col gwf(4); 61 | 62 | rf.Gradient(coordinates.col(0), grf); 63 | wf.Gradient(coordinates.col(1), gwf); 64 | 65 | gradient.col(0) = grf; 66 | gradient.col(1) = gwf; 67 | } 68 | 69 | template 70 | inline void RosenbrockWoodFunction::Gradient(const MatType& coordinates, 71 | GradType& gradient) const 72 | { 73 | Gradient(coordinates, 0, gradient, 1); 74 | } 75 | 76 | } // namespace test 77 | } // namespace ens 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/schaffer_function_n1.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file schaffer_function_n1.hpp 3 | * @author Sayan Goswami 4 | * 5 | * Implementation of Schaffer function N.1. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #ifndef ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N1_HPP 14 | #define ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N1_HPP 15 | 16 | namespace ens { 17 | namespace test { 18 | 19 | /** 20 | * The Schaffer function N.1 is defined by 21 | * 22 | * \f[ 23 | * f_1(x) = x^2 24 | * f_2(x) = (x-2)^2 25 | * \f] 26 | * 27 | * The optimal solutions to this multi-objective function lie in the 28 | * range [0, 2]. 29 | * 30 | * @tparam arma::mat Type of matrix to optimize. 31 | */ 32 | template 33 | class SchafferFunctionN1 34 | { 35 | private: 36 | size_t numObjectives; 37 | size_t numVariables; 38 | 39 | public: 40 | //! Initialize the SchafferFunctionN1 41 | SchafferFunctionN1() : numObjectives(2), numVariables(1) 42 | {/* Nothing to do here. */} 43 | 44 | /** 45 | * Evaluate the objectives with the given coordinate. 46 | * 47 | * @param coords The function coordinates. 48 | * @return arma::Col 49 | */ 50 | arma::Col Evaluate(const MatType& coords) 51 | { 52 | // Convenience typedef. 53 | typedef typename MatType::elem_type ElemType; 54 | 55 | arma::Col objectives(numObjectives); 56 | 57 | objectives(0) = std::pow(coords[0], 2); 58 | objectives(1) = std::pow(coords[0] - 2, 2); 59 | 60 | return objectives; 61 | } 62 | 63 | //! Get the starting point. 64 | MatType GetInitialPoint() 65 | { 66 | // Convenience typedef. 67 | typedef typename MatType::elem_type ElemType; 68 | 69 | return arma::Col(numVariables, 1, arma::fill::zeros); 70 | } 71 | 72 | struct ObjectiveA 73 | { 74 | typename MatType::elem_type Evaluate(const MatType& coords) 75 | { 76 | return std::pow(coords[0], 2); 77 | } 78 | } objectiveA; 79 | 80 | struct ObjectiveB 81 | { 82 | typename MatType::elem_type Evaluate(const MatType& coords) 83 | { 84 | return std::pow(coords[0] - 2, 2); 85 | } 86 | } objectiveB; 87 | 88 | //! Get objective functions. 89 | std::tuple GetObjectives() 90 | { 91 | return std::make_tuple(objectiveA, objectiveB); 92 | } 93 | }; 94 | } // namespace test 95 | } // namespace ens 96 | 97 | #endif -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/schaffer_function_n4.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file schaffer_function_n4.hpp 3 | * @author Suryoday Basak 4 | * 5 | * Definition of Schaffer function N.4. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_HPP 13 | #define ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_HPP 14 | 15 | namespace ens { 16 | namespace test { 17 | 18 | /** 19 | * The Schaffer function N.4, defined by 20 | * 21 | * \f[ 22 | * f(x1, x2) = 0.5 + (cos^2(sin(|x1^2 - x2^2|)) - 0.5) / 23 | * (1 + 0.001 * (x1^2 + x2^2))^2 24 | * \f] 25 | * 26 | * This should optimize to f(x1, x2) = 0.292579, at (x1, x2) = [0, 1.25313], or 27 | * (x1, x2) = [0, -1.25313]. 28 | * 29 | * For more information, please refer to: 30 | * 31 | * @code 32 | * @misc{LevyFunction, 33 | * URL = {http://benchmarkfcns.xyz/benchmarkfcns/schaffern4fcn.html} 34 | * } 35 | * @endcode 36 | */ 37 | class SchafferFunctionN4 38 | { 39 | public: 40 | //! Initialize the SchafferFunctionN4. 41 | SchafferFunctionN4(); 42 | 43 | /** 44 | * Shuffle the order of function visitation. This may be called by the 45 | * optimizer. 46 | */ 47 | void Shuffle(); 48 | 49 | //! Return 1 (the number of functions). 50 | size_t NumFunctions() const { return 1; } 51 | 52 | //! Get the starting point. 53 | template 54 | MatType GetInitialPoint() const { return MatType("-5; 5"); } 55 | 56 | /** 57 | * Evaluate a function for a particular batch-size. 58 | * 59 | * @param coordinates The function coordinates. 60 | * @param begin The first function. 61 | * @param batchSize Number of points to process. 62 | */ 63 | template 64 | typename MatType::elem_type Evaluate(const MatType& coordinates, 65 | const size_t begin, 66 | const size_t batchSize) const; 67 | 68 | /** 69 | * Evaluate a function with the given coordinates. 70 | * 71 | * @param coordinates The function coordinates. 72 | */ 73 | template 74 | typename MatType::elem_type Evaluate(const MatType& coordinates) const; 75 | }; 76 | 77 | } // namespace test 78 | } // namespace ens 79 | 80 | // Include implementation. 81 | #include "schaffer_function_n4_impl.hpp" 82 | 83 | #endif // ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_HPP 84 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/schaffer_function_n4_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file schaffer_function_n4_impl.hpp 3 | * @author Suryoday Basak 4 | * 5 | * Implementation of Schaffer function N.4. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "schaffer_function_n4.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline SchafferFunctionN4::SchafferFunctionN4() { /* Nothing to do here */ } 22 | 23 | inline void SchafferFunctionN4::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type SchafferFunctionN4::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const ElemType objective = 0.5 + (std::pow(std::cos(std::sin(std::abs( 39 | std::pow(x1, 2) - std::pow(x2, 2)))), 2) - 0.5) / std::pow(1 + 0.001 * 40 | (std::pow(x1, 2) + std::pow(x2, 2)), 2); 41 | 42 | return objective; 43 | } 44 | 45 | template 46 | typename MatType::elem_type SchafferFunctionN4::Evaluate( 47 | const MatType& coordinates) const 48 | { 49 | return Evaluate(coordinates, 0, NumFunctions()); 50 | } 51 | 52 | } // namespace test 53 | } // namespace ens 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/schwefel_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file schwefel_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Schwefel function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SCHWEFEL_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_SCHWEFEL_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "schwefel_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline SchwefelFunction::SchwefelFunction(const size_t n) : 22 | n(n), 23 | visitationOrder(arma::linspace >(0, n - 1, n)) 24 | 25 | { 26 | initialPoint.set_size(n, 1); 27 | initialPoint.fill(-300); 28 | } 29 | 30 | inline void SchwefelFunction::Shuffle() 31 | { 32 | visitationOrder = arma::shuffle( 33 | arma::linspace >(0, n - 1, n)); 34 | } 35 | 36 | template 37 | typename MatType::elem_type SchwefelFunction::Evaluate( 38 | const MatType& coordinates, 39 | const size_t begin, 40 | const size_t batchSize) const 41 | { 42 | typename MatType::elem_type objective = 0; 43 | for (size_t j = begin; j < begin + batchSize; ++j) 44 | { 45 | const size_t p = visitationOrder[j]; 46 | objective += coordinates(p) * std::sin(std::sqrt(std::abs(coordinates(p)))); 47 | } 48 | objective -= 418.9829 * batchSize; 49 | 50 | return objective; 51 | } 52 | 53 | template 54 | typename MatType::elem_type SchwefelFunction::Evaluate( 55 | const MatType& coordinates) const 56 | { 57 | return Evaluate(coordinates, 0, NumFunctions()); 58 | } 59 | 60 | template 61 | inline void SchwefelFunction::Gradient(const MatType& coordinates, 62 | const size_t begin, 63 | GradType& gradient, 64 | const size_t batchSize) const 65 | { 66 | gradient.zeros(n, 1); 67 | 68 | for (size_t j = begin; j < begin + batchSize; ++j) 69 | { 70 | const size_t p = visitationOrder[j]; 71 | gradient(p) += (std::pow(coordinates(p), 2) * 72 | std::cos(std::sqrt(std::abs(coordinates(p)))) / 73 | (2 * std::pow(std::abs(coordinates(p)), 1.5)) + 74 | std::sin(std::sqrt(std::abs(coordinates(p))))); 75 | } 76 | } 77 | 78 | template 79 | inline void SchwefelFunction::Gradient(const MatType& coordinates, 80 | GradType& gradient) 81 | { 82 | Gradient(coordinates, 0, gradient, NumFunctions()); 83 | } 84 | 85 | } // namespace test 86 | } // namespace ens 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/sgd_test_function.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sgd_test_function.hpp 3 | * @author Ryan Curtin 4 | * 5 | * Very simple test function for SGD. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SGD_TEST_FUNCTION_HPP 13 | #define ENSMALLEN_PROBLEMS_SGD_TEST_FUNCTION_HPP 14 | 15 | namespace ens { 16 | namespace test { 17 | 18 | //! Very, very simple test function which is the composite of three other 19 | //! functions. The gradient is not very steep far away from the optimum, so a 20 | //! larger step size may be required to optimize it in a reasonable number of 21 | //! iterations. 22 | class SGDTestFunction 23 | { 24 | private: 25 | arma::Col visitationOrder; 26 | 27 | public: 28 | //! Initialize the SGDTestFunction. 29 | SGDTestFunction(); 30 | 31 | /** 32 | * Shuffle the order of function visitation. This may be called by the 33 | * optimizer. 34 | */ 35 | void Shuffle(); 36 | 37 | //! Return 3 (the number of functions). 38 | size_t NumFunctions() const { return 3; } 39 | 40 | //! Evaluate a function for a particular batch-size. 41 | template 42 | typename MatType::elem_type Evaluate(const MatType& coordinates, 43 | const size_t begin, 44 | const size_t batchSize) const; 45 | 46 | //! Evaluate the gradient of a function for a particular batch-size 47 | template 48 | void Gradient(const MatType& coordinates, 49 | const size_t begin, 50 | GradType& gradient, 51 | const size_t batchSize) const; 52 | 53 | // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not 54 | // required for using ensmallen to optimize this function! They are 55 | // specifically used as a convenience just for ensmallen's testing 56 | // infrastructure. 57 | 58 | //! Get the starting point. 59 | template 60 | MatType GetInitialPoint() const { return MatType("6; -45.6; 6.2"); } 61 | 62 | //! Get the final point. 63 | template 64 | MatType GetFinalPoint() const { return MatType("0.0; 0.0; 0.0"); } 65 | 66 | //! Get the final objective. 67 | double GetFinalObjective() const { return -1.0; } 68 | }; 69 | 70 | } // namespace test 71 | } // namespace ens 72 | 73 | // Include implementation. 74 | #include "sgd_test_function_impl.hpp" 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/sgd_test_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sgd_test_function_impl.hpp 3 | * @author Ryan Curtin 4 | * 5 | * Implementation of very simple test function for stochastic gradient descent 6 | * (SGD). 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_PROBLEMS_SGD_TEST_FUNCTION_IMPL_HPP 14 | #define ENSMALLEN_PROBLEMS_SGD_TEST_FUNCTION_IMPL_HPP 15 | 16 | // In case it hasn't been included yet. 17 | #include "sgd_test_function.hpp" 18 | 19 | namespace ens { 20 | namespace test { 21 | 22 | inline SGDTestFunction::SGDTestFunction() : 23 | visitationOrder(arma::linspace>(0, NumFunctions() - 1, 24 | NumFunctions())) 25 | { } 26 | 27 | inline void SGDTestFunction::Shuffle() 28 | { 29 | visitationOrder = arma::shuffle(arma::linspace >(0, 30 | (NumFunctions() - 1), NumFunctions())); 31 | } 32 | 33 | template 34 | typename MatType::elem_type SGDTestFunction::Evaluate( 35 | const MatType& coordinates, 36 | const size_t begin, 37 | const size_t batchSize) const 38 | { 39 | typename MatType::elem_type objective = 0; 40 | 41 | for (size_t i = begin; i < begin + batchSize; i++) 42 | { 43 | switch (visitationOrder(i)) 44 | { 45 | case 0: 46 | objective -= std::exp(-std::abs(coordinates[0])); 47 | break; 48 | 49 | case 1: 50 | objective += std::pow(coordinates[1], 2); 51 | break; 52 | 53 | case 2: 54 | objective += std::pow(coordinates[2], 4) + \ 55 | 3 * std::pow(coordinates[2], 2); 56 | break; 57 | } 58 | } 59 | 60 | return objective; 61 | } 62 | 63 | template 64 | void SGDTestFunction::Gradient(const MatType& coordinates, 65 | const size_t begin, 66 | GradType& gradient, 67 | const size_t batchSize) const 68 | { 69 | gradient.zeros(3); 70 | 71 | for (size_t i = begin; i < begin + batchSize; ++i) 72 | { 73 | switch (visitationOrder(i)) 74 | { 75 | case 0: 76 | if (coordinates[0] >= 0) 77 | gradient[0] += std::exp(-coordinates[0]); 78 | else 79 | gradient[0] += -std::exp(coordinates[0]); 80 | break; 81 | 82 | case 1: 83 | gradient[1] += 2 * coordinates[1]; 84 | break; 85 | 86 | case 2: 87 | gradient[2] += 4 * std::pow(coordinates[2], 3) + 6 * coordinates[2]; 88 | break; 89 | } 90 | } 91 | } 92 | 93 | } // namespace test 94 | } // namespace ens 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/sparse_test_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sparse_test_function_impl.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Sparse test function for Parallel SGD. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SPARSE_TEST_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_SPARSE_TEST_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "sparse_test_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline SparseTestFunction::SparseTestFunction() 22 | { 23 | intercepts = arma::vec("20 12 15 100"); 24 | bi = arma::vec("-4 -2 -3 -8"); 25 | } 26 | 27 | //! Evaluate a function. 28 | template 29 | inline typename MatType::elem_type SparseTestFunction::Evaluate( 30 | const MatType& coordinates, 31 | const size_t i, 32 | const size_t batchSize) const 33 | { 34 | typename MatType::elem_type result = 0.0; 35 | for (size_t j = i; j < i + batchSize; ++j) 36 | { 37 | result += coordinates[j] * coordinates[j] + bi[j] * coordinates[j] + 38 | intercepts[j]; 39 | } 40 | 41 | return result; 42 | } 43 | 44 | //! Evaluate all the functions. 45 | template 46 | inline typename MatType::elem_type SparseTestFunction::Evaluate( 47 | const MatType& coordinates) const 48 | { 49 | typename MatType::elem_type objective = 0.0; 50 | for (size_t i = 0; i < NumFunctions(); ++i) 51 | { 52 | objective += coordinates[i] * coordinates[i] + bi[i] * coordinates[i] + 53 | intercepts[i]; 54 | } 55 | 56 | return objective; 57 | } 58 | 59 | //! Evaluate the gradient of a function. 60 | template 61 | inline void SparseTestFunction::Gradient(const MatType& coordinates, 62 | const size_t i, 63 | GradType& gradient, 64 | const size_t batchSize) const 65 | { 66 | gradient.zeros(arma::size(coordinates)); 67 | for (size_t j = i; j < i + batchSize; ++j) 68 | gradient[j] = 2 * coordinates[j] + bi[j]; 69 | } 70 | 71 | //! Evaluate the gradient of a feature function. 72 | template 73 | inline void SparseTestFunction::PartialGradient(const MatType& coordinates, 74 | const size_t j, 75 | GradType& gradient) const 76 | { 77 | gradient.zeros(arma::size(coordinates)); 78 | gradient[j] = 2 * coordinates[j] + bi[j]; 79 | } 80 | 81 | } // namespace test 82 | } // namespace ens 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/sphere_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sphere_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Sphere function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SPHERE_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_SPHERE_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "sphere_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline SphereFunction::SphereFunction(const size_t n) : 22 | n(n), 23 | visitationOrder(arma::linspace >(0, n - 1, n)) 24 | { 25 | initialPoint.set_size(n, 1); 26 | 27 | for (size_t i = 0; i < n; ++i) // Set to [-5 5 -5 5 -5 5...]. 28 | { 29 | if (i % 2 == 1) 30 | initialPoint(i) = 5; 31 | else 32 | initialPoint(i) = -5; 33 | } 34 | } 35 | 36 | inline void SphereFunction::Shuffle() 37 | { 38 | visitationOrder = arma::shuffle( 39 | arma::linspace >(0, n - 1, n)); 40 | } 41 | 42 | template 43 | typename MatType::elem_type SphereFunction::Evaluate( 44 | const MatType& coordinates, 45 | const size_t begin, 46 | const size_t batchSize) const 47 | { 48 | typename MatType::elem_type objective = 0.0; 49 | for (size_t j = begin; j < begin + batchSize; ++j) 50 | { 51 | const size_t p = visitationOrder[j]; 52 | objective += std::pow(coordinates(p), 2); 53 | } 54 | 55 | return objective; 56 | } 57 | 58 | template 59 | typename MatType::elem_type SphereFunction::Evaluate( 60 | const MatType& coordinates) const 61 | { 62 | return Evaluate(coordinates, 0, NumFunctions()); 63 | } 64 | 65 | template 66 | void SphereFunction::Gradient(const MatType& coordinates, 67 | const size_t begin, 68 | GradType& gradient, 69 | const size_t batchSize) const 70 | { 71 | gradient.zeros(n, 1); 72 | 73 | for (size_t j = begin; j < begin + batchSize; ++j) 74 | { 75 | const size_t p = visitationOrder[j]; 76 | gradient(p) += 2.0 * coordinates[p]; 77 | } 78 | } 79 | 80 | template 81 | void SphereFunction::Gradient(const MatType& coordinates, 82 | GradType& gradient) 83 | { 84 | Gradient(coordinates, 0, gradient, NumFunctions()); 85 | } 86 | 87 | } // namespace test 88 | } // namespace ens 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/styblinski_tang_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file styblinski_tang_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Styblinski-Tang function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_STYBLINSKI_TANG_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_STYBLINSKI_TANG_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "styblinski_tang_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline StyblinskiTangFunction::StyblinskiTangFunction(const size_t n) : 22 | n(n), 23 | visitationOrder(arma::linspace >(0, n - 1, n)) 24 | 25 | { 26 | initialPoint.set_size(n, 1); 27 | initialPoint.fill(-5); 28 | } 29 | 30 | inline void StyblinskiTangFunction::Shuffle() 31 | { 32 | visitationOrder = arma::shuffle( 33 | arma::linspace >(0, n - 1, n)); 34 | } 35 | 36 | template 37 | typename MatType::elem_type StyblinskiTangFunction::Evaluate( 38 | const MatType& coordinates, 39 | const size_t begin, 40 | const size_t batchSize) const 41 | { 42 | typename MatType::elem_type objective = 0.0; 43 | for (size_t j = begin; j < begin + batchSize; ++j) 44 | { 45 | const size_t p = visitationOrder[j]; 46 | objective += std::pow(coordinates(p), 4) - 16 * 47 | std::pow(coordinates(p), 2) + 5 * coordinates(p); 48 | } 49 | objective /= 2; 50 | 51 | return objective; 52 | } 53 | 54 | template 55 | typename MatType::elem_type StyblinskiTangFunction::Evaluate( 56 | const MatType& coordinates) const 57 | { 58 | return Evaluate(coordinates, 0, NumFunctions()); 59 | } 60 | 61 | template 62 | void StyblinskiTangFunction::Gradient(const MatType& coordinates, 63 | const size_t begin, 64 | GradType& gradient, 65 | const size_t batchSize) const 66 | { 67 | gradient.zeros(n, 1); 68 | 69 | for (size_t j = begin; j < begin + batchSize; ++j) 70 | { 71 | const size_t p = visitationOrder[j]; 72 | gradient(p) += 0.5 * (4 * std::pow(coordinates(p), 3) - 73 | 32.0 * coordinates(p) + 5.0); 74 | } 75 | } 76 | 77 | template 78 | void StyblinskiTangFunction::Gradient(const MatType& coordinates, 79 | GradType& gradient) 80 | { 81 | Gradient(coordinates, 0, gradient, NumFunctions()); 82 | } 83 | 84 | } // namespace test 85 | } // namespace ens 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /include/ensmallen_bits/qhadam/qhadam_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file qhadam_impl.hpp 3 | * @author Niteya Shah 4 | * 5 | * Implementation of QHAdam class wrapper. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADAM_QHADAM_IMPL_HPP 13 | #define ENSMALLEN_ADAM_QHADAM_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "qhadam.hpp" 17 | 18 | namespace ens { 19 | 20 | inline QHAdam::QHAdam( 21 | const double stepSize, 22 | const size_t batchSize, 23 | const double v1, 24 | const double v2, 25 | const double beta1, 26 | const double beta2, 27 | const double epsilon, 28 | const size_t maxIterations, 29 | const double tolerance, 30 | const bool shuffle, 31 | const bool resetPolicy, 32 | const bool exactObjective) : 33 | optimizer(stepSize, 34 | batchSize, 35 | maxIterations, 36 | tolerance, 37 | shuffle, 38 | QHAdamUpdate(epsilon, beta1, beta2, v1, v2), 39 | NoDecay(), 40 | resetPolicy, 41 | exactObjective) 42 | { /* Nothing to do. */ } 43 | 44 | } // namespace ens 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sa/exponential_schedule.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file exponential_schedule.hpp 3 | * @author Zhihao Lou 4 | * 5 | * Exponential (geometric) cooling schedule used in SA. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SA_EXPONENTIAL_SCHEDULE_HPP 13 | #define ENSMALLEN_SA_EXPONENTIAL_SCHEDULE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The exponential cooling schedule cools the temperature T at every step 19 | * according to the equation 20 | * 21 | * \f[ 22 | * T_{n+1} = (1-\lambda) T_{n} 23 | * \f] 24 | * 25 | * where \f$ 0<\lambda<1 \f$ is the cooling speed. The smaller \f$ \lambda \f$ 26 | * is, the slower the cooling speed, and better the final result will be. Some 27 | * literature uses \f$ \alpha = (-1 \lambda) \f$ instead. In practice, 28 | * \f$ \alpha \f$ is very close to 1 and will be awkward to input (e.g. 29 | * alpha = 0.999999 vs lambda = 1e-6). 30 | */ 31 | class ExponentialSchedule 32 | { 33 | public: 34 | /* 35 | * Construct the ExponentialSchedule with the given parameter. 36 | * 37 | * @param lambda Cooling speed. 38 | */ 39 | ExponentialSchedule(const double lambda = 0.001) : lambda(lambda) { } 40 | 41 | /** 42 | * Returns the next temperature given current status. The current system's 43 | * energy is not used in this calculation. 44 | * 45 | * @param currentTemperature Current temperature of system. 46 | * @param currentEnergy Current energy of system (not used). 47 | */ 48 | template 49 | double NextTemperature( 50 | const double currentTemperature, 51 | const ElemType /* currentEnergy */) 52 | { 53 | return (1 - lambda) * currentTemperature; 54 | } 55 | 56 | //! Get the cooling speed, lambda. 57 | double Lambda() const { return lambda; } 58 | //! Modify the cooling speed, lambda. 59 | double& Lambda() { return lambda; } 60 | 61 | private: 62 | //! The cooling speed. 63 | double lambda; 64 | }; 65 | 66 | } // namespace ens 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sarah/sarah_plus_update.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sarah_plus_update.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the SARAH+ update rule which provides an automatic and 6 | * adaptive choice of the inner loop size. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_SARAH_SARAH_PLUS_UPDATE_HPP 14 | #define ENSMALLEN_SARAH_SARAH_PLUS_UPDATE_HPP 15 | 16 | namespace ens { 17 | 18 | /** 19 | * SARAH+ provides an automatic and adaptive choice of the inner loop size. 20 | */ 21 | class SARAHPlusUpdate 22 | { 23 | public: 24 | /* 25 | * Construct the SARAH+ update policy. 26 | * 27 | * @param gamma Adaptive parameter for the inner loop. 28 | */ 29 | SARAHPlusUpdate(const double gamma = 0.125) : gamma(gamma) 30 | { 31 | /* Nothing to do here. */ 32 | } 33 | 34 | /** 35 | * Update step for SARAH+. The function parameters are updated in the negative 36 | * direction of the gradient. 37 | * 38 | * @param iterate Parameters that minimize the function. 39 | * @param v Unbiased estimator of the gradient. 40 | * @param gradient The current gradient matrix at time t. 41 | * @param gradient0 The old gradient matrix at time t - 1. 42 | * @param batchSize Batch size to be used for the given iteration. 43 | * @param stepSize Step size to be used for the given iteration. 44 | * @param vNorm The norm of the full gradient. 45 | */ 46 | template 47 | bool Update(MatType& iterate, 48 | GradType& v, 49 | const GradType& gradient, 50 | const GradType& gradient0, 51 | const size_t batchSize, 52 | const double stepSize, 53 | const double vNorm) 54 | { 55 | v += (gradient - gradient0) / (double) batchSize; 56 | iterate -= stepSize * v; 57 | 58 | if (arma::norm(v) <= gamma * vNorm) 59 | return true; 60 | 61 | return false; 62 | } 63 | 64 | private: 65 | //! Adaptive parameter for the inner loop. 66 | double gamma; 67 | }; 68 | 69 | } // namespace ens 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sarah/sarah_update.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file svrg_update.hpp 3 | * @author Marcus Edel 4 | * 5 | * Vanilla update for SARAH. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SARAH_SARAH_UPDATE_HPP 13 | #define ENSMALLEN_SARAH_SARAH_UPDATE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Vanilla update policy for SARAH. 19 | */ 20 | class SARAHUpdate 21 | { 22 | public: 23 | /** 24 | * Update step for SARAH. The function parameters are updated in the negative 25 | * direction of the gradient. 26 | * 27 | * @param iterate Parameters that minimize the function. 28 | * @param v Unbiased estimator of the gradient. 29 | * @param gradient The current gradient matrix at time t. 30 | * @param gradient0 The old gradient matrix at time t - 1. 31 | * @param batchSize Batch size to be used for the given iteration. 32 | * @param stepSize Step size to be used for the given iteration. 33 | * @param vNorm The norm of the full gradient. 34 | */ 35 | template 36 | bool Update(MatType& iterate, 37 | GradType& v, 38 | const GradType& gradient, 39 | const GradType& gradient0, 40 | const size_t batchSize, 41 | const double stepSize, 42 | const double /* vNorm */) 43 | { 44 | v += (gradient - gradient0) / (double) batchSize; 45 | iterate -= stepSize * v; 46 | return false; 47 | } 48 | }; 49 | 50 | } // namespace ens 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sdp/lrsdp_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file lrsdp.cpp 3 | * @author Ryan Curtin 4 | * 5 | * An implementation of Monteiro and Burer's formulation of low-rank 6 | * semidefinite programs (LR-SDP). 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_SDP_LRSDP_IMPL_HPP 14 | #define ENSMALLEN_SDP_LRSDP_IMPL_HPP 15 | 16 | #include "lrsdp.hpp" 17 | 18 | namespace ens { 19 | 20 | template 21 | LRSDP::LRSDP(const size_t numSparseConstraints, 22 | const size_t numDenseConstraints, 23 | const arma::Mat& initialPoint, 24 | const size_t maxIterations) : 25 | function(numSparseConstraints, numDenseConstraints, initialPoint), 26 | maxIterations(maxIterations) 27 | { } 28 | 29 | template 30 | template 31 | typename MatType::elem_type LRSDP::Optimize( 32 | MatType& coordinates, CallbackTypes&&... callbacks) 33 | { 34 | function.RRTAny().Clean(); 35 | function.RRTAny().template Set( 36 | new MatType(coordinates * coordinates.t())); 37 | 38 | augLag.Sigma() = 10; 39 | augLag.MaxIterations() = maxIterations; 40 | augLag.Optimize(function, coordinates, callbacks...); 41 | 42 | return function.Evaluate(coordinates); 43 | } 44 | 45 | } // namespace ens 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sgd/update_policies/vanilla_update.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file vanilla_update.hpp 3 | * @author Arun Reddy 4 | * 5 | * Vanilla update for Stochastic Gradient Descent. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SGD_EMPTY_UPDATE_HPP 13 | #define ENSMALLEN_SGD_EMPTY_UPDATE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Vanilla update policy for Stochastic Gradient Descent (SGD). The following 19 | * update scheme is used to update SGD in every iteration: 20 | * 21 | * \f[ 22 | * A_{j + 1} = A_j + \alpha \nabla f_i(A) 23 | * \f] 24 | * 25 | * where \f$ \alpha \f$ is a parameter which specifies the step size. \f$ i \f$ 26 | * is chosen according to \f$ j \f$ (the iteration number). 27 | */ 28 | class VanillaUpdate 29 | { 30 | public: 31 | /** 32 | * The UpdatePolicyType policy classes must contain an internal 'Policy' 33 | * template class with two template arguments: MatType and GradType. This is 34 | * instantiated at the start of the optimization. 35 | */ 36 | template 37 | class Policy 38 | { 39 | public: 40 | /** 41 | * This is called by the optimizer method before the start of the iteration 42 | * update process. The vanilla update doesn't initialize anything. 43 | * 44 | * @param parent Instantiated parent class. 45 | * @param rows Number of rows in the gradient matrix. 46 | * @param cols Number of columns in the gradient matrix. 47 | */ 48 | Policy(const VanillaUpdate& /* parent */, 49 | const size_t /* rows */, 50 | const size_t /* cols */) 51 | { /* Do nothing. */ } 52 | 53 | /** 54 | * Update step for SGD. The function parameters are updated in the negative 55 | * direction of the gradient. 56 | * 57 | * @param iterate Parameters that minimize the function. 58 | * @param stepSize Step size to be used for the given iteration. 59 | * @param gradient The gradient matrix. 60 | */ 61 | void Update(MatType& iterate, 62 | const double stepSize, 63 | const GradType& gradient) 64 | { 65 | // Perform the vanilla SGD update. 66 | iterate -= stepSize * gradient; 67 | } 68 | }; 69 | }; 70 | 71 | } // namespace ens 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sgdr/sgdr_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sgdr_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of SGDR method. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SGDR_SGDR_IMPL_HPP 13 | #define ENSMALLEN_SGDR_SGDR_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "sgdr.hpp" 17 | 18 | namespace ens { 19 | 20 | template 21 | SGDR::SGDR( 22 | const size_t epochRestart, 23 | const double multFactor, 24 | const size_t batchSize, 25 | const double stepSize, 26 | const size_t maxIterations, 27 | const double tolerance, 28 | const bool shuffle, 29 | const UpdatePolicyType& updatePolicy, 30 | const bool resetPolicy, 31 | const bool exactObjective) : 32 | batchSize(batchSize), 33 | optimizer(OptimizerType(stepSize, 34 | batchSize, 35 | maxIterations, 36 | tolerance, 37 | shuffle, 38 | updatePolicy, 39 | CyclicalDecay( 40 | epochRestart, 41 | multFactor, 42 | stepSize), 43 | resetPolicy, 44 | exactObjective)) 45 | { 46 | /* Nothing to do here */ 47 | } 48 | 49 | template 50 | template 52 | typename std::enable_if::value, 53 | typename MatType::elem_type>::type 54 | SGDR::Optimize( 55 | SeparableFunctionType& function, 56 | MatType& iterate, 57 | CallbackTypes&&... callbacks) 58 | { 59 | // If a user changed the step size he hasn't update the step size of the 60 | // cyclical decay instantiation, so we have to do it here. 61 | if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize()) 62 | { 63 | optimizer.DecayPolicy().StepSize() = optimizer.StepSize(); 64 | } 65 | 66 | optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() / 67 | double(optimizer.BatchSize()); 68 | 69 | // If a user changed the batch size we have to update the restart fraction 70 | // of the cyclical decay instantiation. 71 | if (optimizer.BatchSize() != batchSize) 72 | { 73 | batchSize = optimizer.BatchSize(); 74 | } 75 | 76 | return optimizer.Optimize(function, iterate, callbacks...); 77 | } 78 | 79 | } // namespace ens 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /include/ensmallen_bits/smorms3/smorms3_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file smorms3_impl.hpp 3 | * @author Vivek Pal 4 | * 5 | * Implementation of the SMORMS3 constructor. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SMORMS3_SMORMS3_IMPL_HPP 13 | #define ENSMALLEN_SMORMS3_SMORMS3_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "smorms3.hpp" 17 | 18 | namespace ens { 19 | 20 | inline SMORMS3::SMORMS3(const double stepSize, 21 | const size_t batchSize, 22 | const double epsilon, 23 | const size_t maxIterations, 24 | const double tolerance, 25 | const bool shuffle, 26 | const bool resetPolicy, 27 | const bool exactObjective) : 28 | optimizer(stepSize, 29 | batchSize, 30 | maxIterations, 31 | tolerance, 32 | shuffle, 33 | SMORMS3Update(epsilon), 34 | NoDecay(), 35 | resetPolicy, 36 | exactObjective) 37 | { /* Nothing to do. */ } 38 | 39 | } // namespace ens 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /include/ensmallen_bits/svrg/svrg_update.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file svrg_update.hpp 3 | * @author Marcus Edel 4 | * 5 | * Vanilla update for stochastic variance reduced gradient (SVRG). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SVRG_SVRG_UPDATE_HPP 13 | #define ENSMALLEN_SVRG_SVRG_UPDATE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Vanilla update policy for Stochastic variance reduced gradient (SVRG). 19 | * The following update scheme is used to update SGD in every iteration: 20 | */ 21 | class SVRGUpdate 22 | { 23 | public: 24 | /** 25 | * The UpdatePolicyType policy classes must contain an internal 'Policy' 26 | * template class with two template arguments: MatType and GradType. This is 27 | * instantiated at the start of the optimization. 28 | */ 29 | template 30 | class Policy 31 | { 32 | public: 33 | /** 34 | * This is called by the optimizer method before the start of the iteration 35 | * update process. 36 | * 37 | * @param parent Instantiated parent class. 38 | * @param rows Number of rows in the gradient matrix. 39 | * @param cols Number of columns in the gradient matrix. 40 | */ 41 | Policy(SVRGUpdate& /* parent */, 42 | const size_t /* rows */, 43 | const size_t /* cols */) 44 | { /* Do nothing. */ } 45 | 46 | /** 47 | * Update step for SVRG. The function parameters are updated in the negative 48 | * direction of the gradient. 49 | * 50 | * @param iterate Parameters that minimize the function. 51 | * @param fullGradient The computed full gradient. 52 | * @param gradient The current gradient matrix at time t. 53 | * @param gradient0 The old gradient matrix at time t - 1. 54 | * @param batchSize Batch size to be used for the given iteration. 55 | * @param stepSize Step size to be used for the given iteration. 56 | */ 57 | void Update(MatType& iterate, 58 | const GradType& fullGradient, 59 | const GradType& gradient, 60 | const GradType& gradient0, 61 | const size_t batchSize, 62 | const double stepSize) 63 | { 64 | // Perform the vanilla SVRG update. 65 | iterate -= stepSize * (fullGradient + (gradient - gradient0) / 66 | (double) batchSize); 67 | } 68 | }; 69 | }; 70 | 71 | } // namespace ens 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /include/ensmallen_bits/swats/swats_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file swats_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the SWATS optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SWATS_SWATS_IMPL_HPP 13 | #define ENSMALLEN_SWATS_SWATS_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "swats.hpp" 17 | 18 | namespace ens { 19 | 20 | inline SWATS::SWATS( 21 | const double stepSize, 22 | const size_t batchSize, 23 | const double beta1, 24 | const double beta2, 25 | const double epsilon, 26 | const size_t maxIterations, 27 | const double tolerance, 28 | const bool shuffle, 29 | const bool resetPolicy, 30 | const bool exactObjective) : 31 | optimizer(stepSize, 32 | batchSize, 33 | maxIterations, 34 | tolerance, 35 | shuffle, 36 | SWATSUpdate(epsilon, beta1, beta2), 37 | NoDecay(), 38 | resetPolicy, 39 | exactObjective) 40 | { /* Nothing to do. */ } 41 | 42 | } // namespace ens 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /include/ensmallen_bits/wn_grad/wn_grad_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file wn_grad_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the WNGrad optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_WN_GRAD_WN_GRAD_IMPL_HPP 13 | #define ENSMALLEN_WN_GRAD_WN_GRAD_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "wn_grad.hpp" 17 | 18 | namespace ens { 19 | 20 | inline WNGrad::WNGrad( 21 | const double stepSize, 22 | const size_t batchSize, 23 | const size_t maxIterations, 24 | const double tolerance, 25 | const bool shuffle, 26 | const bool resetPolicy, 27 | const bool exactObjective) : 28 | optimizer(stepSize, 29 | batchSize, 30 | maxIterations, 31 | tolerance, 32 | shuffle, 33 | WNGradUpdate(), 34 | NoDecay(), 35 | resetPolicy, 36 | exactObjective) 37 | { /* Nothing to do. */ } 38 | 39 | } // namespace ens 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /include/ensmallen_bits/yogi/yogi_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file yogi_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of Yogi class wrapper. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_YOGI_YOGI_IMPL_HPP 13 | #define ENSMALLEN_YOGI_YOGI_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "yogi.hpp" 17 | 18 | namespace ens { 19 | 20 | inline Yogi::Yogi( 21 | const double stepSize, 22 | const size_t batchSize, 23 | const double beta1, 24 | const double beta2, 25 | const double epsilon, 26 | const size_t maxIterations, 27 | const double tolerance, 28 | const bool shuffle, 29 | const bool resetPolicy, 30 | const bool exactObjective) : 31 | optimizer(stepSize, 32 | batchSize, 33 | maxIterations, 34 | tolerance, 35 | shuffle, 36 | YogiUpdate(epsilon, beta1, beta2), 37 | NoDecay(), 38 | resetPolicy, 39 | exactObjective) 40 | { /* Nothing to do. */ } 41 | 42 | } // namespace ens 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /scripts/history-update-check.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Check each PR has an entry in HISTORY.md during a CI routine. 4 | # 5 | # Arguments: 6 | # $ history-update-check.sh 7 | # 8 | # This should be run from the root of the repository. 9 | res=$(git diff origin/master --name-only | grep ^HISTORY.md | wc -l) 10 | echo "Files Changed:" 11 | git diff --name-only 12 | if [ $res -gt 0 ]; then 13 | echo "HISTORY.md was updated with a change for the PR ..." 14 | else 15 | echo "Please describe your PR changes in HISTORY.md ..." 16 | echo "Exiting CI process ... " 17 | exit 1 18 | fi 19 | 20 | exit 0 21 | -------------------------------------------------------------------------------- /scripts/update-website-after-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This script is used to update the website after a release is made. Push 4 | # access to the ensmallen.org website is needed. Generally, this script will be 5 | # run by mlpack-bot, so it never needs to be run by hand. 6 | # 7 | # Usage: update-website-after-release.sh 8 | 9 | MAJOR=$1; 10 | MINOR=$2; 11 | PATCH=$3; 12 | 13 | # Make sure that the mlpack repository exists. 14 | dest_remote_name=`git remote -v |\ 15 | grep "mlpack/ensmallen (fetch)" |\ 16 | head -1 |\ 17 | awk -F' ' '{ print $1 }'`; 18 | 19 | if [ "a$dest_remote_name" == "a" ]; then 20 | echo "No git remote found for mlpack/ensmallen!"; 21 | echo "Make sure that you've got the ensmallen repository as a remote, and" \ 22 | "that the master branch from that remote is checked out."; 23 | echo "You can do this with a fresh repository via \`git clone" \ 24 | "https://github.com/mlpack/ensmallen\`."; 25 | exit 1; 26 | fi 27 | 28 | # Update the checked out repository, so that we can get the tags. 29 | git fetch $dest_remote_name; 30 | 31 | # Check out a copy of the ensmallen.org repository. 32 | git clone git@github.com:mlpack/ensmallen.org /tmp/ensmallen.org/; 33 | 34 | # Create the release file. 35 | git archive --prefix=ensmallen-$MAJOR.$MINOR.$PATCH/ $MAJOR.$MINOR.$PATCH |\ 36 | gzip > /tmp/ensmallen.org/files/ensmallen-$MAJOR.$MINOR.$PATCH.tar.gz; 37 | 38 | # Now update the website. 39 | wd=`pwd`; 40 | cd /tmp/ensmallen.org/; 41 | git add files/ensmallen-$MAJOR.$MINOR.$PATCH.tar.gz; 42 | 43 | # Update the link to the latest version. 44 | cd files/; 45 | rm ensmallen-latest.tar.gz; 46 | ln -s ensmallen-$MAJOR.$MINOR.$PATCH.tar.gz ensmallen-latest.tar.gz; 47 | cd ../ 48 | 49 | # Update the index page. 50 | sed -i 's/\[ensmallen-[0-9]*\.[0-9]*\.[0-9]\.tar\.gz\](files\/ensmallen-[0-9]*\.[0-9]*\.[0-9]*\.tar.gz)/[ensmallen-'$MAJOR'.'$MINOR'.'$PATCH'.tar.gz](files\/ensmallen-'$MAJOR'.'$MINOR'.'$PATCH'.tar.gz)/' index.md 51 | 52 | git add files/ensmallen-latest.tar.gz; 53 | git add index.md; 54 | git commit -m "Release version $MAJOR.$MINOR.$PATCH."; 55 | 56 | # Finally, push, and we're done. 57 | git push origin; 58 | cd $wd; 59 | 60 | rm -rf /tmp/ensmallen.org; 61 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # The tests that need to be compiled. 2 | set(ENSMALLEN_TESTS_SOURCES 3 | main.cpp 4 | active_cmaes_test.cpp 5 | ada_belief_test.cpp 6 | ada_bound_test.cpp 7 | ada_delta_test.cpp 8 | ada_grad_test.cpp 9 | ada_sqrt_test.cpp 10 | adam_test.cpp 11 | aug_lagrangian_test.cpp 12 | bigbatch_sgd_test.cpp 13 | callbacks_test.cpp 14 | cd_test.cpp 15 | cmaes_test.cpp 16 | cne_test.cpp 17 | de_test.cpp 18 | demon_adam_test.cpp 19 | demon_sgd_test.cpp 20 | eve_test.cpp 21 | frankwolfe_test.cpp 22 | ftml_test.cpp 23 | function_test.cpp 24 | gradient_descent_test.cpp 25 | grid_search_test.cpp 26 | iqn_test.cpp 27 | indicators_test.cpp 28 | katyusha_test.cpp 29 | lbfgs_test.cpp 30 | line_search_test.cpp 31 | lookahead_test.cpp 32 | lrsdp_test.cpp 33 | moead_test.cpp 34 | agemoea_test.cpp 35 | momentum_sgd_test.cpp 36 | nesterov_momentum_sgd_test.cpp 37 | nsga2_test.cpp 38 | parallel_sgd_test.cpp 39 | pop_cmaes_test.cpp 40 | proximal_test.cpp 41 | pso_test.cpp 42 | quasi_hyperbolic_momentum_sgd_test.cpp 43 | rmsprop_test.cpp 44 | sa_test.cpp 45 | sarah_test.cpp 46 | sdp_primal_dual_test.cpp 47 | sgdr_test.cpp 48 | sgd_test.cpp 49 | smorms3_test.cpp 50 | snapshot_ensembles.cpp 51 | spalera_sgd_test.cpp 52 | spsa_test.cpp 53 | svrg_test.cpp 54 | swats_test.cpp 55 | wn_grad_test.cpp 56 | yogi_test.cpp 57 | ) 58 | 59 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) 60 | add_executable(ensmallen_tests EXCLUDE_FROM_ALL ${ENSMALLEN_TESTS_SOURCES}) 61 | target_link_libraries(ensmallen_tests PRIVATE ensmallen) 62 | 63 | # Copy test data into place. 64 | add_custom_command(TARGET ensmallen_tests 65 | POST_BUILD 66 | COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/data/ 67 | ${CMAKE_BINARY_DIR}/data/ 68 | ) 69 | 70 | enable_testing() 71 | add_test(NAME ensmallen_tests COMMAND ensmallen_tests 72 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) 73 | -------------------------------------------------------------------------------- /tests/LICENSE_1_0.txt: -------------------------------------------------------------------------------- 1 | Boost Software License - Version 1.0 - August 17th, 2003 2 | 3 | Permission is hereby granted, free of charge, to any person or organization 4 | obtaining a copy of the software and accompanying documentation covered by 5 | this license (the "Software") to use, reproduce, display, distribute, 6 | execute, and transmit the Software, and to prepare derivative works of the 7 | Software, and to permit third-parties to whom the Software is furnished to 8 | do so, all subject to the following: 9 | 10 | The copyright notices in the Software and this entire statement, including 11 | the above license grant, this restriction and the following disclaimer, 12 | must be included in all copies of the Software, in whole or in part, and 13 | all derivative works of the Software, unless such copies or derivative 14 | works are solely in the form of machine-executable object code generated by 15 | a source language processor. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 20 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 21 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /tests/ada_belief_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_belief_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include "catch.hpp" 13 | #include "test_function_tools.hpp" 14 | 15 | using namespace ens; 16 | using namespace ens::test; 17 | 18 | /** 19 | * Test the AdaBelief optimizer on the Sphere function. 20 | */ 21 | TEST_CASE("AdaBeliefSphereFunctionTest", "[AdaBeliefTest]") 22 | { 23 | AdaBelief optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false); 24 | FunctionTest(optimizer, 0.5, 0.1); 25 | } 26 | 27 | /** 28 | * Test the AdaBelief optimizer on the Sphere function with arma::fmat. 29 | */ 30 | TEST_CASE("AdaBeliefSphereFunctionTestFMat", "[AdaBeliefTest]") 31 | { 32 | AdaBelief optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false); 33 | FunctionTest(optimizer, 0.5, 0.1); 34 | } 35 | 36 | /** 37 | * Test the AdaBelief optimizer on the McCormick function. 38 | */ 39 | TEST_CASE("AdaBeliefMcCormickFunctionTest", "[AdaBeliefTest]") 40 | { 41 | AdaBelief optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false); 42 | FunctionTest(optimizer, 0.5, 0.1); 43 | } 44 | 45 | /** 46 | * Run AdaBelief on logistic regression and make sure the results are 47 | * acceptable. 48 | */ 49 | TEST_CASE("AdaBeliefLogisticRegressionTest", "[AdaBeliefTest]") 50 | { 51 | AdaBelief optimizer; 52 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 53 | } 54 | 55 | /** 56 | * Run AdaBelief on logistic regression and make sure the results are 57 | * acceptable, using arma::fmat. 58 | */ 59 | TEST_CASE("AdaBeliefLogisticRegressionFMatTest", "[AdaBeliefTest]") 60 | { 61 | arma::fmat data, testData, shuffledData; 62 | arma::Row responses, testResponses, shuffledResponses; 63 | 64 | LogisticRegressionTestData(data, testData, shuffledData, 65 | responses, testResponses, shuffledResponses); 66 | LogisticRegression lr(shuffledData, shuffledResponses, 0.5); 67 | 68 | AdaBelief optimizer; 69 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 70 | } 71 | 72 | -------------------------------------------------------------------------------- /tests/ada_delta_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_delta_test.cpp 3 | * @author Marcus Edel 4 | * @author Vasanth Kalingeri 5 | * @author Abhinav Moudgil 6 | * @author Conrad Sanderson 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | 18 | using namespace ens; 19 | using namespace ens::test; 20 | 21 | /** 22 | * Run AdaDelta on logistic regression and make sure the results are acceptable. 23 | */ 24 | TEST_CASE("AdaDeltaLogisticRegressionTest", "[AdaDeltaTest]") 25 | { 26 | AdaDelta adaDelta; 27 | LogisticRegressionFunctionTest(adaDelta, 0.003, 0.006, 1); 28 | } 29 | 30 | /** 31 | * Run AdaDelta on logistic regression and make sure the results are acceptable 32 | * with arma::fmat as the type. 33 | */ 34 | TEST_CASE("AdaDeltaLogisticRegressionTestFMat", "[AdaDeltaTest]") 35 | { 36 | AdaDelta adaDelta; 37 | LogisticRegressionFunctionTest(adaDelta, 0.003, 0.006, 1); 38 | } 39 | -------------------------------------------------------------------------------- /tests/ada_grad_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_grad_test.cpp 3 | * @author Abhinav Moudgil 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Run AdaGrad on logistic regression and make sure the results are acceptable. 22 | */ 23 | TEST_CASE("AdaGradLogisticRegressionTest", "[AdaGradTest]") 24 | { 25 | AdaGrad adagrad(0.99, 32, 1e-8, 5000000, 1e-9, true); 26 | LogisticRegressionFunctionTest(adagrad, 0.003, 0.006); 27 | } 28 | 29 | /** 30 | * Run AdaGrad on logistic regression and make sure the results are acceptable. 31 | */ 32 | TEST_CASE("AdaGradLogisticRegressionTestFMat", "[AdaGradTest]") 33 | { 34 | AdaGrad adagrad(0.99, 32, 1e-8, 5000000, 1e-9, true); 35 | LogisticRegressionFunctionTest(adagrad, 0.003, 0.006); 36 | } 37 | -------------------------------------------------------------------------------- /tests/ada_sqrt_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_sqrt_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include "catch.hpp" 13 | #include "test_function_tools.hpp" 14 | 15 | using namespace ens; 16 | using namespace ens::test; 17 | 18 | /** 19 | * Run AdaSqrt on logistic regression and make sure the results are acceptable. 20 | */ 21 | TEST_CASE("AdaSqrtLogisticRegressionTest", "[AdaSqrtTest]") 22 | { 23 | AdaSqrt optimizer(0.01, 32, 1e-8, 5000000, 1e-9, true); 24 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 25 | } 26 | 27 | /** 28 | * Run AdaSqrt on logistic regression and make sure the results are acceptable. 29 | */ 30 | TEST_CASE("AdaSqrtLogisticRegressionTestFMat", "[AdaSqrtTest]") 31 | { 32 | AdaSqrt optimizer(0.01, 32, 1e-8, 5000000, 1e-9, true); 33 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 34 | } 35 | -------------------------------------------------------------------------------- /tests/cmaes_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file cmaes_test.cpp 3 | * @author Marcus Edel 4 | * @author Kartik Nighania 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Run CMA-ES with the full selection policy on logistic regression and 22 | * make sure the results are acceptable. 23 | */ 24 | TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") 25 | { 26 | BoundaryBoxConstraint<> b(-10, 10); 27 | CMAES> cmaes(0, b, 32, 500, 1e-3); 28 | cmaes.StepSize() = 0.6; 29 | LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); 30 | } 31 | 32 | /** 33 | * Run CMA-ES with the random selection policy on logistic regression and 34 | * make sure the results are acceptable. 35 | */ 36 | TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") 37 | { 38 | BoundaryBoxConstraint<> b(-10, 10); 39 | ApproxCMAES> cmaes(256, b, 16, 500, 1e-3); 40 | cmaes.StepSize() = 0.6; 41 | LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); 42 | } 43 | 44 | /** 45 | * Run CMA-ES with the full selection policy on logistic regression and 46 | * make sure the results are acceptable. Use arma::fmat. 47 | */ 48 | TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") 49 | { 50 | BoundaryBoxConstraint b(-10, 10); 51 | CMAES> cmaes(120, b, 32, 500, 1e-3); 52 | LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); 53 | } 54 | 55 | /** 56 | * Run CMA-ES with the random selection policy on logistic regression and 57 | * make sure the results are acceptable. Use arma::fmat. 58 | */ 59 | TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") 60 | { 61 | BoundaryBoxConstraint b(-10, 10); 62 | ApproxCMAES> cmaes(0, b, 16, 500, 1e-3); 63 | LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); 64 | } 65 | 66 | /** 67 | * Run CMA-ES with the random selection and empty transformation policies 68 | * on logistic regression and make sure the results are acceptable. 69 | * Use arma::fmat. 70 | */ 71 | TEST_CASE("ApproxCMAESEmptyTransformationLogisticRegressionFMatTest", 72 | "[CMAESTest]") 73 | { 74 | ApproxCMAES> 75 | cmaes(0, EmptyTransformation(), 16, 500, 1e-3); 76 | LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); 77 | } 78 | -------------------------------------------------------------------------------- /tests/data/r10.txt: -------------------------------------------------------------------------------- 1 | 2.388061771742711503e+00 -7.998843946705324059e-01 0.000000000000000000e+00 -6.055455953973358785e-01 -8.978447303086828368e-01 -8.478705136616015992e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2 | -7.998843946705324059e-01 3.426145759425113901e+00 -1.638176200441511243e-01 0.000000000000000000e+00 -4.796520189361042075e-01 -7.078326540851567206e-01 -9.383403917548253093e-01 -3.366186799343440228e-01 0.000000000000000000e+00 0.000000000000000000e+00 3 | 0.000000000000000000e+00 -1.638176200441511243e-01 1.048508772479271434e+00 -8.846911524351203093e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4 | -6.055455953973358785e-01 0.000000000000000000e+00 -8.846911524351203093e-01 2.023397783192608657e+00 -5.331610353601525798e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5 | -8.978447303086828368e-01 -4.796520189361042075e-01 0.000000000000000000e+00 -5.331610353601525798e-01 2.697683931814737868e+00 -5.616838162810112722e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -2.253423309287867493e-01 6 | -8.478705136616015992e-02 -7.078326540851567206e-01 0.000000000000000000e+00 0.000000000000000000e+00 -5.616838162810112722e-01 2.784761835214184078e+00 -5.704882222059795005e-01 0.000000000000000000e+00 0.000000000000000000e+00 -8.599700912758765359e-01 7 | 0.000000000000000000e+00 -9.383403917548253093e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -5.704882222059795005e-01 1.890408204565853900e+00 -3.815795906050489794e-01 0.000000000000000000e+00 0.000000000000000000e+00 8 | 0.000000000000000000e+00 -3.366186799343440228e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -3.815795906050489794e-01 7.410893491077712447e-01 -2.289107856837824251e-02 0.000000000000000000e+00 9 | 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -2.289107856837824251e-02 2.289107856837824251e-02 0.000000000000000000e+00 10 | 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -2.253423309287867493e-01 -8.599700912758765359e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.085312422204663285e+00 11 | -------------------------------------------------------------------------------- /tests/de_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file de_test.cpp 3 | * @author Rahul Ganesh Prabhu 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include "catch.hpp" 13 | #include "test_function_tools.hpp" 14 | 15 | using namespace ens; 16 | using namespace ens::test; 17 | 18 | /** 19 | * Train and test a logistic regression function using DE optimizer. 20 | */ 21 | TEST_CASE("DELogisticRegressionTest", "[DETest]") 22 | { 23 | DE opt(200, 1000, 0.6, 0.8, 1e-5); 24 | LogisticRegressionFunctionTest(opt, 0.01, 0.02, 3); 25 | } 26 | 27 | /** 28 | * Train and test a logistic regression function using DE optimizer. Use 29 | * arma::fmat. 30 | */ 31 | TEST_CASE("DELogisticRegressionFMatTest", "[DETest]") 32 | { 33 | DE opt(200, 1000, 0.6, 0.8, 1e-5); 34 | LogisticRegressionFunctionTest(opt, 0.03, 0.06, 3); 35 | } 36 | -------------------------------------------------------------------------------- /tests/demon_adam_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file demon_adam_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include "catch.hpp" 13 | #include "test_function_tools.hpp" 14 | 15 | using namespace ens; 16 | using namespace ens::test; 17 | 18 | /** 19 | * Run DemonAdam on logistic regression and make sure the results are 20 | * acceptable. 21 | */ 22 | TEST_CASE("DemonAdamLogisticRegressionTest", "[DemonAdamTest]") 23 | { 24 | DemonAdam optimizer(0.2, 32, 0.9, 0.9, 0.999, 1e-8, 25 | 10000, 1e-9, true, true, true); 26 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006, 6); 27 | } 28 | 29 | /** 30 | * Test the Adam optimizer on the Sphere function. 31 | */ 32 | TEST_CASE("DemonAdamSphereFunctionTest", "[DemonAdamTest]") 33 | { 34 | SphereFunction f(2); 35 | DemonAdam optimizer(0.5, 2, 0.9); 36 | FunctionTest(optimizer, 1.0, 0.1); 37 | } 38 | 39 | /** 40 | * Test the DemonAdam optimizer on the Matyas function. 41 | */ 42 | TEST_CASE("DemonAdamMatyasFunctionTest", "[DemonAdamTest]") 43 | { 44 | DemonAdam optimizer(0.5, 1, 0.9); 45 | FunctionTest(optimizer, 0.1, 0.01); 46 | } 47 | 48 | /** 49 | * Test the Adam optimizer on the Sphere function. 50 | */ 51 | TEST_CASE("DemonAdamSphereFunctionTestFloat", "[DemonAdamTest]") 52 | { 53 | DemonAdam optimizer(0.5, 2, 0.9); 54 | FunctionTest(optimizer, 1.0, 0.1); 55 | } 56 | 57 | /** 58 | * Test the DemonAdam optimizer on the Matyas function. 59 | */ 60 | TEST_CASE("DemonAdamMatyasFunctionTestFloat", "[DemonAdamTest]") 61 | { 62 | DemonAdam optimizer(0.5, 1, 0.9); 63 | FunctionTest(optimizer, 0.1, 0.01); 64 | } 65 | 66 | /** 67 | * Run DemonAdam (AdaMax update) on logistic regression and make sure the 68 | * results are acceptable. 69 | */ 70 | TEST_CASE("DemonAdaMaxLogisticRegressionTest", "[DemonAdamTest]") 71 | { 72 | DemonAdamType optimizer(0.5, 10, 0.9, 0.9, 0.999, 1e-8, 73 | 10000, 1e-9, true, true, true); 74 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006, 3); 75 | } 76 | -------------------------------------------------------------------------------- /tests/demon_sgd_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file demon_sgd_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include "catch.hpp" 13 | #include "test_function_tools.hpp" 14 | 15 | using namespace ens; 16 | using namespace ens::test; 17 | 18 | /** 19 | * Run DemonSGD on logistic regression and make sure the results are 20 | * acceptable. 21 | */ 22 | TEST_CASE("DemonSGDLogisticRegressionTest", "[DemonSGDTest]") 23 | { 24 | DemonSGD optimizer(0.1, 32, 0.9, 1000000, 1e-9, true, true, true); 25 | LogisticRegressionFunctionTest(optimizer, 0.006, 0.006, 6); 26 | } 27 | 28 | /** 29 | * Tests the DemonSGD optimizer using a simple test function. 30 | */ 31 | TEST_CASE("DemonSGDSimpleTestFunctionFloat", "[DemonSGDTest]") 32 | { 33 | SGDTestFunction f; 34 | DemonSGD optimizer(1e-2, 1, 0.9, 400000); 35 | 36 | arma::fmat coordinates = f.GetInitialPoint(); 37 | optimizer.Optimize(f, coordinates); 38 | 39 | REQUIRE(coordinates(0) == Approx(0.0).margin(0.1)); 40 | REQUIRE(coordinates(1) == Approx(0.0).margin(0.1)); 41 | REQUIRE(coordinates(2) == Approx(0.0).margin(0.1)); 42 | } 43 | 44 | -------------------------------------------------------------------------------- /tests/eve_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file eve_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * Test file for the Eve optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Run Eve on logistic regression and make sure the results are acceptable. 22 | */ 23 | TEST_CASE("EveLogisticRegressionTest","[EveTest]") 24 | { 25 | Eve optimizer(1e-3, 1, 0.9, 0.999, 0.999, 1e-8, 10000, 500000, 1e-9, true); 26 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 27 | } 28 | 29 | /** 30 | * Test the Eve optimizer on the Sphere function. 31 | */ 32 | TEST_CASE("EveSphereFunctionTest","[EveTest]") 33 | { 34 | Eve optimizer(1e-3, 2, 0.9, 0.999, 0.999, 1e-8, 10000, 500000, 1e-9, true); 35 | FunctionTest(optimizer, 0.5, 0.1); 36 | } 37 | 38 | /** 39 | * Test the Eve optimizer on the Styblinski-Tang function. 40 | */ 41 | TEST_CASE("EveStyblinskiTangFunctionTest","[EveTest]") 42 | { 43 | Eve optimizer(1e-3, 2, 0.9, 0.999, 0.999, 1e-8, 10000, 500000, 1e-9, true); 44 | FunctionTest(optimizer, 0.5, 0.1); 45 | } 46 | 47 | /** 48 | * Test the Eve optimizer on the Styblinski-Tang function using arma::fmat as 49 | * the objective type. 50 | */ 51 | TEST_CASE("EveStyblinskiTangFunctionFMatTest","[EveTest]") 52 | { 53 | Eve optimizer(1e-3, 2, 0.9, 0.999, 0.999, 1e-8, 10000, 500000, 1e-9, true); 54 | FunctionTest(optimizer, 0.5, 0.1); 55 | } 56 | 57 | /** 58 | * Test the Eve optimizer on the Styblinski-Tang function, using arma::sp_mat as 59 | * the objective type. 60 | */ 61 | TEST_CASE("EveStyblinskiTangFunctionSpMatTest","[EveTest]") 62 | { 63 | Eve optimizer(1e-3, 2, 0.9, 0.999, 0.999, 1e-8, 10000, 500000, 1e-9, true); 64 | FunctionTest(optimizer, 0.5, 0.1); 65 | } 66 | -------------------------------------------------------------------------------- /tests/ftml_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ftml_test.cpp 3 | * @author Ryan Curtin 4 | * 5 | * Test file for the FTML optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Run FTML on logistic regression and make sure the results are acceptable. 22 | */ 23 | TEST_CASE("FTMLLogisticRegressionTest", "[FTMLTest]") 24 | { 25 | FTML optimizer(0.001, 1, 0.9, 0.999, 1e-8, 100000, 1e-5, true); 26 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 27 | } 28 | 29 | /** 30 | * Test the FTML optimizer on the Sphere function. 31 | */ 32 | TEST_CASE("FTMLSphereFunctionTest", "[FTMLTest]") 33 | { 34 | FTML optimizer(0.001, 2, 0.9, 0.999, 1e-8, 500000, 1e-9, true); 35 | FunctionTest(optimizer, 0.5, 0.1); 36 | } 37 | 38 | /** 39 | * Test the FTML optimizer on the Styblinski-Tang function. 40 | */ 41 | TEST_CASE("FTMLStyblinskiTangFunctionTest", "[FTMLTest]") 42 | { 43 | FTML optimizer(0.001, 2, 0.9, 0.999, 1e-8, 100000, 1e-5, true); 44 | FunctionTest(optimizer, 0.5, 0.1); 45 | } 46 | 47 | /** 48 | * Test the FTML optimizer on the Styblinski-Tang function using arma::fmat as 49 | * the objective type. 50 | */ 51 | TEST_CASE("FTMLStyblinskiTangFunctionFMatTest", "[FTMLTest]") 52 | { 53 | FTML optimizer(0.001, 2, 0.9, 0.999, 1e-8, 100000, 1e-5, true); 54 | FunctionTest(optimizer, 0.5, 0.1); 55 | } 56 | 57 | // A test with sp_mat is not done, because FTML uses some parts internally that 58 | // assume the objective is dense. 59 | -------------------------------------------------------------------------------- /tests/gradient_descent_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file gradient_descent_test.cpp 3 | * @author Sumedh Ghaisas 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | TEST_CASE("SimpleGDTestFunction", "[GradientDescentTest]") 21 | { 22 | GradientDescent s(0.01, 5000000, 1e-9); 23 | FunctionTest(s, 0.1, 0.01); 24 | } 25 | 26 | TEST_CASE("GDRosenbrockTest", "[GradientDescentTest]") 27 | { 28 | GradientDescent s(0.001, 0, 1e-15); 29 | FunctionTest(s, 0.01, 0.001); 30 | } 31 | 32 | TEST_CASE("GDRosenbrockFMatTest", "[GradientDescentTest]") 33 | { 34 | GradientDescent s(0.001, 0, 1e-15); 35 | FunctionTest(s, 0.1, 0.01); 36 | } 37 | -------------------------------------------------------------------------------- /tests/iqn_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file iqn_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * Test file for IQN (incremental Quasi-Newton). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Run IQN on logistic regression and make sure the results are acceptable. 22 | */ 23 | TEST_CASE("IQNLogisticRegressionTest", "[IQNTest]") 24 | { 25 | // Run on a couple of batch sizes. 26 | for (size_t batchSize = 1; batchSize < 9; batchSize += 4) 27 | { 28 | IQN iqn(0.01, batchSize, 5000, 0.01); 29 | LogisticRegressionFunctionTest(iqn, 0.013, 0.016); 30 | } 31 | } 32 | 33 | /** 34 | * Run IQN on logistic regression and make sure the results are acceptable. Use 35 | * arma::fmat. 36 | */ 37 | TEST_CASE("IQNLogisticRegressionFMatTest", "[IQNTest]") 38 | { 39 | // Run on a couple of batch sizes. 40 | for (size_t batchSize = 1; batchSize < 9; batchSize += 4) 41 | { 42 | IQN iqn(0.001, batchSize, 5000, 0.01); 43 | LogisticRegressionFunctionTest(iqn, 0.013, 0.016); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /tests/line_search_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file line_search_test.cpp 3 | * @author Chenzhe Diao 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | 16 | using namespace arma; 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Simple test of Line Search with TestFuncFW function. 22 | */ 23 | TEST_CASE("FuncFWTest", "[LineSearchTest]") 24 | { 25 | mat x1 = zeros(3, 1); 26 | mat x2 = { 0.2, 0.4, 0.6 }; 27 | x2 = x2.t(); 28 | 29 | TestFuncFW<> f; 30 | LineSearch s; 31 | 32 | double result = s.Optimize(f, x1, x2); 33 | 34 | REQUIRE(result == Approx(0.0).margin(1e-10)); 35 | REQUIRE((x2(0) - 0.1) == Approx(0.0).margin(1e-10)); 36 | REQUIRE((x2(1) - 0.2) == Approx(0.0).margin(1e-10)); 37 | REQUIRE((x2(2) - 0.3) == Approx(0.0).margin(1e-10)); 38 | } 39 | 40 | /** 41 | * Simple test of Line Search with TestFuncFW function. Use arma::fmat. 42 | */ 43 | TEST_CASE("FuncFWFMatTest", "[LineSearchTest]") 44 | { 45 | fmat x1 = zeros(3, 1); 46 | fmat x2 = { 0.2, 0.4, 0.6 }; 47 | x2 = x2.t(); 48 | 49 | TestFuncFW f; 50 | LineSearch s; 51 | 52 | float result = s.Optimize(f, x1, x2); 53 | 54 | REQUIRE(result == Approx(0.0).margin(1e-5)); 55 | REQUIRE((x2(0) - 0.1) == Approx(0.0).margin(1e-5)); 56 | REQUIRE((x2(1) - 0.2) == Approx(0.0).margin(1e-5)); 57 | REQUIRE((x2(2) - 0.3) == Approx(0.0).margin(1e-5)); 58 | } 59 | -------------------------------------------------------------------------------- /tests/lookahead_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file lookahead_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include "catch.hpp" 13 | #include "test_function_tools.hpp" 14 | 15 | using namespace ens; 16 | using namespace ens::test; 17 | 18 | /** 19 | * Test the Lookahead - Adam optimizer on the Sphere function. 20 | */ 21 | TEST_CASE("LookaheadAdamSphereFunctionTest", "[LookaheadTest]") 22 | { 23 | Lookahead<> optimizer(0.5, 5, 100000, 1e-5, NoDecay(), false, true); 24 | optimizer.BaseOptimizer().StepSize() = 0.1; 25 | optimizer.BaseOptimizer().BatchSize() = 2; 26 | optimizer.BaseOptimizer().Beta1() = 0.7; 27 | optimizer.BaseOptimizer().Tolerance() = 1e-15; 28 | // We allow a few trials. 29 | FunctionTest(optimizer, 0.5, 0.2, 3); 30 | } 31 | 32 | /** 33 | * Test the Lookahead - AdaGrad optimizer on the SphereFunction function. 34 | */ 35 | TEST_CASE("LookaheadAdaGradSphereFunction", "[LookaheadTest]") 36 | { 37 | AdaGrad adagrad(0.99, 1, 1e-8, 5, 1e-15, true); 38 | Lookahead optimizer(adagrad, 0.5, 5, 5000000, 1e-15, NoDecay(), 39 | false, true); 40 | FunctionTest(optimizer, 0.5, 0.2, 3); 41 | } 42 | 43 | /** 44 | * Run Lookahead - Adam on logistic regression and make sure the results are 45 | * acceptable. 46 | */ 47 | TEST_CASE("LookaheadAdamLogisticRegressionTest","[LookaheadTest]") 48 | { 49 | Adam adam(0.001, 32, 0.9, 0.999, 1e-8, 5, 1e-19); 50 | Lookahead optimizer(adam, 0.5, 20, 100000, 1e-15, NoDecay(), 51 | false, true); 52 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 53 | } 54 | 55 | /** 56 | * Test the Lookahead - Adam optimizer on the Sphere function (float). 57 | */ 58 | TEST_CASE("LookaheadAdamSimpleSphereFunctionFloat", "[LookaheadTest]") 59 | { 60 | Adam adam(0.001, 1, 0.9, 0.999, 1e-8, 5, 1e-19, false, true); 61 | Lookahead optimizer(adam, 0.5, 5, 100000, 1e-15, NoDecay(), 62 | false, true); 63 | FunctionTest(optimizer, 0.5, 0.2, 3); 64 | } 65 | -------------------------------------------------------------------------------- /tests/main.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file main.cpp 3 | * @author Conrad Sanderson 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include 13 | 14 | //#define CATCH_CONFIG_MAIN // catch.hpp will define main() 15 | #define CATCH_CONFIG_RUNNER // we will define main() 16 | #include "catch.hpp" 17 | 18 | int main(int argc, char** argv) 19 | { 20 | Catch::Session session; 21 | const int returnCode = session.applyCommandLine(argc, argv); 22 | // Check for a command line error. 23 | if (returnCode != 0) 24 | return returnCode; 25 | 26 | std::cout << "ensmallen version: " << ens::version::as_string() << std::endl; 27 | std::cout << "armadillo version: " << arma::arma_version::as_string() << std::endl; 28 | 29 | // Use Catch2 command-line to set the random seed. 30 | // -rng-seed <'time'|number> 31 | // If a number is provided this is used directly as the seed. Alternatively 32 | // if the keyword 'time' is provided then the result of calling std::time(0) 33 | // is used. 34 | const size_t seed = session.config().rngSeed(); 35 | std::cout << "random seed: " << seed << std::endl; 36 | srand((unsigned int) seed); 37 | arma::arma_rng::set_seed(seed); 38 | 39 | return session.run(); 40 | } 41 | -------------------------------------------------------------------------------- /tests/proximal_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file proximal_test.cpp 3 | * @author Chenzhe Diao 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | 16 | using namespace arma; 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Approximate vector using a vector with l1 norm small than or equal to tau. 22 | */ 23 | TEST_CASE("ProjectToL1", "[ProximalTest]") 24 | { 25 | int D = 100; // Dimension of the problem. 26 | 27 | // Norm of L1 ball. 28 | double tau1 = 1.5; 29 | double tau2 = 0.5; 30 | 31 | // Vector to be projected, with unit l1 norm. 32 | vec v = randu(D); 33 | v = normalise(v, 1); 34 | 35 | // v is inside the l1 ball, so the projection will not change v. 36 | vec v1 = v; 37 | Proximal::ProjectToL1Ball(v1, tau1); 38 | REQUIRE(norm(v - v1, 2) == Approx(0.0).margin(1e-10)); 39 | 40 | // v is outside the l1 ball, so the projection should find the closest. 41 | vec v2 = v; 42 | Proximal::ProjectToL1Ball(v2, tau2); 43 | double distance = norm(v2 - v, 2); 44 | for (size_t i = 1; i < 1000; i++) 45 | { 46 | // Randomly generate a vector on the surface of the l1 ball with norm tau2. 47 | vec vSurface = randu(D); 48 | vSurface = tau2 * normalise(vSurface, 1); 49 | 50 | double distanceNew = norm(vSurface - v, 2); 51 | 52 | REQUIRE(distanceNew >= distance); 53 | } 54 | } 55 | 56 | /** 57 | * Approximate a vector with a tau-sparse vector. 58 | */ 59 | TEST_CASE("ProjectToL0", "[ProximalTest]") 60 | { 61 | int D = 100; // Dimension of the problem. 62 | int tau = 25; // Sparsity requirement. 63 | 64 | // Vector to be projected. 65 | vec v = randn(D); 66 | 67 | vec v0 = v; 68 | Proximal::ProjectToL0Ball(v0, tau); 69 | double distance = norm(v0 - v, 2); 70 | 71 | for (size_t i = 1; i < 1000; i++) 72 | { 73 | // Randomly find a subset of the support of v, generate a tau-sparse 74 | // vector by restricting v to this support. 75 | uvec indices = linspace(0, D - 1, D); 76 | indices = shuffle(indices); 77 | indices = indices.head(tau); 78 | vec vNew = zeros(D); 79 | vNew.elem(indices) = v.elem(indices); 80 | 81 | double distanceNew = norm(v - vNew, 2); 82 | REQUIRE(distanceNew >= distance); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /tests/quasi_hyperbolic_momentum_sgd_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file quasi_hyperbolic_momentum_sgd_test.cpp 3 | * @author Niteya Shah 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #include 11 | #include "catch.hpp" 12 | #include "test_function_tools.hpp" 13 | 14 | using namespace ens; 15 | using namespace ens::test; 16 | 17 | /** 18 | * Tests the Quasi Hyperbolic Momentum SGD update policy. 19 | */ 20 | TEST_CASE("QHSphereFunction", "[QHMomentumSGDTest]") 21 | { 22 | QHUpdate update(0.4, 0.9); 23 | QHSGD s(0.0025, 1, 500000, 1e-10, true, update, NoDecay(), true, true); 24 | FunctionTest(s, 0.03, 0.003); 25 | } 26 | 27 | /** 28 | * Tests the Quasi Hyperbolic Momentum SGD update policy using arma::fmat. 29 | */ 30 | TEST_CASE("QHSphereFunctionFMat", "[QHMomentumSGDTest]") 31 | { 32 | QHUpdate update(0.9, 0.9); 33 | QHSGD s(0.002, 1, 2500000, 1e-9, true, update); 34 | FunctionTest(s, 0.3, 0.03); 35 | } 36 | 37 | /** 38 | * Tests the Quasi Hyperbolic Momentum SGD update policy using arma::sp_mat. 39 | */ 40 | TEST_CASE("QHSpMatTestSphereFunction", "[QHMomentumSGDTest]") 41 | { 42 | QHUpdate update(0.9, 0.9); 43 | QHSGD s(0.002, 1, 2500000, 1e-15, true, update); 44 | s.ExactObjective() = true; 45 | FunctionTest(s, 0.03, 0.003); 46 | } 47 | 48 | /** 49 | * Tests the Quasi hyperbolic SGD with Generalized Rosenbrock Test. 50 | */ 51 | TEST_CASE("QHSGDSGDGeneralizedRosenbrockTest", "[QHMomentumSGDTest]") 52 | { // Loop over several variants. 53 | for (size_t i = 10; i < 50; i += 5) 54 | { 55 | // Create the generalized Rosenbrock function. 56 | GeneralizedRosenbrockFunction f(i); 57 | QHUpdate update(0.9, 0.99); 58 | QHSGD s(0.0005, 1, 2500000, 1e-15, true, update, NoDecay(), true, true); 59 | 60 | arma::mat coordinates = f.GetInitialPoint(); 61 | double result = s.Optimize(f, coordinates); 62 | 63 | REQUIRE(result == Approx(0.0).margin(1e-4)); 64 | for (size_t j = 0; j < i; ++j) 65 | REQUIRE(coordinates(j) == Approx(1.0).epsilon(1e-4)); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /tests/rmsprop_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file rmsprop_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | 12 | #include 13 | #include "catch.hpp" 14 | #include "test_function_tools.hpp" 15 | 16 | using namespace ens; 17 | using namespace ens::test; 18 | 19 | /** 20 | * Run RMSProp on logistic regression and make sure the results are acceptable. 21 | */ 22 | TEST_CASE("RMSPropLogisticRegressionTest", "[rmsprop]") 23 | { 24 | RMSProp optimizer; 25 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 26 | } 27 | 28 | /** 29 | * Run RMSProp on logistic regression and make sure the results are acceptable. 30 | * Use arma::fmat. 31 | */ 32 | TEST_CASE("RMSPropLogisticRegressionFMatTest", "[rmsprop]") 33 | { 34 | RMSProp optimizer; 35 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 36 | } 37 | 38 | /** 39 | * Run RMSProp on logistic regression and make sure the results are acceptable. 40 | * Use arma::sp_mat. 41 | */ 42 | TEST_CASE("RMSPropLogisticRegressionSpMatTest", "[rmsprop]") 43 | { 44 | RMSProp optimizer; 45 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 46 | } 47 | -------------------------------------------------------------------------------- /tests/sgd_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sgd_test.cpp 3 | * @author Ryan Curtin 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace std; 18 | using namespace arma; 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEST_CASE("GeneralizedRosenbrockTest", "[SGDTest]") 23 | { 24 | // Loop over several variants. 25 | for (size_t i = 10; i < 50; i += 5) 26 | { 27 | // Create the generalized Rosenbrock function. 28 | GeneralizedRosenbrockFunction f(i); 29 | 30 | VanillaUpdate vanillaUpdate; 31 | StandardSGD s(0.001, 1, 0, 1e-15, true, vanillaUpdate, NoDecay(), true, 32 | true); 33 | 34 | arma::mat coordinates = f.GetInitialPoint(); 35 | double result = s.Optimize(f, coordinates); 36 | 37 | REQUIRE(result == Approx(0.0).margin(1e-10)); 38 | for (size_t j = 0; j < i; ++j) 39 | REQUIRE(coordinates(j) == Approx(1.0).epsilon(1e-5)); 40 | } 41 | } 42 | 43 | TEST_CASE("GeneralizedRosenbrockTestFloat", "[SGDTest]") 44 | { 45 | // Loop over several variants. 46 | for (size_t i = 10; i < 50; i += 5) 47 | { 48 | // Create the generalized Rosenbrock function. 49 | GeneralizedRosenbrockFunction f(i); 50 | 51 | // Allow a few trials. 52 | for (size_t trial = 0; trial < 5; ++trial) 53 | { 54 | StandardSGD s(0.001, 1, 0, 1e-15, true); 55 | 56 | arma::fmat coordinates = f.GetInitialPoint(); 57 | float result = s.Optimize(f, coordinates); 58 | 59 | if (trial != 4) 60 | { 61 | if (result != Approx(0.0).margin(1e-5)) 62 | continue; 63 | for (size_t j = 0; j < i; ++j) 64 | { 65 | if (coordinates(j) != Approx(1.0).epsilon(1e-3)) 66 | continue; 67 | } 68 | } 69 | 70 | REQUIRE(result == Approx(0.0).margin(1e-5)); 71 | for (size_t j = 0; j < i; ++j) 72 | REQUIRE(coordinates(j) == Approx(1.0).epsilon(1e-3)); 73 | break; 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tests/smorms3_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file snorms3_test.cpp 3 | * @author Vivek Pal 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Run SMORMS3 on logistic regression and make sure the results are acceptable. 22 | */ 23 | TEST_CASE("SMORMS3LogisticRegressionTest","[SMORMS3Test]") 24 | { 25 | SMORMS3 smorms3; 26 | LogisticRegressionFunctionTest(smorms3, 0.003, 0.006); 27 | } 28 | 29 | /** 30 | * Run SMORMS3 on logistic regression and make sure the results are acceptable. 31 | * Use arma::fmat. 32 | */ 33 | TEST_CASE("SMORMS3LogisticRegressionFMatTest","[SMORMS3Test]") 34 | { 35 | SMORMS3 smorms3; 36 | LogisticRegressionFunctionTest(smorms3, 0.003, 0.006); 37 | } 38 | -------------------------------------------------------------------------------- /tests/spalera_sgd_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file spalera_sgd_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | 12 | #include 13 | #include "catch.hpp" 14 | #include "test_function_tools.hpp" 15 | 16 | using namespace ens; 17 | using namespace ens::test; 18 | 19 | /** 20 | * Run SPALeRA SGD on logistic regression and make sure the results are 21 | * acceptable. 22 | */ 23 | TEST_CASE("LogisticRegressionTest","[SPALeRASGDTest]") 24 | { 25 | // Run SPALeRA SGD with a couple of batch sizes. 26 | for (size_t batchSize = 30; batchSize < 50; batchSize += 5) 27 | { 28 | SPALeRASGD<> optimizer(0.05 / batchSize, batchSize, 10000, 1e-4); 29 | LogisticRegressionFunctionTest(optimizer, 0.015, 0.024, 3); 30 | } 31 | } 32 | 33 | /** 34 | * Run SPALeRA SGD on logistic regression and make sure the results are 35 | * acceptable. Use arma::fmat. 36 | */ 37 | TEST_CASE("LogisticRegressionFMatTest","[SPALeRASGDTest]") 38 | { 39 | // Run SPALeRA SGD with a couple of batch sizes. 40 | for (size_t batchSize = 30; batchSize < 50; batchSize += 5) 41 | { 42 | SPALeRASGD<> optimizer(0.05 / batchSize, batchSize, 10000, 1e-4); 43 | LogisticRegressionFunctionTest(optimizer, 0.015, 0.024, 3); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /tests/spsa_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file spsa_test.cpp 3 | * @author N Rajiv Vaidyanathan 4 | * @author Marcus Edel 5 | * 6 | * Test file for the SPSA optimizer. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | 18 | using namespace arma; 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | /** 23 | * Test the SPSA optimizer on the Sphere function. 24 | */ 25 | TEST_CASE("SPSASphereFunctionTest", "[SPSATest]") 26 | { 27 | SPSA optimizer(0.1, 0.102, 0.16, 0.3, 100000, 0); 28 | FunctionTest(optimizer, 1.0, 0.1); 29 | } 30 | 31 | /** 32 | * Test the SPSA optimizer on the Sphere function using arma::fmat. 33 | */ 34 | TEST_CASE("SPSASphereFunctionFMatTest", "[SPSATest]") 35 | { 36 | SPSA optimizer(0.1, 0.102, 0.16, 0.3, 100000, 0); 37 | FunctionTest(optimizer, 1.0, 0.1); 38 | } 39 | 40 | /** 41 | * Test the SPSA optimizer on the Sphere function using arma::sp_mat. 42 | */ 43 | TEST_CASE("SPSASphereFunctionSpMatTest", "[SPSATest]") 44 | { 45 | SPSA optimizer(0.1, 0.102, 0.16, 0.3, 100000, 0); 46 | FunctionTest(optimizer, 1.0, 0.1); 47 | } 48 | 49 | /** 50 | * Test the SPSA optimizer on the Matyas function. 51 | */ 52 | TEST_CASE("SPSAMatyasFunctionTest", "[SPSATest]") 53 | { 54 | SPSA optimizer(0.1, 0.102, 0.16, 0.3, 100000, 0); 55 | FunctionTest(optimizer, 0.1, 0.01); 56 | } 57 | 58 | /** 59 | * Run SPSA on logistic regression and make sure the results are acceptable. 60 | */ 61 | TEST_CASE("SPSALogisticRegressionTest", "[SPSATest]") 62 | { 63 | // We allow 10 trials, because SPSA is definitely not guaranteed to 64 | // converge. 65 | SPSA optimizer(0.5, 0.102, 0.002, 0.3, 5000, 1e-8); 66 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006, 10); 67 | } 68 | -------------------------------------------------------------------------------- /tests/swats_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file swats_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * Test file for the SWATS optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Run SWATS on logistic regression and make sure the results are acceptable. 22 | */ 23 | TEST_CASE("SWATSLogisticRegressionTestFunction", "[SWATSTest]") 24 | { 25 | SWATS optimizer(1e-3, 10, 0.9, 0.999, 1e-6, 600000, 1e-9, true); 26 | // We allow a few trials in case of poor convergence. 27 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006, 5); 28 | } 29 | 30 | /** 31 | * Test the SWATS optimizer on the Sphere function. 32 | */ 33 | TEST_CASE("SWATSSphereFunctionTest", "[SWATSTest]") 34 | { 35 | SWATS optimizer(1e-3, 2, 0.9, 0.999, 1e-6, 500000, 1e-9, true); 36 | FunctionTest(optimizer, 1.0, 0.1); 37 | } 38 | 39 | /** 40 | * Test the SWATS optimizer on the Styblinski-Tang function. 41 | */ 42 | TEST_CASE("SWATSStyblinskiTangFunctionTest", "[SWATSTest]") 43 | { 44 | SWATS optimizer(1e-3, 2, 0.9, 0.999, 1e-6, 500000, 1e-9, true); 45 | FunctionTest(optimizer, 0.3, 0.03); 46 | } 47 | 48 | /** 49 | * Test the SWATS optimizer on the Styblinski-Tang function. Use arma::fmat. 50 | */ 51 | TEST_CASE("SWATSStyblinskiTangFunctionFMatTest", "[SWATSTest]") 52 | { 53 | SWATS optimizer(1e-3, 2, 0.9, 0.999, 1e-6, 500000, 1e-9, true); 54 | FunctionTest(optimizer, 3.0, 0.3); 55 | } 56 | 57 | /** 58 | * Test the SWATS optimizer on the Styblinski-Tang function. Use arma::sp_mat. 59 | */ 60 | TEST_CASE("SWATSStyblinskiTangFunctionSpMatTest", "[SWATSTest]") 61 | { 62 | SWATS optimizer(1e-3, 2, 0.9, 0.999, 1e-6, 500000, 1e-9, true); 63 | FunctionTest(optimizer, 0.3, 0.03); 64 | } 65 | -------------------------------------------------------------------------------- /tests/wn_grad_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file wn_grad_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * Test file for the WNGrad optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_function_tools.hpp" 16 | 17 | using namespace ens; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Run WNGrad on logistic regression and make sure the results are acceptable. 22 | */ 23 | TEST_CASE("WNGradLogisticRegressionTest","[WNGradTest]") 24 | { 25 | WNGrad optimizer(0.56, 1, 500000, 1e-9, true); 26 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 27 | } 28 | 29 | /** 30 | * Test the WNGrad optimizer on the Sphere function. 31 | */ 32 | TEST_CASE("WNGradSphereFunctionTest","[WNGradTest]") 33 | { 34 | WNGrad optimizer(0.56, 2, 500000, 1e-9, true); 35 | FunctionTest(optimizer, 1.0, 0.1); 36 | } 37 | 38 | /** 39 | * Test the WNGrad optimizer on the StyblinskiTangFunction. 40 | */ 41 | TEST_CASE("WNGradStyblinskiTangFunctionTest","[WNGradTest]") 42 | { 43 | WNGrad optimizer(0.56, 2, 500000, 1e-9, true); 44 | FunctionTest(optimizer, 0.3, 0.03); 45 | } 46 | 47 | /** 48 | * Test the WNGrad optimizer on the StyblinskiTangFunction. Use arma::fmat. 49 | */ 50 | TEST_CASE("WNGradStyblinskiTangFunctionFMatTest", "[WNGradTest]") 51 | { 52 | WNGrad optimizer(0.56, 2, 500000, 1e-9, true); 53 | FunctionTest(optimizer, 3.0, 0.3); 54 | } 55 | 56 | /** 57 | * Test the WNGrad optimizer on the StyblinskiTangFunction. Use arma::sp_mat. 58 | */ 59 | TEST_CASE("WNGradStyblinskiTangFunctionSpMatTest", "[WNGradTest]") 60 | { 61 | WNGrad optimizer(0.56, 2, 500000, 1e-9, true); 62 | FunctionTest(optimizer, 0.3, 0.03); 63 | } 64 | -------------------------------------------------------------------------------- /tests/yogi_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file yogi_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include "catch.hpp" 13 | #include "test_function_tools.hpp" 14 | 15 | using namespace ens; 16 | using namespace ens::test; 17 | 18 | /** 19 | * Test the Yogi optimizer on the Sphere function. 20 | */ 21 | TEST_CASE("YogiSphereFunctionTest", "[YogiTest]") 22 | { 23 | SphereFunction f(2); 24 | Yogi optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false); 25 | 26 | arma::mat coordinates = f.GetInitialPoint(); 27 | optimizer.Optimize(f, coordinates); 28 | 29 | REQUIRE(coordinates(0) == Approx(0.0).margin(0.1)); 30 | REQUIRE(coordinates(1) == Approx(0.0).margin(0.1)); 31 | } 32 | 33 | /** 34 | * Test the Yogi optimizer on the Sphere function with arma::fmat. 35 | */ 36 | TEST_CASE("YogiSphereFunctionTestFMat", "[YogiTest]") 37 | { 38 | SphereFunction f(2); 39 | Yogi optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false); 40 | 41 | arma::fmat coordinates = f.GetInitialPoint(); 42 | optimizer.Optimize(f, coordinates); 43 | 44 | REQUIRE(coordinates(0) == Approx(0.0).margin(0.1)); 45 | REQUIRE(coordinates(1) == Approx(0.0).margin(0.1)); 46 | } 47 | 48 | /** 49 | * Test the Yogi optimizer on the McCormick function. 50 | */ 51 | TEST_CASE("YogiMcCormickFunctionTest", "[YogiTest]") 52 | { 53 | Yogi optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false); 54 | FunctionTest(optimizer, 0.5, 0.1); 55 | } 56 | 57 | /** 58 | * Run Yogi on logistic regression and make sure the results are acceptable. 59 | */ 60 | TEST_CASE("YogiLogisticRegressionTest", "[YogiTest]") 61 | { 62 | Yogi optimizer; 63 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 64 | } 65 | 66 | /** 67 | * Run Yogi on logistic regression and make sure the results are acceptable, 68 | * using arma::fmat. 69 | */ 70 | TEST_CASE("YogiLogisticRegressionFMatTest", "[YogiTest]") 71 | { 72 | Yogi optimizer; 73 | LogisticRegressionFunctionTest(optimizer, 0.003, 0.006); 74 | } 75 | --------------------------------------------------------------------------------