├── .gitignore ├── CMake ├── ensmallen-config.cmake.in ├── ARMA_FindARPACK.cmake ├── COOT_FindNVRTC.cmake ├── ARMA_FindOpenBLAS.cmake ├── COOT_FindCLBLAS.cmake ├── ARMA_FindACML.cmake ├── ARMA_FindBLAS.cmake ├── COOT_FindCLBlast.cmake ├── ARMA_FindACMLMP.cmake ├── ARMA_FindLAPACK.cmake ├── ARMA_FindCBLAS.cmake ├── ARMA_FindCLAPACK.cmake └── ARMA_FindMKL.cmake ├── doc └── README.md ├── .github ├── ISSUE_TEMPLATE │ ├── 2-question.md │ ├── 1-documentation.md │ └── 0-bug.md └── workflows │ ├── auto-approve.yml │ ├── stale.yml │ ├── stickers.yml │ └── bandicoot-test.yml ├── scripts ├── history-update-check.sh └── update-website-after-release.sh ├── tests ├── ada_belief_test.cpp ├── smorms3_test.cpp ├── ada_grad_test.cpp ├── de_test.cpp ├── demon_sgd_test.cpp ├── ada_delta_test.cpp ├── rmsprop_test.cpp ├── ada_sqrt_test.cpp ├── spalera_sgd_test.cpp ├── iqn_test.cpp ├── line_search_test.cpp ├── gradient_descent_test.cpp ├── LICENSE_1_0.txt ├── sarah_test.cpp ├── bigbatch_sgd_test.cpp ├── delta_bar_delta_test.cpp ├── katyusha_test.cpp ├── momentum_delta_bar_delta_test.cpp ├── wn_grad_test.cpp ├── svrg_test.cpp ├── main.cpp ├── swats_test.cpp ├── eve_test.cpp ├── spsa_test.cpp ├── ftml_test.cpp ├── yogi_test.cpp ├── CMakeLists.txt ├── sgdr_test.cpp ├── demon_adam_test.cpp ├── cmaes_test.cpp ├── lookahead_test.cpp ├── data │ └── r10.txt ├── snapshot_ensembles.cpp ├── active_cmaes_test.cpp ├── proximal_test.cpp └── ada_bound_test.cpp ├── UPDATING.txt ├── include └── ensmallen_bits │ ├── wn_grad │ └── wn_grad_impl.hpp │ ├── problems │ ├── gradient_descent_test_function_impl.hpp │ ├── fw_test_function.hpp │ ├── holder_table_function_impl.hpp │ ├── schaffer_function_n4_impl.hpp │ ├── cross_in_tray_function_impl.hpp │ ├── quadratic_function_impl.hpp │ ├── gradient_descent_test_function.hpp │ ├── problems.hpp │ ├── rosenbrock_wood_function_impl.hpp │ ├── schaffer_function_n4.hpp │ ├── booth_function_impl.hpp │ ├── sgd_test_function.hpp │ ├── sphere_function_impl.hpp │ ├── matyas_function_impl.hpp │ └── mc_cormick_function_impl.hpp │ ├── yogi │ └── yogi_impl.hpp │ ├── delta_bar_delta │ ├── delta_bar_delta_impl.hpp │ └── momentum_delta_bar_delta_impl.hpp │ ├── swats │ └── swats_impl.hpp │ ├── ada_sqrt │ └── ada_sqrt_impl.hpp │ ├── utility │ └── detect_callbacks.hpp │ ├── ada_belief │ └── ada_belief_impl.hpp │ ├── qhadam │ └── qhadam_impl.hpp │ ├── smorms3 │ └── smorms3_impl.hpp │ ├── ada_grad │ └── ada_grad_impl.hpp │ ├── ftml │ └── ftml_impl.hpp │ ├── adam │ └── adam_impl.hpp │ ├── ada_bound │ └── ada_bound_impl.hpp │ ├── ada_delta │ └── ada_delta_impl.hpp │ ├── cmaes │ ├── transformation_policies │ │ └── empty_transformation.hpp │ └── full_selection.hpp │ ├── config.hpp │ ├── parallel_sgd │ └── decay_policies │ │ └── constant_step.hpp │ ├── sarah │ ├── sarah_update.hpp │ └── sarah_plus_update.hpp │ ├── callbacks │ ├── grad_clip_by_value.hpp │ ├── print_loss.hpp │ ├── grad_clip_by_norm.hpp │ └── store_best_coordinates.hpp │ ├── fw │ ├── proximal │ │ └── proximal.hpp │ ├── update_classic.hpp │ └── func_sq.hpp │ ├── ens_version.hpp │ ├── moead │ ├── weight_init_policies │ │ ├── dirichlet_init.hpp │ │ └── bbs_init.hpp │ └── decomposition_policies │ │ ├── weighted_decomposition.hpp │ │ └── tchebycheff_decomposition.hpp │ ├── fbs │ ├── l1_penalty.hpp │ └── l1_penalty_impl.hpp │ ├── sa │ └── exponential_schedule.hpp │ ├── cd │ └── descent_policies │ │ ├── cyclic_descent.hpp │ │ ├── random_descent.hpp │ │ └── greedy_descent.hpp │ ├── sdp │ └── lrsdp_impl.hpp │ ├── sgd │ └── update_policies │ │ └── vanilla_update.hpp │ ├── svrg │ └── svrg_update.hpp │ ├── grid_search │ └── grid_search.hpp │ └── log.hpp ├── .travis.yml ├── example.cpp └── .appveyor.yml /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | -------------------------------------------------------------------------------- /CMake/ensmallen-config.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include(${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake) 4 | check_required_components(ensmallen) 5 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | The documentation in this directory is what is used to generate the page 2 | 3 | http://www.ensmallen.org/docs.html 4 | 5 | You can read it here directly also. 6 | 7 | If you are implementing a new optimizer, be sure to add documentation to 8 | optimizers.md and links in function_types.md! 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Use this template for other problems, requests, or questions. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 19 | -------------------------------------------------------------------------------- /scripts/history-update-check.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Check each PR has an entry in HISTORY.md during a CI routine. 4 | # 5 | # Arguments: 6 | # $ history-update-check.sh 7 | # 8 | # This should be run from the root of the repository. 9 | res=$(git diff origin/master --name-only | grep ^HISTORY.md | wc -l) 10 | echo "Files Changed:" 11 | git diff --name-only 12 | if [ $res -gt 0 ]; then 13 | echo "HISTORY.md was updated with a change for the PR ..." 14 | else 15 | echo "Please describe your PR changes in HISTORY.md ..." 16 | echo "Exiting CI process ... " 17 | exit 1 18 | fi 19 | 20 | exit 0 21 | -------------------------------------------------------------------------------- /.github/workflows/auto-approve.yml: -------------------------------------------------------------------------------- 1 | # Once a PR has been approved by one member of the mlpack organization, a second 2 | # approving review will automatically be added 24 hours later. This allows time 3 | # for other maintainers to take a look. 4 | name: Auto-approve pull requests 5 | on: 6 | schedule: 7 | # Run roughly every four hours. 8 | - cron: "15 0,4,8,12,16,20 * * *" 9 | 10 | jobs: 11 | auto-approve: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | pull-requests: write 15 | 16 | steps: 17 | - name: Auto-approve pull requests 18 | uses: rcurtin/actions/auto-approve@v1 19 | with: 20 | repo-token: ${{ secrets.GITHUB_TOKEN }} 21 | approval-message: 22 | 'Second approval provided automatically after 24 hours. :+1:' 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/1-documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation issue 3 | about: Use this template to report an issue you've found with the documentation. 4 | title: '' 5 | labels: 't: bug report, c: documentation, s: unanswered' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 18 | 19 | #### Problem location 20 | 21 | 23 | 24 | #### Description of problem 25 | 26 | 27 | -------------------------------------------------------------------------------- /tests/ada_belief_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_belief_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("AdaBelief_LogisticRegressionFunction", "[AdaBelief]", 23 | ENS_ALL_CPU_TEST_TYPES) 24 | { 25 | AdaBelief adaBelief(0.032); 26 | LogisticRegressionFunctionTest(adaBelief); 27 | } 28 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close inactive issues 2 | on: 3 | schedule: 4 | - cron: "30 1 * * *" 5 | 6 | jobs: 7 | close-issues: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | issues: write 11 | pull-requests: write 12 | steps: 13 | - uses: actions/stale@v9 14 | with: 15 | days-before-issues-stale: 30 16 | days-before-issue-close: 7 17 | stale-issue-label: "s: stale" 18 | stale-pr-label: "s: stale" 19 | stale-issue-message: "This issue has been automatically marked as stale because it has not had any recent activity. It will be closed in 7 days if no further activity occurs. Thank you for your contributions! :+1:" 20 | days-before-pr-stale: 30, 21 | days-before-pr-close: 14 22 | repo-token: ${{ secrets.GITHUB_TOKEN }} 23 | exempt-issue-labels: "s: keep open" 24 | exempt-pr-labels: "s: keep open" 25 | -------------------------------------------------------------------------------- /tests/smorms3_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file snorms3_test.cpp 3 | * @author Vivek Pal 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | #include "catch.hpp" 18 | #include "test_function_tools.hpp" 19 | #include "test_types.hpp" 20 | 21 | using namespace ens; 22 | using namespace ens::test; 23 | 24 | TEMPLATE_TEST_CASE("SMORMS3_LogisticRegressionFunction", "[SMORMS3]", 25 | ENS_ALL_TEST_TYPES) 26 | { 27 | SMORMS3 smorms3(0.032); 28 | LogisticRegressionFunctionTest(smorms3); 29 | } 30 | -------------------------------------------------------------------------------- /UPDATING.txt: -------------------------------------------------------------------------------- 1 | ensmallen uses semantic versioning for its versioning conventions 2 | (http://semver.org). 3 | 4 | The current version of ensmallen can be found with the ens::version::as_string() 5 | function, or the ENS_VERSION_MAJOR, ENS_VERSION_MINOR, and ENS_VERSION_PATCH 6 | macros. 7 | 8 | In practice, this generally means: 9 | 10 | * A MAJOR version bump implies a backwards-incompatible change to the 11 | user-facing optimizers. Code using one major version of ensmallen may not 12 | work with a newer major version of ensmallen. 13 | 14 | * A MINOR version bump happens for feature additions to ensmallen. Typically 15 | this might be the addition of a new optimizer. Code using one version of 16 | ensmallen will work with all subsequent minor releases of the same major 17 | release line. 18 | 19 | * A PATCH version bump indicates some small bug fixes or optimizations. No new 20 | functionality is added and no compatibility is broken. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/0-bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Use this template for reporting a bug that you have found in ensmallen. 4 | title: '' 5 | labels: 't: bug report, s: unanswered' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 16 | 17 | #### Issue description 18 | 19 | 20 | 21 | #### Your environment 22 | 23 | * version of ensmallen: 24 | * operating system: 25 | * compiler: 26 | * version of Armadillo: 27 | * any other environment information you think is relevant: 28 | 29 | #### Steps to reproduce 30 | 31 | 33 | 34 | #### Expected behavior 35 | 36 | 37 | 38 | #### Actual behavior 39 | 40 | 41 | -------------------------------------------------------------------------------- /CMake/ARMA_FindARPACK.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find ARPACK 2 | # Once done this will define 3 | # 4 | # ARPACK_FOUND - system has ARPACK 5 | # ARPACK_LIBRARY - Link this to use ARPACK 6 | 7 | 8 | find_library(ARPACK_LIBRARY 9 | NAMES arpack 10 | PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib 11 | ) 12 | 13 | 14 | if (ARPACK_LIBRARY) 15 | set(ARPACK_FOUND YES) 16 | else () 17 | # Search for PARPACK. 18 | find_library(ARPACK_LIBRARY 19 | NAMES parpack 20 | PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib 21 | ) 22 | 23 | if (ARPACK_LIBRARY) 24 | set(ARPACK_FOUND YES) 25 | else () 26 | set(ARPACK_FOUND NO) 27 | endif () 28 | endif () 29 | 30 | 31 | if (ARPACK_FOUND) 32 | if (NOT ARPACK_FIND_QUIETLY) 33 | message(STATUS "Found an ARPACK library: ${ARPACK_LIBRARY}") 34 | endif () 35 | else () 36 | if (ARPACK_FIND_REQUIRED) 37 | message(FATAL_ERROR "Could not find an ARPACK library") 38 | endif () 39 | endif () 40 | -------------------------------------------------------------------------------- /tests/ada_grad_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_grad_test.cpp 3 | * @author Abhinav Moudgil 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | #include "catch.hpp" 18 | #include "test_function_tools.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | TEMPLATE_TEST_CASE("AdaGrad_LogisticRegressionFunction", "[AdaGrad]", 24 | ENS_ALL_TEST_TYPES) 25 | { 26 | AdaGrad adagrad(31.5, 32, 10 * Tolerances::Obj, 500000, 27 | Tolerances::Obj, true); 28 | LogisticRegressionFunctionTest(adagrad); 29 | } 30 | -------------------------------------------------------------------------------- /tests/de_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file de_test.cpp 3 | * @author Rahul Ganesh Prabhu 4 | * @author Marcus Edel 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | #include "test_types.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | TEMPLATE_TEST_CASE("DE_LogisticRegressionFunction", "[DE]", ENS_ALL_TEST_TYPES) 24 | { 25 | DE opt(200, 1000, 0.6, 0.8, 1e-5); 26 | LogisticRegressionFunctionTest>( 27 | opt, 28 | 4 * Tolerances::LRTrainAcc, 29 | 4 * Tolerances::LRTestAcc, 30 | 3); 31 | } 32 | -------------------------------------------------------------------------------- /CMake/COOT_FindNVRTC.cmake: -------------------------------------------------------------------------------- 1 | # - Find clBlast (includes and library) 2 | # This module defines 3 | # CLBLAST_INCLUDE_DIR 4 | # CLBLAST_LIBRARIES 5 | # CLBLAST_FOUND 6 | # also defined, but not for general use are 7 | # CLBLAST_LIBRARY, where to find the library. 8 | 9 | set(NVRTC_NAMES ${NVRTC_NAMES} nvrtc) 10 | find_library(NVRTC_LIBRARY 11 | NAMES ${NVRTC_NAMES} 12 | PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib /usr/lib/x86_64-linux-gnu/ 13 | ) 14 | 15 | if (NVRTC_LIBRARY) 16 | set(NVRTC_LIBRARIES ${NVRTC_LIBRARY}) 17 | set(NVRTC_FOUND "YES") 18 | else () 19 | set(NVRTC_FOUND "NO") 20 | endif () 21 | 22 | if (NVRTC_FOUND) 23 | if (NOT NVRTC_FIND_QUIETLY) 24 | message(STATUS "Found NVRTC library: ${NVRTC_LIBRARIES}") 25 | endif () 26 | else () 27 | if (NVRTC_FIND_REQUIRED) 28 | message(FATAL_ERROR "Could not find NVRTC library") 29 | endif () 30 | endif () 31 | 32 | # Deprecated declarations. 33 | get_filename_component (NATIVE_NVRTC_LIB_PATH ${NVRTC_LIBRARY} PATH) 34 | 35 | mark_as_advanced(NVRTC_LIBRARY) 36 | -------------------------------------------------------------------------------- /tests/demon_sgd_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file demon_sgd_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("DemonSGD_LogisticRegressionFunction", "[DemonSGD]", 23 | ENS_ALL_TEST_TYPES) 24 | { 25 | DemonSGD optimizer(32.0 /* huge step size needed! */, 16, 0.9, 10000000, 1e-9, 26 | true, true, true); 27 | // It could take several tries to get everything to converge. 28 | LogisticRegressionFunctionTest(optimizer, 29 | Tolerances::LRTrainAcc, 30 | Tolerances::LRTestAcc, 31 | 10); 32 | } 33 | -------------------------------------------------------------------------------- /CMake/ARMA_FindOpenBLAS.cmake: -------------------------------------------------------------------------------- 1 | # - Find the OpenBLAS library (no includes) 2 | # This module defines 3 | # OpenBLAS_LIBRARIES, the libraries needed to use OpenBLAS. 4 | # OpenBLAS_FOUND, If false, do not try to use OpenBLAS. 5 | # also defined, but not for general use are 6 | # OpenBLAS_LIBRARY, where to find the OpenBLAS library. 7 | 8 | set(OpenBLAS_NAMES ${OpenBLAS_NAMES} openblas) 9 | find_library(OpenBLAS_LIBRARY 10 | NAMES ${OpenBLAS_NAMES} 11 | PATHS /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib 12 | ) 13 | 14 | if (OpenBLAS_LIBRARY) 15 | set(OpenBLAS_LIBRARIES ${OpenBLAS_LIBRARY}) 16 | set(OpenBLAS_FOUND "YES") 17 | else () 18 | set(OpenBLAS_FOUND "NO") 19 | endif () 20 | 21 | 22 | if (OpenBLAS_FOUND) 23 | if (NOT OpenBLAS_FIND_QUIETLY) 24 | message(STATUS "Found the OpenBLAS library: ${OpenBLAS_LIBRARIES}") 25 | endif () 26 | else () 27 | if (OpenBLAS_FIND_REQUIRED) 28 | message(FATAL_ERROR "Could not find the OpenBLAS library") 29 | endif () 30 | endif () 31 | 32 | # Deprecated declarations. 33 | get_filename_component (NATIVE_OpenBLAS_LIB_PATH ${OpenBLAS_LIBRARY} PATH) 34 | 35 | mark_as_advanced( 36 | OpenBLAS_LIBRARY 37 | ) 38 | -------------------------------------------------------------------------------- /tests/ada_delta_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_delta_test.cpp 3 | * @author Marcus Edel 4 | * @author Vasanth Kalingeri 5 | * @author Abhinav Moudgil 6 | * @author Conrad Sanderson 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #if defined(ENS_USE_COOT) 14 | #include 15 | #include 16 | #endif 17 | #include 18 | #include "catch.hpp" 19 | #include "test_function_tools.hpp" 20 | #include "test_types.hpp" 21 | 22 | using namespace ens; 23 | using namespace ens::test; 24 | 25 | TEMPLATE_TEST_CASE("AdaDelta_LogisticRegressionFunction", "[AdaDelta]", 26 | ENS_ALL_TEST_TYPES) 27 | { 28 | typedef typename TestType::elem_type ElemType; 29 | 30 | // Use a large epsilon if we are using FP16, to avoid underflow in the first 31 | // iterations. 32 | AdaDelta adaDelta(32.0, 32, 0.95, sizeof(ElemType) == 2 ? 1e-4 : 1e-6); 33 | LogisticRegressionFunctionTest(adaDelta); 34 | } 35 | -------------------------------------------------------------------------------- /.github/workflows/stickers.yml: -------------------------------------------------------------------------------- 1 | # Post a message to new contributors that they can get some stickers mailed to 2 | # them. 3 | name: 'Stickers for new contributors' 4 | on: 5 | pull_request: 6 | types: [closed] 7 | 8 | jobs: 9 | sticker_comment: 10 | runs-on: ubuntu-latest 11 | if: github.event.pull_request.merged == true 12 | steps: 13 | # Forked version of first-interaction that runs only on first merged PR. 14 | - uses: rcurtin/actions/stickers@v1 15 | with: 16 | repo-token: ${{ secrets.GITHUB_TOKEN }} 17 | pr-message: "Hello there! Thanks for your contribution. Congratulations on your first contribution to mlpack! If you'd like to add your name to the list of contributors in `COPYRIGHT.txt` and you haven't already, please feel free to push a change to this PR---or, if it gets merged before you can, feel free to open another PR.\n\nIn addition, if you'd like some stickers to put on your laptop, we can get them in the mail for you. Just send an email with your physical mailing address to stickers@mlpack.org, and then one of the mlpack maintainers will put some stickers in an envelope for you. It may take a few weeks to get them, depending on your location. :+1:" 18 | -------------------------------------------------------------------------------- /tests/rmsprop_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file rmsprop_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | #include "test_types.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | TEMPLATE_TEST_CASE("RMSProp_LogisticRegressionFunction", "[RMSProp]", 24 | ENS_ALL_TEST_TYPES, ENS_SPARSE_TEST_TYPES) 25 | { 26 | RMSProp optimizer(0.32); 27 | LogisticRegressionFunctionTest( 28 | optimizer, 29 | Tolerances::LRTrainAcc, 30 | Tolerances::LRTestAcc, 31 | // Low-precision may need a few trials because it sometimes diverges 32 | // (gradient or update is too large). 33 | (sizeof(typename TestType::elem_type) < 4) ? 5 : 1); 34 | } 35 | -------------------------------------------------------------------------------- /tests/ada_sqrt_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_sqrt_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("AdaSqrt_LogisticRegressionFunction", "[AdaSqrt]", 23 | ENS_ALL_TEST_TYPES) 24 | { 25 | AdaSqrt optimizer(8.0, 32, 10 * Tolerances::Obj, 150000, 1e-9, 26 | true); 27 | // We allow a few trials for lower precision types because AdaSqrt can have 28 | // trouble converging in that case. 29 | LogisticRegressionFunctionTest( 30 | optimizer, 31 | Tolerances::LRTrainAcc, 32 | Tolerances::LRTestAcc, 33 | sizeof(typename TestType::elem_type) < 4 ? 5 : 1); 34 | } 35 | -------------------------------------------------------------------------------- /tests/spalera_sgd_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file spalera_sgd_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | #include "test_types.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | TEMPLATE_TEST_CASE("SPALeRASGD_LogisticRegressionFunction", "[SPALeRASGD]", 24 | ENS_ALL_TEST_TYPES) 25 | { 26 | // Run SPALeRA SGD with a couple of batch sizes. 27 | for (size_t batchSize = 30; batchSize < 50; batchSize += 5) 28 | { 29 | SPALeRASGD<> optimizer(0.05 / batchSize, batchSize, 10000, 1e-4); 30 | LogisticRegressionFunctionTest( 31 | optimizer, 32 | 5 * Tolerances::LRTrainAcc, 33 | 4 * Tolerances::LRTestAcc, 34 | 3); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /CMake/COOT_FindCLBLAS.cmake: -------------------------------------------------------------------------------- 1 | # - Find clBLAS (includes and library) 2 | # This module defines 3 | # CLBLAS_INCLUDE_DIR 4 | # CLBLAS_LIBRARIES 5 | # CLBLAS_FOUND 6 | # also defined, but not for general use are 7 | # CLBLAS_LIBRARY, where to find the library. 8 | 9 | find_path(CLBLAS_INCLUDE_DIR clBLAS.h 10 | /usr/include/ 11 | /usr/local/include/ 12 | ) 13 | 14 | set(CLBLAS_NAMES ${CLBLAS_NAMES} clBLAS) 15 | find_library(CLBLAS_LIBRARY 16 | NAMES ${CLBLAS_NAMES} 17 | PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib 18 | ) 19 | 20 | if (CLBLAS_LIBRARY AND CLBLAS_INCLUDE_DIR) 21 | set(CLBLAS_LIBRARIES ${CLBLAS_LIBRARY}) 22 | set(CLBLAS_FOUND "YES") 23 | else () 24 | set(CLBLAS_FOUND "NO") 25 | endif () 26 | 27 | if (CLBLAS_FOUND) 28 | if (NOT CLBLAS_FIND_QUIETLY) 29 | message(STATUS "Found a clBLAS library: ${CLBLAS_LIBRARIES}") 30 | endif () 31 | else () 32 | if (CLBLAS_FIND_REQUIRED) 33 | message(FATAL_ERROR "Could not find a clBLAS library") 34 | endif () 35 | endif () 36 | 37 | # Deprecated declarations. 38 | set (NATIVE_CLBLAS_INCLUDE_PATH ${CLBLAS_INCLUDE_DIR} ) 39 | get_filename_component (NATIVE_CLBLAS_LIB_PATH ${CLBLAS_LIBRARY} PATH) 40 | 41 | mark_as_advanced( 42 | CLBLAS_LIBRARY 43 | CLBLAS_INCLUDE_DIR 44 | ) 45 | -------------------------------------------------------------------------------- /CMake/ARMA_FindACML.cmake: -------------------------------------------------------------------------------- 1 | # - Find AMD's ACML library (no includes) which provides optimised BLAS and LAPACK functions 2 | # This module defines 3 | # ACML_LIBRARIES, the libraries needed to use ACML. 4 | # ACML_FOUND, If false, do not try to use ACML. 5 | # also defined, but not for general use are 6 | # ACML_LIBRARY, where to find the ACML library. 7 | 8 | set(ACML_NAMES ${ACML_NAMES} acml) 9 | find_library(ACML_LIBRARY 10 | NAMES ${ACML_NAMES} 11 | PATHS /usr/lib64 /usr/lib /usr/*/lib64 /usr/*/lib /usr/*/gfortran64/lib/ /usr/*/gfortran32/lib/ /usr/local/lib64 /usr/local/lib /opt/lib64 /opt/lib /opt/*/lib64 /opt/*/lib /opt/*/gfortran64/lib/ /opt/*/gfortran32/lib/ 12 | ) 13 | 14 | if (ACML_LIBRARY) 15 | set(ACML_LIBRARIES ${ACML_LIBRARY}) 16 | set(ACML_FOUND "YES") 17 | else () 18 | set(ACML_FOUND "NO") 19 | endif () 20 | 21 | 22 | if (ACML_FOUND) 23 | if (NOT ACML_FIND_QUIETLY) 24 | message(STATUS "Found the ACML library: ${ACML_LIBRARIES}") 25 | endif () 26 | else () 27 | if (ACML_FIND_REQUIRED) 28 | message(FATAL_ERROR "Could not find the ACML library") 29 | endif () 30 | endif () 31 | 32 | # Deprecated declarations. 33 | get_filename_component (NATIVE_ACML_LIB_PATH ${ACML_LIBRARY} PATH) 34 | 35 | mark_as_advanced( 36 | ACML_LIBRARY 37 | ) 38 | -------------------------------------------------------------------------------- /include/ensmallen_bits/wn_grad/wn_grad_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file wn_grad_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the WNGrad optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_WN_GRAD_WN_GRAD_IMPL_HPP 13 | #define ENSMALLEN_WN_GRAD_WN_GRAD_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "wn_grad.hpp" 17 | 18 | namespace ens { 19 | 20 | inline WNGrad::WNGrad( 21 | const double stepSize, 22 | const size_t batchSize, 23 | const size_t maxIterations, 24 | const double tolerance, 25 | const bool shuffle, 26 | const bool resetPolicy, 27 | const bool exactObjective) : 28 | optimizer(stepSize, 29 | batchSize, 30 | maxIterations, 31 | tolerance, 32 | shuffle, 33 | WNGradUpdate(), 34 | NoDecay(), 35 | resetPolicy, 36 | exactObjective) 37 | { /* Nothing to do. */ } 38 | 39 | } // namespace ens 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /tests/iqn_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file iqn_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * Test file for IQN (incremental Quasi-Newton). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | #include "catch.hpp" 18 | #include "test_function_tools.hpp" 19 | #include "test_types.hpp" 20 | 21 | using namespace ens; 22 | using namespace ens::test; 23 | 24 | // NOTE: IQN cannot use arma::hmat because pinv() is required. 25 | 26 | TEMPLATE_TEST_CASE("IQN_LogisticRegressionFunction", "[IQN]", ENS_TEST_TYPES) 27 | { 28 | // Run on a couple of batch sizes. 29 | for (size_t batchSize = 1; batchSize < 9; batchSize += 4) 30 | { 31 | IQN iqn(0.01, batchSize, 200000, 0.01); 32 | // It could take a few attempts to converge. 33 | LogisticRegressionFunctionTest(iqn, 34 | Tolerances::LRTrainAcc, 35 | Tolerances::LRTestAcc, 36 | 5); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /CMake/ARMA_FindBLAS.cmake: -------------------------------------------------------------------------------- 1 | # - Find a BLAS library (no includes) 2 | # This module defines 3 | # BLAS_LIBRARIES, the libraries needed to use BLAS. 4 | # BLAS_FOUND, If false, do not try to use BLAS. 5 | # also defined, but not for general use are 6 | # BLAS_LIBRARY, where to find the BLAS library. 7 | 8 | set(BLAS_NAMES ${BLAS_NAMES} blas) 9 | 10 | # Find the ATLAS version preferentially. 11 | find_library(BLAS_LIBRARY 12 | NAMES ${BLAS_NAMES} 13 | PATHS /usr/lib64/atlas /usr/lib/atlas /usr/local/lib64/atlas /usr/local/lib/atlas 14 | NO_DEFAULT_PATH) 15 | 16 | find_library(BLAS_LIBRARY 17 | NAMES ${BLAS_NAMES} 18 | PATHS /usr/lib64/atlas /usr/lib/atlas /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib 19 | ) 20 | 21 | if (BLAS_LIBRARY) 22 | set(BLAS_LIBRARIES ${BLAS_LIBRARY}) 23 | set(BLAS_FOUND "YES") 24 | else () 25 | set(BLAS_FOUND "NO") 26 | endif () 27 | 28 | 29 | if (BLAS_FOUND) 30 | if (NOT BLAS_FIND_QUIETLY) 31 | message(STATUS "Found BLAS: ${BLAS_LIBRARIES}") 32 | endif () 33 | else () 34 | if (BLAS_FIND_REQUIRED) 35 | message(FATAL_ERROR "Could not find BLAS") 36 | endif () 37 | endif () 38 | 39 | # Deprecated declarations. 40 | get_filename_component (NATIVE_BLAS_LIB_PATH ${BLAS_LIBRARY} PATH) 41 | 42 | mark_as_advanced( 43 | BLAS_LIBRARY 44 | ) 45 | -------------------------------------------------------------------------------- /CMake/COOT_FindCLBlast.cmake: -------------------------------------------------------------------------------- 1 | # - Find clBlast (includes and library) 2 | # This module defines 3 | # CLBLAST_INCLUDE_DIR 4 | # CLBLAST_LIBRARIES 5 | # CLBLAST_FOUND 6 | # also defined, but not for general use are 7 | # CLBLAST_LIBRARY, where to find the library. 8 | 9 | find_path(CLBLAST_INCLUDE_DIR clblast.h 10 | /usr/include/ 11 | /usr/local/include/ 12 | ) 13 | 14 | set(CLBLAST_NAMES ${CLBLAST_NAMES} clblast) 15 | find_library(CLBLAST_LIBRARY 16 | NAMES ${CLBLAST_NAMES} 17 | PATHS /usr/lib64/ /usr/local/lib64/ /usr/lib /usr/local/lib 18 | ) 19 | 20 | if (CLBLAST_LIBRARY AND CLBLAST_INCLUDE_DIR) 21 | set(CLBLAST_LIBRARIES ${CLBLAST_LIBRARY}) 22 | set(CLBLAST_FOUND "YES") 23 | else () 24 | set(CLBLAST_FOUND "NO") 25 | endif () 26 | 27 | if (CLBLAST_FOUND) 28 | if (NOT CLBLAST_FIND_QUIETLY) 29 | message(STATUS "Found a clBlast library: ${CLBLAST_LIBRARIES}") 30 | endif () 31 | else () 32 | if (CLBLAST_FIND_REQUIRED) 33 | message(FATAL_ERROR "Could not find a clBlast library") 34 | endif () 35 | endif () 36 | 37 | # Deprecated declarations. 38 | set (NATIVE_CLBLAST_INCLUDE_PATH ${CLBLAST_INCLUDE_DIR} ) 39 | get_filename_component (NATIVE_CLBLAST_LIB_PATH ${CLBLAST_LIBRARY} PATH) 40 | 41 | mark_as_advanced( 42 | CLBLAST_LIBRARY 43 | CLBLAST_INCLUDE_DIR 44 | ) 45 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/gradient_descent_test_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file gradient_descent_test_function_impl.hpp 3 | * @author Sumedh Ghaisas 4 | * 5 | * Implementation of very simple test function for gradient descent. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_GRADIENT_DESCENT_TEST_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_GRADIENT_DESCENT_TEST_FUNCTION_IMPL_HPP 14 | 15 | #include "gradient_descent_test_function.hpp" 16 | 17 | namespace ens { 18 | namespace test { 19 | 20 | template 21 | inline typename MatType::elem_type GDTestFunction::Evaluate( 22 | const MatType& coordinates) const 23 | { 24 | MatType temp = trans(coordinates) * coordinates; 25 | return temp(0, 0); 26 | } 27 | 28 | template 29 | inline void GDTestFunction::Gradient(const MatType& coordinates, 30 | GradType& gradient) const 31 | { 32 | gradient = 2 * coordinates; 33 | } 34 | 35 | } // namespace test 36 | } // namespace ens 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | dist: focal 3 | language: cpp 4 | 5 | env: 6 | - ARMADILLO=latest SANITY_HISTORY=perform 7 | - ARMADILLO=minimum 8 | 9 | stages: 10 | - test 11 | - name: sanity 12 | if: type = pull_request AND env(SANITY_HISTORY) = "perform" 13 | 14 | jobs: 15 | include: 16 | - stage: sanity 17 | name: "HISTORY.md Check" 18 | script: sh ./scripts/history-update-check.sh 19 | 20 | script: 21 | - sudo apt-get update 22 | - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ xz-utils 23 | - if [ $ARMADILLO == "latest" ]; then 24 | curl https://ftp.fau.de/macports/distfiles/armadillo/`curl https://ftp.fau.de/macports/distfiles/armadillo/ -- | grep '.tar.xz' | sed 's/^.* 20 | struct IsAllNonMatrix; 21 | 22 | template 23 | struct IsAllNonMatrix 24 | { 25 | constexpr static bool tIsClass = std::is_class::type>::type>::value; 27 | 28 | constexpr static bool value = 29 | tIsClass && !IsMatrixType::value && 30 | IsAllNonMatrix::value; 31 | }; 32 | 33 | template<> 34 | struct IsAllNonMatrix<> 35 | { 36 | constexpr static bool value = true; 37 | }; 38 | 39 | } // namespace ens 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /tests/line_search_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file line_search_test.cpp 3 | * @author Chenzhe Diao 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | #include "test_types.hpp" 16 | 17 | using namespace arma; 18 | using namespace ens; 19 | using namespace ens::test; 20 | 21 | /** 22 | * Simple test of Line Search with TestFuncFW function. 23 | */ 24 | TEMPLATE_TEST_CASE("FuncFWTest", "[LineSearch]", ENS_TEST_TYPES) 25 | { 26 | typedef typename TestType::elem_type ElemType; 27 | 28 | TestType x1 = zeros(3, 1); 29 | TestType x2 = { 0.2, 0.4, 0.6 }; 30 | x2 = x2.t(); 31 | 32 | TestFuncFW f; 33 | LineSearch s; 34 | 35 | ElemType result = s.Optimize(f, x1, x2); 36 | 37 | REQUIRE(result == Approx(0.0).margin(Tolerances::Obj)); 38 | REQUIRE((x2(0) - 0.1) == Approx(0.0).margin(Tolerances::Coord)); 39 | REQUIRE((x2(1) - 0.2) == Approx(0.0).margin(Tolerances::Coord)); 40 | REQUIRE((x2(2) - 0.3) == Approx(0.0).margin(Tolerances::Coord)); 41 | } 42 | -------------------------------------------------------------------------------- /tests/gradient_descent_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file gradient_descent_test.cpp 3 | * @author Sumedh Ghaisas 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | #include "catch.hpp" 18 | #include "test_function_tools.hpp" 19 | #include "test_types.hpp" 20 | 21 | using namespace ens; 22 | using namespace ens::test; 23 | 24 | TEMPLATE_TEST_CASE("GradientDescent_GDTestFunction", "[GradientDescent]", 25 | ENS_ALL_TEST_TYPES) 26 | { 27 | GradientDescent s(0.01, 5000000, 1e-9); 28 | FunctionTest(s, 29 | Tolerances::LargeObj, 30 | Tolerances::LargeCoord); 31 | } 32 | 33 | TEMPLATE_TEST_CASE("GradientDescent_RosenbrockFunction", "[GradientDescent]", 34 | ENS_ALL_TEST_TYPES) 35 | { 36 | GradientDescent s(0.002, 0, Tolerances::Obj / 1000); 37 | FunctionTest(s, 38 | 10 * Tolerances::LargeObj, 39 | 10 * Tolerances::LargeCoord); 40 | } 41 | -------------------------------------------------------------------------------- /tests/LICENSE_1_0.txt: -------------------------------------------------------------------------------- 1 | Boost Software License - Version 1.0 - August 17th, 2003 2 | 3 | Permission is hereby granted, free of charge, to any person or organization 4 | obtaining a copy of the software and accompanying documentation covered by 5 | this license (the "Software") to use, reproduce, display, distribute, 6 | execute, and transmit the Software, and to prepare derivative works of the 7 | Software, and to permit third-parties to whom the Software is furnished to 8 | do so, all subject to the following: 9 | 10 | The copyright notices in the Software and this entire statement, including 11 | the above license grant, this restriction and the following disclaimer, 12 | must be included in all copies of the Software, in whole or in part, and 13 | all derivative works of the Software, unless such copies or derivative 14 | works are solely in the form of machine-executable object code generated by 15 | a source language processor. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 20 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 21 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_belief/ada_belief_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_belief_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of AdaBelief class wrapper. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP 13 | #define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "ada_belief.hpp" 17 | 18 | namespace ens { 19 | 20 | inline AdaBelief::AdaBelief( 21 | const double stepSize, 22 | const size_t batchSize, 23 | const double beta1, 24 | const double beta2, 25 | const double epsilon, 26 | const size_t maxIterations, 27 | const double tolerance, 28 | const bool shuffle, 29 | const bool resetPolicy, 30 | const bool exactObjective) : 31 | optimizer(stepSize, 32 | batchSize, 33 | maxIterations, 34 | tolerance, 35 | shuffle, 36 | AdaBeliefUpdate(epsilon, beta1, beta2), 37 | NoDecay(), 38 | resetPolicy, 39 | exactObjective) 40 | { /* Nothing to do. */ } 41 | 42 | } // namespace ens 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /include/ensmallen_bits/delta_bar_delta/momentum_delta_bar_delta_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file momentum_delta_bar_delta_impl.hpp 3 | * @author Ranjodh Singh 4 | * 5 | * Implementation of MomentumDeltaBarDelta class wrapper. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP 13 | #define ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "./momentum_delta_bar_delta.hpp" 17 | 18 | namespace ens { 19 | 20 | inline MomentumDeltaBarDelta::MomentumDeltaBarDelta( 21 | const double stepSize, 22 | const size_t maxIterations, 23 | const double tolerance, 24 | const double kappa, 25 | const double phi, 26 | const double momentum, 27 | const double minGain, 28 | const bool resetPolicy) : 29 | optimizer(stepSize, 30 | maxIterations, 31 | tolerance, 32 | MomentumDeltaBarDeltaUpdate(kappa, phi, momentum, minGain), 33 | NoDecay(), 34 | resetPolicy) 35 | { 36 | /* Nothing to do. */ 37 | } 38 | 39 | } // namespace ens 40 | 41 | #endif // ENSMALLEN_MOMENTUM_DELTA_BAR_DELTA_IMPL_HPP 42 | -------------------------------------------------------------------------------- /include/ensmallen_bits/qhadam/qhadam_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file qhadam_impl.hpp 3 | * @author Niteya Shah 4 | * 5 | * Implementation of QHAdam class wrapper. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADAM_QHADAM_IMPL_HPP 13 | #define ENSMALLEN_ADAM_QHADAM_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "qhadam.hpp" 17 | 18 | namespace ens { 19 | 20 | inline QHAdam::QHAdam( 21 | const double stepSize, 22 | const size_t batchSize, 23 | const double v1, 24 | const double v2, 25 | const double beta1, 26 | const double beta2, 27 | const double epsilon, 28 | const size_t maxIterations, 29 | const double tolerance, 30 | const bool shuffle, 31 | const bool resetPolicy, 32 | const bool exactObjective) : 33 | optimizer(stepSize, 34 | batchSize, 35 | maxIterations, 36 | tolerance, 37 | shuffle, 38 | QHAdamUpdate(epsilon, beta1, beta2, v1, v2), 39 | NoDecay(), 40 | resetPolicy, 41 | exactObjective) 42 | { /* Nothing to do. */ } 43 | 44 | } // namespace ens 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /include/ensmallen_bits/smorms3/smorms3_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file smorms3_impl.hpp 3 | * @author Vivek Pal 4 | * 5 | * Implementation of the SMORMS3 constructor. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SMORMS3_SMORMS3_IMPL_HPP 13 | #define ENSMALLEN_SMORMS3_SMORMS3_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "smorms3.hpp" 17 | 18 | namespace ens { 19 | 20 | inline SMORMS3::SMORMS3(const double stepSize, 21 | const size_t batchSize, 22 | const double epsilon, 23 | const size_t maxIterations, 24 | const double tolerance, 25 | const bool shuffle, 26 | const bool resetPolicy, 27 | const bool exactObjective) : 28 | optimizer(stepSize, 29 | batchSize, 30 | maxIterations, 31 | tolerance, 32 | shuffle, 33 | SMORMS3Update(epsilon), 34 | NoDecay(), 35 | resetPolicy, 36 | exactObjective) 37 | { /* Nothing to do. */ } 38 | 39 | } // namespace ens 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_grad/ada_grad_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_grad_impl.hpp 3 | * @author Abhinav Moudgil 4 | * 5 | * Implementation of AdaGrad optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADA_GRAD_ADA_GRAD_IMPL_HPP 13 | #define ENSMALLEN_ADA_GRAD_ADA_GRAD_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "ada_grad.hpp" 17 | 18 | namespace ens { 19 | 20 | inline AdaGrad::AdaGrad(const double stepSize, 21 | const size_t batchSize, 22 | const double epsilon, 23 | const size_t maxIterations, 24 | const double tolerance, 25 | const bool shuffle, 26 | const bool resetPolicy, 27 | const bool exactObjective) : 28 | optimizer(stepSize, 29 | batchSize, 30 | maxIterations, 31 | tolerance, 32 | shuffle, 33 | AdaGradUpdate(epsilon), 34 | NoDecay(), 35 | resetPolicy, 36 | exactObjective) 37 | { /* Nothing to do. */ } 38 | 39 | } // namespace ens 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /tests/sarah_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sarah_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | #include "test_types.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | TEMPLATE_TEST_CASE("SARAH_LogisticRegressionFunction", "[SARAH]", 24 | ENS_ALL_TEST_TYPES) 25 | { 26 | // Run SARAH with a couple of batch sizes. 27 | for (size_t batchSize = 35; batchSize < 45; batchSize += 5) 28 | { 29 | SARAH optimizer(0.01, batchSize, 250, 0, 1e-5, true); 30 | LogisticRegressionFunctionTest(optimizer); 31 | } 32 | } 33 | 34 | TEMPLATE_TEST_CASE("SARAH_Plus_LogisticRegressionFunction", "[SARAH]", 35 | ENS_ALL_TEST_TYPES) 36 | { 37 | // Run SARAH_Plus with a couple of batch sizes. 38 | for (size_t batchSize = 35; batchSize < 45; batchSize += 5) 39 | { 40 | SARAH_Plus optimizer(0.01, batchSize, 250, 0, 1e-5, true); 41 | LogisticRegressionFunctionTest(optimizer); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /CMake/ARMA_FindCBLAS.cmake: -------------------------------------------------------------------------------- 1 | # - Find CBLAS (includes and library) 2 | # This module defines 3 | # CBLAS_INCLUDE_DIR 4 | # CBLAS_LIBRARIES 5 | # CBLAS_FOUND 6 | # also defined, but not for general use are 7 | # CBLAS_LIBRARY, where to find the library. 8 | 9 | find_path(CBLAS_INCLUDE_DIR cblas.h 10 | /usr/include/atlas/ 11 | /usr/local/include/atlas/ 12 | /usr/include/ 13 | /usr/local/include/ 14 | ) 15 | 16 | set(CBLAS_NAMES ${CBLAS_NAMES} cblas) 17 | find_library(CBLAS_LIBRARY 18 | NAMES ${CBLAS_NAMES} 19 | PATHS /usr/lib64/atlas-sse3 /usr/lib64/atlas /usr/lib64 /usr/local/lib64/atlas /usr/local/lib64 /usr/lib/atlas-sse3 /usr/lib/atlas-sse2 /usr/lib/atlas-sse /usr/lib/atlas-3dnow /usr/lib/atlas /usr/lib /usr/local/lib/atlas /usr/local/lib 20 | ) 21 | 22 | if (CBLAS_LIBRARY AND CBLAS_INCLUDE_DIR) 23 | set(CBLAS_LIBRARIES ${CBLAS_LIBRARY}) 24 | set(CBLAS_FOUND "YES") 25 | else () 26 | set(CBLAS_FOUND "NO") 27 | endif () 28 | 29 | 30 | if (CBLAS_FOUND) 31 | if (NOT CBLAS_FIND_QUIETLY) 32 | message(STATUS "Found a CBLAS library: ${CBLAS_LIBRARIES}") 33 | endif () 34 | else () 35 | if (CBLAS_FIND_REQUIRED) 36 | message(FATAL_ERROR "Could not find a CBLAS library") 37 | endif () 38 | endif () 39 | 40 | # Deprecated declarations. 41 | set (NATIVE_CBLAS_INCLUDE_PATH ${CBLAS_INCLUDE_DIR} ) 42 | get_filename_component (NATIVE_CBLAS_LIB_PATH ${CBLAS_LIBRARY} PATH) 43 | 44 | mark_as_advanced( 45 | CBLAS_LIBRARY 46 | CBLAS_INCLUDE_DIR 47 | ) 48 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ftml/ftml_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ftml_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Follow the Moving Leader (FTML) optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_FTML_FTML_IMPL_HPP 13 | #define ENSMALLEN_FTML_FTML_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "ftml.hpp" 17 | 18 | namespace ens { 19 | 20 | inline FTML::FTML(const double stepSize, 21 | const size_t batchSize, 22 | const double beta1, 23 | const double beta2, 24 | const double epsilon, 25 | const size_t maxIterations, 26 | const double tolerance, 27 | const bool shuffle, 28 | const bool resetPolicy, 29 | const bool exactObjective) : 30 | optimizer(stepSize, 31 | batchSize, 32 | maxIterations, 33 | tolerance, 34 | shuffle, 35 | FTMLUpdate(epsilon, beta1, beta2), 36 | NoDecay(), 37 | resetPolicy, 38 | exactObjective) 39 | { /* Nothing to do. */ } 40 | 41 | } // namespace ens 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /tests/bigbatch_sgd_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file bigbatch_sgd_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | #include "test_types.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | TEMPLATE_TEST_CASE("BBS_BB_LogisticRegressionFunction", "[BigBatchSGD]", 24 | ENS_ALL_TEST_TYPES) 25 | { 26 | // Run big-batch SGD with a couple of batch sizes. 27 | for (size_t batchSize = 350; batchSize < 360; batchSize += 5) 28 | { 29 | BBS_BB bbsgd(batchSize, 0.001, 0.1, 10000, 1e-8, true, true); 30 | LogisticRegressionFunctionTest(bbsgd); 31 | } 32 | } 33 | 34 | TEMPLATE_TEST_CASE("BBS_Armijo_LogisticRegressionFunction", "[BigBatchSGD]", 35 | ENS_ALL_CPU_TEST_TYPES) 36 | { 37 | // Run big-batch SGD with a couple of batch sizes. 38 | for (size_t batchSize = 40; batchSize < 50; batchSize += 1) 39 | { 40 | BBS_Armijo bbsgd(batchSize, 0.005, 0.1, 10000, 1e-6, true, true); 41 | LogisticRegressionFunctionTest(bbsgd); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /include/ensmallen_bits/adam/adam_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file adam_impl.hpp 3 | * @author Ryan Curtin 4 | * @author Vasanth Kalingeri 5 | * @author Marcus Edel 6 | * @author Vivek Pal 7 | * 8 | * Implementation of the Adam, AdaMax, AMSGrad, Nadam and NadaMax optimizer. 9 | * 10 | * ensmallen is free software; you may redistribute it and/or modify it under 11 | * the terms of the 3-clause BSD license. You should have received a copy of 12 | * the 3-clause BSD license along with ensmallen. If not, see 13 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 14 | */ 15 | #ifndef ENSMALLEN_ADAM_ADAM_IMPL_HPP 16 | #define ENSMALLEN_ADAM_ADAM_IMPL_HPP 17 | 18 | // In case it hasn't been included yet. 19 | #include "adam.hpp" 20 | 21 | namespace ens { 22 | 23 | template 24 | AdamType::AdamType( 25 | const double stepSize, 26 | const size_t batchSize, 27 | const double beta1, 28 | const double beta2, 29 | const double epsilon, 30 | const size_t maxIterations, 31 | const double tolerance, 32 | const bool shuffle, 33 | const bool resetPolicy, 34 | const bool exactObjective) : 35 | optimizer(stepSize, 36 | batchSize, 37 | maxIterations, 38 | tolerance, 39 | shuffle, 40 | UpdateRule(epsilon, beta1, beta2), 41 | NoDecay(), 42 | resetPolicy, 43 | exactObjective) 44 | { /* Nothing to do. */ } 45 | 46 | } // namespace ens 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_bound/ada_bound_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file qhadam_impl.hpp 3 | * @author Niteya Shah 4 | * 5 | * Implementation of QHAdam class wrapper. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_ADA_BOUND_IMPL_HPP 13 | #define ENSMALLEN_ADA_BOUND_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "ada_bound.hpp" 17 | 18 | namespace ens { 19 | 20 | template 21 | AdaBoundType::AdaBoundType( 22 | const double stepSize, 23 | const size_t batchSize, 24 | const double finalLr, 25 | const double gamma, 26 | const double beta1, 27 | const double beta2, 28 | const double epsilon, 29 | const size_t maxIterations, 30 | const double tolerance, 31 | const bool shuffle, 32 | const bool resetPolicy, 33 | const bool exactObjective) : 34 | optimizer(stepSize, 35 | batchSize, 36 | maxIterations, 37 | tolerance, 38 | shuffle, 39 | UpdatePolicyType(finalLr, gamma, epsilon, beta1, beta2), 40 | NoDecay(), 41 | resetPolicy, 42 | exactObjective) 43 | { /* Nothing to do. */ } 44 | 45 | } // namespace ens 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /CMake/ARMA_FindCLAPACK.cmake: -------------------------------------------------------------------------------- 1 | # - Find a version of CLAPACK (includes and library) 2 | # This module defines 3 | # CLAPACK_INCLUDE_DIR 4 | # CLAPACK_LIBRARIES 5 | # CLAPACK_FOUND 6 | # also defined, but not for general use are 7 | # CLAPACK_LIBRARY, where to find the library. 8 | 9 | find_path(CLAPACK_INCLUDE_DIR clapack.h 10 | /usr/include/atlas/ 11 | /usr/local/include/atlas/ 12 | /usr/include/ 13 | /usr/local/include/ 14 | ) 15 | 16 | set(CLAPACK_NAMES ${CLAPACK_NAMES} lapack_atlas) 17 | set(CLAPACK_NAMES ${CLAPACK_NAMES} clapack) 18 | find_library(CLAPACK_LIBRARY 19 | NAMES ${CLAPACK_NAMES} 20 | PATHS /usr/lib64/atlas-sse3 /usr/lib64/atlas /usr/lib64 /usr/local/lib64/atlas /usr/local/lib64 /usr/lib/atlas-sse3 /usr/lib/atlas-sse2 /usr/lib/atlas-sse /usr/lib/atlas-3dnow /usr/lib/atlas /usr/lib /usr/local/lib/atlas /usr/local/lib 21 | ) 22 | 23 | if (CLAPACK_LIBRARY AND CLAPACK_INCLUDE_DIR) 24 | set(CLAPACK_LIBRARIES ${CLAPACK_LIBRARY}) 25 | set(CLAPACK_FOUND "YES") 26 | else () 27 | set(CLAPACK_FOUND "NO") 28 | endif () 29 | 30 | 31 | if (CLAPACK_FOUND) 32 | if (NOT CLAPACK_FIND_QUIETLY) 33 | message(STATUS "Found a CLAPACK library: ${CLAPACK_LIBRARIES}") 34 | endif () 35 | else () 36 | if (CLAPACK_FIND_REQUIRED) 37 | message(FATAL_ERROR "Could not find a CLAPACK library") 38 | endif () 39 | endif () 40 | 41 | # Deprecated declarations. 42 | set (NATIVE_CLAPACK_INCLUDE_PATH ${CLAPACK_INCLUDE_DIR} ) 43 | get_filename_component (NATIVE_CLAPACK_LIB_PATH ${CLAPACK_LIBRARY} PATH) 44 | 45 | mark_as_advanced( 46 | CLAPACK_LIBRARY 47 | CLAPACK_INCLUDE_DIR 48 | ) 49 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ada_delta/ada_delta_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_delta_impl.hpp 3 | * @author Ryan Curtin 4 | * @author Vasanth Kalingeri 5 | * @author Abhinav Moudgil 6 | * 7 | * Implementation of the AdaDelta optimizer. 8 | * 9 | * ensmallen is free software; you may redistribute it and/or modify it under 10 | * the terms of the 3-clause BSD license. You should have received a copy of 11 | * the 3-clause BSD license along with ensmallen. If not, see 12 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 13 | */ 14 | #ifndef ENSMALLEN_ADA_DELTA_ADA_DELTA_IMPL_HPP 15 | #define ENSMALLEN_ADA_DELTA_ADA_DELTA_IMPL_HPP 16 | 17 | // In case it hasn't been included yet. 18 | #include "ada_delta.hpp" 19 | 20 | namespace ens { 21 | 22 | inline AdaDelta::AdaDelta(const double stepSize, 23 | const size_t batchSize, 24 | const double rho, 25 | const double epsilon, 26 | const size_t maxIterations, 27 | const double tolerance, 28 | const bool shuffle, 29 | const bool resetPolicy, 30 | const bool exactObjective) : 31 | optimizer(stepSize, 32 | batchSize, 33 | maxIterations, 34 | tolerance, 35 | shuffle, 36 | AdaDeltaUpdate(rho, epsilon), 37 | NoDecay(), 38 | resetPolicy, 39 | exactObjective) 40 | { /* Nothing to do. */ } 41 | 42 | } // namespace ens 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file empty_transformation.hpp 3 | * @author Suvarsha Chennareddy 4 | * 5 | * Empty Transformation, can also be called an Indentity Transformation. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP 13 | #define ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * This is an empty transformation. As the name indicates, it does 19 | * not do anything. It is essentially an identity 20 | * transformation and is meant to be used when there are no 21 | * sorts of constraints on the coordinates. 22 | * 23 | * @tparam MatType The matrix type of the coordinates. 24 | */ 25 | template 26 | class EmptyTransformation 27 | { 28 | public: 29 | /** 30 | * Transforms coordinates to themselves (effectively no transformation). 31 | * 32 | * @param x Input coordinates. 33 | * @return Transformed coordinates (the coordinates themselves). 34 | */ 35 | MatType Transform(const MatType& x) { return x; } 36 | 37 | /** 38 | * Return a suitable initial step size. 39 | * 40 | * @return initial step size. 41 | */ 42 | typename MatType::elem_type InitialStepSize() { return 1; } 43 | }; 44 | 45 | } // namespace ens 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /include/ensmallen_bits/config.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file config.hpp 3 | * @author Conrad Sanderson 4 | * @author Marcus Edel 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | 12 | 13 | #if !defined(ENS_PRINT_INFO) 14 | // #define ENS_PRINT_INFO 15 | #endif 16 | 17 | #if !defined(ENS_PRINT_WARN) 18 | // #define ENS_PRINT_WARN 19 | #endif 20 | 21 | #if defined(ARMA_USE_OPENMP) 22 | #undef ENS_USE_OPENMP 23 | #define ENS_USE_OPENMP 24 | #endif 25 | 26 | 27 | // 28 | 29 | 30 | #if defined(ENS_DONT_PRINT_INFO) 31 | #undef ENS_PRINT_INFO 32 | #endif 33 | 34 | #if defined(ENS_DONT_PRINT_WARN) 35 | #undef ENS_PRINT_WARN 36 | #endif 37 | 38 | #if defined(ENS_DONT_USE_OPENMP) 39 | #undef ENS_USE_OPENMP 40 | #endif 41 | 42 | 43 | // 44 | 45 | 46 | #if defined(ENS_USE_OPENMP) 47 | #define ENS_PRAGMA_OMP_PARALLEL _Pragma("omp parallel") 48 | #define ENS_PRAGMA_OMP_ATOMIC _Pragma("omp atomic") 49 | #define ENS_PRAGMA_OMP_CRITICAL _Pragma("omp critical") 50 | #define ENS_PRAGMA_OMP_CRITICAL_NAMED _Pragma("omp critical(section)") 51 | #else 52 | #define ENS_PRAGMA_OMP_PARALLEL 53 | #define ENS_PRAGMA_OMP_ATOMIC 54 | #define ENS_PRAGMA_OMP_CRITICAL 55 | #define ENS_PRAGMA_OMP_CRITICAL_NAMED 56 | #endif 57 | 58 | 59 | // undefine conflicting macros 60 | #if defined(As) 61 | #pragma message ("WARNING: undefined conflicting 'As' macro") 62 | #undef As 63 | #endif 64 | -------------------------------------------------------------------------------- /tests/delta_bar_delta_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file delta_bar_delta_test.cpp 3 | * @author Ranjodh Singh 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("DeltaBarDelta_GDTestFunction", "[DeltaBarDelta]", 23 | ENS_ALL_TEST_TYPES) 24 | { 25 | DeltaBarDelta s(0.9, 50, 1e-9, 0.001, 0.2, 0.5); 26 | FunctionTest(s, 27 | Tolerances::LargeObj, 28 | Tolerances::LargeCoord); 29 | } 30 | 31 | TEMPLATE_TEST_CASE("DeltaBarDelta_RosenbrockFunction", "[DeltaBarDelta]", 32 | ENS_ALL_CPU_TEST_TYPES) 33 | { 34 | DeltaBarDelta s(0.001, 100000, Tolerances::Obj / 100, 35 | 0.0001, 0.2, 0.5); 36 | FunctionTest(s, 37 | 10 * Tolerances::LargeObj, 38 | 10 * Tolerances::LargeCoord); 39 | } 40 | 41 | TEMPLATE_TEST_CASE("DeltaBarDelta_LogisticRegressionFunction", 42 | "[DeltaBarDelta]", ENS_ALL_TEST_TYPES) 43 | { 44 | DeltaBarDelta s(0.0001, 500, Tolerances::Obj, 45 | 0.00008, 0.2, 0.5); 46 | LogisticRegressionFunctionTest(s); 47 | } 48 | -------------------------------------------------------------------------------- /include/ensmallen_bits/parallel_sgd/decay_policies/constant_step.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file constant_step.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Constant step size policy for parallel Stochastic Gradient Descent. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PARALLEL_SGD_CONSTANT_STEP_HPP 13 | #define ENSMALLEN_PARALLEL_SGD_CONSTANT_STEP_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Implementation of the ConstantStep stepsize decay policy for parallel SGD. 19 | */ 20 | class ConstantStep 21 | { 22 | public: 23 | /** 24 | * Member initialization constructor. 25 | * 26 | * The defaults here are not necessarily good for the given problem, so it is 27 | * suggested that the values used be tailored to the task at hand. 28 | * 29 | * @param step The intial stepsize to use. 30 | */ 31 | ConstantStep(const double step = 0.01) : step(step) { /* Nothing to do */ } 32 | 33 | /** 34 | * This function is called in each iteration before the gradient update. 35 | * 36 | * @param numEpoch The iteration number for which the stepsize is to be 37 | * calculated. 38 | * @return The step size for the current iteration. 39 | */ 40 | double StepSize(const size_t /* numEpoch */) 41 | { 42 | return step; 43 | } 44 | 45 | private: 46 | //! The initial stepsize, which remains unchanged. 47 | double step; 48 | }; 49 | 50 | } // namespace ens 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /CMake/ARMA_FindMKL.cmake: -------------------------------------------------------------------------------- 1 | # - Find the MKL libraries (no includes) 2 | # This module defines 3 | # MKL_LIBRARIES, the libraries needed to use Intel's implementation of BLAS & LAPACK. 4 | # MKL_FOUND, If false, do not try to use MKL. 5 | 6 | set(MKL_NAMES ${MKL_NAMES} mkl_lapack) 7 | set(MKL_NAMES ${MKL_NAMES} mkl_intel_thread) 8 | set(MKL_NAMES ${MKL_NAMES} mkl_core) 9 | set(MKL_NAMES ${MKL_NAMES} guide) 10 | set(MKL_NAMES ${MKL_NAMES} mkl) 11 | set(MKL_NAMES ${MKL_NAMES} iomp5) 12 | #set(MKL_NAMES ${MKL_NAMES} pthread) 13 | 14 | if(CMAKE_SIZEOF_VOID_P EQUAL 8) 15 | set(MKL_NAMES ${MKL_NAMES} mkl_intel_lp64) 16 | else() 17 | set(MKL_NAMES ${MKL_NAMES} mkl_intel) 18 | endif() 19 | 20 | foreach (MKL_NAME ${MKL_NAMES}) 21 | find_library(${MKL_NAME}_LIBRARY 22 | NAMES ${MKL_NAME} 23 | PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib /opt/intel/lib/intel64 /opt/intel/lib/ia32 /opt/intel/mkl/lib/lib64 /opt/intel/mkl/lib/intel64 /opt/intel/mkl/lib/ia32 /opt/intel/mkl/lib /opt/intel/*/mkl/lib/intel64 /opt/intel/*/mkl/lib/ia32/ /opt/mkl/*/lib/em64t /opt/mkl/*/lib/32 /opt/intel/mkl/*/lib/em64t /opt/intel/mkl/*/lib/32 24 | ) 25 | 26 | set(TMP_LIBRARY ${${MKL_NAME}_LIBRARY}) 27 | 28 | if(TMP_LIBRARY) 29 | set(MKL_LIBRARIES ${MKL_LIBRARIES} ${TMP_LIBRARY}) 30 | endif() 31 | endforeach() 32 | 33 | if (MKL_LIBRARIES) 34 | set(MKL_FOUND "YES") 35 | else () 36 | set(MKL_FOUND "NO") 37 | endif () 38 | 39 | if (MKL_FOUND) 40 | if (NOT MKL_FIND_QUIETLY) 41 | message(STATUS "Found MKL libraries: ${MKL_LIBRARIES}") 42 | endif () 43 | else () 44 | if (MKL_FIND_REQUIRED) 45 | message(FATAL_ERROR "Could not find MKL libraries") 46 | endif () 47 | endif () 48 | 49 | # mark_as_advanced(MKL_LIBRARY) 50 | -------------------------------------------------------------------------------- /tests/katyusha_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file katyusha_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("Katyusha_LogisticRegressionFunction", "[Katyusha]", 23 | ENS_ALL_TEST_TYPES) 24 | { 25 | // Run with a couple of batch sizes. 26 | for (size_t batchSize = 30; batchSize < 45; batchSize += 5) 27 | { 28 | Katyusha optimizer(1.0, 10.0, batchSize, 200, 0, 29 | Tolerances::Obj, true); 30 | LogisticRegressionFunctionTest( 31 | optimizer, 32 | Tolerances::LRTrainAcc, 33 | Tolerances::LRTestAcc); 34 | } 35 | } 36 | 37 | TEMPLATE_TEST_CASE("KatyushaProximal_LogisticRegressionFunction", "[Katyusha]", 38 | ENS_FULLPREC_TEST_TYPES) 39 | { 40 | // Run with a couple of batch sizes. 41 | for (size_t batchSize = 30; batchSize < 45; batchSize += 5) 42 | { 43 | KatyushaProximal optimizer(1.0, 10.0, batchSize, 200, 0, 44 | Tolerances::Obj, true); 45 | LogisticRegressionFunctionTest( 46 | optimizer, 47 | Tolerances::LRTrainAcc, 48 | Tolerances::LRTestAcc); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tests/momentum_delta_bar_delta_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file momentum_delta_bar_delta_test.cpp 3 | * @author Ranjodh Singh 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("MomentumDeltaBarDelta_GDTestFunction", 23 | "[MomentumDeltaBarDelta]", ENS_ALL_TEST_TYPES) 24 | { 25 | MomentumDeltaBarDelta s(0.1, 1000, 1e-9, 0.2, 0.8, 0.5); 26 | FunctionTest(s, 27 | Tolerances::LargeObj, 28 | Tolerances::LargeCoord); 29 | } 30 | 31 | TEMPLATE_TEST_CASE("MomentumDeltaBarDelta_RosenbrockFunction", 32 | "[MomentumDeltaBarDelta]", ENS_ALL_CPU_TEST_TYPES) 33 | { 34 | MomentumDeltaBarDelta s(0.001, 100000, Tolerances::Obj / 100, 0.2, 35 | 0.8, 0.5); 36 | FunctionTest(s, 37 | 10 * Tolerances::LargeObj, 38 | 10 * Tolerances::LargeCoord); 39 | } 40 | 41 | TEMPLATE_TEST_CASE("MomentumDeltaBarDelta_LogisticRegressionFunction", 42 | "[MomentumDeltaBarDelta]", ENS_ALL_TEST_TYPES) 43 | { 44 | MomentumDeltaBarDelta s(0.00005, 2000, Tolerances::Obj, 45 | 0.2, 0.8, 0.5); 46 | LogisticRegressionFunctionTest(s); 47 | } 48 | -------------------------------------------------------------------------------- /tests/wn_grad_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file wn_grad_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * Test file for the WNGrad optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | #include "catch.hpp" 18 | #include "test_function_tools.hpp" 19 | #include "test_types.hpp" 20 | 21 | using namespace ens; 22 | using namespace ens::test; 23 | 24 | TEMPLATE_TEST_CASE("WNGrad_LogisticRegressionFunction", "[WNGrad]", 25 | ENS_ALL_CPU_TEST_TYPES) 26 | { 27 | WNGrad optimizer(0.56, 1, 500000, 1e-9, true); 28 | LogisticRegressionFunctionTest>(optimizer); 29 | } 30 | 31 | TEMPLATE_TEST_CASE("WNGrad_SphereFunction", "[WNGrad]", ENS_ALL_TEST_TYPES) 32 | { 33 | WNGrad optimizer(1.12, 2, 500000, 1e-9, true); 34 | FunctionTest( 35 | optimizer, 36 | Tolerances::LargeObj, 37 | Tolerances::LargeCoord); 38 | } 39 | 40 | // The Styblinski-Tang function is too difficult to make converge for WNGrad in 41 | // low precision. 42 | TEMPLATE_TEST_CASE("WNGrad_StyblinskiTangFunction", "[WNGrad]", 43 | ENS_FULLPREC_TEST_TYPES, ENS_SPARSE_TEST_TYPES) 44 | { 45 | WNGrad optimizer(1.12, 2, 500000, 1e-9, true); 46 | FunctionTest( 47 | optimizer, 48 | 5 * Tolerances::LargeObj, 49 | 5 * Tolerances::LargeCoord); 50 | } 51 | -------------------------------------------------------------------------------- /tests/svrg_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file svrg_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | #include "test_types.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | TEMPLATE_TEST_CASE("SVRG_LogisticRegressionFunction", "[SVRG]", 24 | ENS_ALL_TEST_TYPES, ENS_SPARSE_TEST_TYPES) 25 | { 26 | // Run SVRG with a couple of batch sizes. 27 | for (size_t batchSize = 35; batchSize < 50; batchSize += 5) 28 | { 29 | SVRG optimizer(0.005, batchSize, 300, 0, 1e-5, true); 30 | LogisticRegressionFunctionTest( 31 | optimizer, 32 | 5 * Tolerances::LRTrainAcc, 33 | 5 * Tolerances::LRTestAcc); 34 | } 35 | } 36 | 37 | TEMPLATE_TEST_CASE("SVRG_BB_LogisticRegressionFunction", "[SVRG_BB]", 38 | ENS_ALL_TEST_TYPES, ENS_SPARSE_TEST_TYPES) 39 | { 40 | // Run SVRG with a couple of batch sizes. 41 | for (size_t batchSize = 35; batchSize < 50; batchSize += 5) 42 | { 43 | SVRG_BB optimizer(0.005, batchSize, 300, 0, 1e-5, true, SVRGUpdate(), 44 | BarzilaiBorweinDecay(0.1)); 45 | LogisticRegressionFunctionTest( 46 | optimizer, 47 | 5 * Tolerances::LRTrainAcc, 48 | 5 * Tolerances::LRTestAcc); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /example.cpp: -------------------------------------------------------------------------------- 1 | // Example implementation of an objective function class for linear regression 2 | // and usage of the L-BFGS optimizer. 3 | // 4 | // Compilation: 5 | // g++ example.cpp -o example -O3 -larmadillo 6 | 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | class LinearRegressionFunction 14 | { 15 | public: 16 | 17 | LinearRegressionFunction(arma::mat& X, arma::vec& y) : X(X), y(y) { } 18 | 19 | double EvaluateWithGradient(const arma::mat& theta, arma::mat& gradient) 20 | { 21 | const arma::vec tmp = X.t() * theta - y; 22 | gradient = 2 * X * tmp; 23 | return arma::dot(tmp,tmp); 24 | } 25 | 26 | private: 27 | 28 | const arma::mat& X; 29 | const arma::vec& y; 30 | }; 31 | 32 | 33 | int main(int argc, char** argv) 34 | { 35 | if (argc < 3) 36 | { 37 | std::cout << "usage: " << argv[0] << " n_dims n_points" << std::endl; 38 | return -1; 39 | } 40 | 41 | int n_dims = atoi(argv[1]); 42 | int n_points = atoi(argv[2]); 43 | 44 | // generate noisy dataset with a slight linear pattern 45 | arma::mat X(n_dims, n_points, arma::fill::randu); 46 | arma::vec y( n_points, arma::fill::randu); 47 | 48 | for (size_t i = 0; i < n_points; ++i) 49 | { 50 | double a = arma::randu(); 51 | X(1, i) += a; 52 | y(i) += a; 53 | } 54 | 55 | LinearRegressionFunction lrf(X, y); 56 | 57 | // create a Limited-memory BFGS optimizer object with default parameters 58 | ens::L_BFGS opt; 59 | opt.MaxIterations() = 10; 60 | 61 | // initial point (uniform random) 62 | arma::vec theta(n_dims, arma::fill::randu); 63 | 64 | opt.Optimize(lrf, theta); 65 | 66 | // theta now contains the optimized parameters 67 | theta.print("theta:"); 68 | 69 | return 0; 70 | } 71 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/fw_test_function.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file fw_test_function.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * Simple test function for classic Frank Wolfe Algorithm: 6 | * 7 | * \f$ f(x) = (x1 - 0.1)^2 + (x2 - 0.2)^2 + (x3 - 0.3)^2 \f$ 8 | * 9 | * ensmallen is free software; you may redistribute it and/or modify it under 10 | * the terms of the 3-clause BSD license. You should have received a copy of 11 | * the 3-clause BSD license along with ensmallen. If not, see 12 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 13 | */ 14 | #ifndef ENSMALLEN_PROBLEMS_FW_TEST_FUNCTION_HPP 15 | #define ENSMALLEN_PROBLEMS_FW_TEST_FUNCTION_HPP 16 | 17 | namespace ens { 18 | 19 | /** 20 | * Simple test function for classic Frank Wolfe Algorithm: 21 | * 22 | * \f$ f(x) = (x1 - 0.1)^2 + (x2 - 0.2)^2 + (x3 - 0.3)^2 \f$. 23 | */ 24 | template 25 | class TestFuncFW 26 | { 27 | public: 28 | TestFuncFW() {/* Nothing to do. */} 29 | 30 | /** 31 | * Evaluation of the function. 32 | * 33 | * @param coords input vector x. 34 | */ 35 | typename MatType::elem_type Evaluate(const MatType& coords) 36 | { 37 | typename MatType::elem_type f = std::pow(coords[0] - 0.1, 2); 38 | f += std::pow(coords[1] - 0.2, 2); 39 | f += std::pow(coords[2] - 0.3, 2); 40 | return f; 41 | } 42 | 43 | /** 44 | * Gradient of the function. 45 | * 46 | * @param coords input vector x. 47 | * @param gradient output gradient vector. 48 | */ 49 | void Gradient(const MatType& coords, GradType& gradient) 50 | { 51 | gradient.set_size(3, 1); 52 | gradient[0] = coords[0] - 0.1; 53 | gradient[1] = coords[1] - 0.2; 54 | gradient[2] = coords[2] - 0.3; 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /tests/main.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file main.cpp 3 | * @author Conrad Sanderson 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | 18 | //#define CATCH_CONFIG_MAIN // catch.hpp will define main() 19 | #define CATCH_CONFIG_RUNNER // we will define main() 20 | #include "catch.hpp" 21 | 22 | int main(int argc, char** argv) 23 | { 24 | #ifdef ENS_HAVE_COOT 25 | coot::get_rt().init(true); 26 | #endif 27 | 28 | Catch::Session session; 29 | const int returnCode = session.applyCommandLine(argc, argv); 30 | // Check for a command line error. 31 | if (returnCode != 0) 32 | return returnCode; 33 | 34 | std::cout << "ensmallen version: " << ens::version::as_string() << std::endl; 35 | std::cout << "armadillo version: " << arma::arma_version::as_string() << std::endl; 36 | 37 | #ifdef ENS_HAVE_COOT 38 | std::cout << "bandicoot version: " << coot::coot_version::as_string() << std::endl; 39 | #endif 40 | 41 | // Use Catch2 command-line to set the random seed. 42 | // -rng-seed <'time'|number> 43 | // If a number is provided this is used directly as the seed. Alternatively 44 | // if the keyword 'time' is provided then the result of calling std::time(0) 45 | // is used. 46 | const size_t seed = session.config().rngSeed(); 47 | std::cout << "random seed: " << seed << std::endl; 48 | srand((unsigned int) seed); 49 | arma::arma_rng::set_seed(seed); 50 | 51 | return session.run(); 52 | } 53 | -------------------------------------------------------------------------------- /.github/workflows/bandicoot-test.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | build: 6 | name: ${{ matrix.config.name }} 7 | runs-on: ${{ matrix.config.os }} 8 | outputs: 9 | tag: ${{ steps.git.outputs.tag }} 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | config: 14 | - { name: 'CUDA', os: self-hosted} 15 | - { name: 'OpenCL', os: self-hosted} 16 | - { name: 'CPU', os: self-hosted} 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Install Dependencies 20 | run: | 21 | cd ../ 22 | rm -rf bandicoot-code 23 | git clone --depth 1 --branch 2.1.1 https://gitlab.com/bandicoot-lib/bandicoot-code.git 24 | cd bandicoot-code/ 25 | mkdir build/ 26 | cd build/ 27 | 28 | if [[ "${{ matrix.config.name }}" == "CUDA" ]]; then 29 | echo "Installing Bandicoot CUDA" 30 | cmake -DFIND_CUDA=ON -DFIND_OPENCL=OFF -DBUILD_TESTS=OFF ../ 31 | make 32 | elif [[ "${{ matrix.config.name }}" == "OpenCL" ]]; then 33 | echo "Installing Bandicoot OpenCL" 34 | cmake -DFIND_CUDA=OFF -DFIND_OPENCL=ON -DBUILD_TESTS=OFF ../ 35 | make 36 | fi 37 | 38 | - name: Build ensmallen 39 | run: | 40 | mkdir build 41 | cd build/ 42 | 43 | if [[ "${{ matrix.config.name }}" == "CPU" ]]; then 44 | cmake -DUSE_BANDICOOT=OFF .. 45 | else 46 | cmake -DBANDICOOT_INCLUDE_DIR=../../bandicoot-code/build/tmp/include/ -DBANDICOOT_LIBRARY=../../bandicoot-code/build/libbandicoot.so .. 47 | fi 48 | 49 | make ensmallen_tests 50 | 51 | - name: Test ensmallen 52 | run: | 53 | cd build/ 54 | ./ensmallen_tests -d yes 55 | -------------------------------------------------------------------------------- /tests/swats_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file swats_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * Test file for the SWATS optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | #include "catch.hpp" 18 | #include "test_function_tools.hpp" 19 | #include "test_types.hpp" 20 | 21 | using namespace ens; 22 | using namespace ens::test; 23 | 24 | TEMPLATE_TEST_CASE("SWATS_LogisticRegressionFunction", "[SWATS]", 25 | ENS_ALL_TEST_TYPES) 26 | { 27 | SWATS optimizer(0.03, 10, 0.9, 0.999, 1e-6, 600000, 1e-9, true); 28 | // We allow a few trials in case of poor convergence. 29 | LogisticRegressionFunctionTest>( 30 | optimizer, 31 | Tolerances::LRTrainAcc, 32 | Tolerances::LRTestAcc, 33 | 5); 34 | } 35 | 36 | TEMPLATE_TEST_CASE("SWATS_SphereFunction", "[SWATS]", ENS_ALL_TEST_TYPES) 37 | { 38 | SWATS optimizer(0.2, 2, 0.9, 0.999, 1e-6, 500000, 1e-9, true); 39 | FunctionTest( 40 | optimizer, 41 | Tolerances::LargeObj, 42 | Tolerances::LargeCoord); 43 | } 44 | 45 | TEMPLATE_TEST_CASE("SWATS_StyblinskiTangFunction", "[SWATS]", 46 | ENS_ALL_TEST_TYPES, ENS_SPARSE_TEST_TYPES) 47 | { 48 | SWATS optimizer(0.4, 2, 0.9, 0.999, 1e-6, 500000, 1e-9, true); 49 | FunctionTest( 50 | optimizer, 51 | 30 * Tolerances::LargeObj, 52 | 30 * Tolerances::LargeCoord); 53 | } 54 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sarah/sarah_update.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file svrg_update.hpp 3 | * @author Marcus Edel 4 | * 5 | * Vanilla update for SARAH. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SARAH_SARAH_UPDATE_HPP 13 | #define ENSMALLEN_SARAH_SARAH_UPDATE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Vanilla update policy for SARAH. 19 | */ 20 | class SARAHUpdate 21 | { 22 | public: 23 | /** 24 | * Update step for SARAH. The function parameters are updated in the negative 25 | * direction of the gradient. 26 | * 27 | * @param iterate Parameters that minimize the function. 28 | * @param v Unbiased estimator of the gradient. 29 | * @param gradient The current gradient matrix at time t. 30 | * @param gradient0 The old gradient matrix at time t - 1. 31 | * @param batchSize Batch size to be used for the given iteration. 32 | * @param stepSize Step size to be used for the given iteration. 33 | * @param vNorm The norm of the full gradient. 34 | */ 35 | template 36 | bool Update(MatType& iterate, 37 | GradType& v, 38 | const GradType& gradient, 39 | const GradType& gradient0, 40 | const size_t batchSize, 41 | const double stepSize, 42 | const double /* vNorm */) 43 | { 44 | typedef typename MatType::elem_type ElemType; 45 | 46 | v += (gradient - gradient0) / (ElemType) batchSize; 47 | iterate -= ElemType(stepSize) * v; 48 | return false; 49 | } 50 | }; 51 | 52 | } // namespace ens 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /tests/eve_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file eve_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * Test file for the Eve optimizer. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | #include "catch.hpp" 18 | #include "test_function_tools.hpp" 19 | #include "test_types.hpp" 20 | 21 | using namespace ens; 22 | using namespace ens::test; 23 | 24 | TEMPLATE_TEST_CASE("Eve_LogisticRegressionFunction", "[Eve]", 25 | ENS_ALL_CPU_TEST_TYPES) 26 | { 27 | Eve optimizer(0.01, 1, 0.9, 0.999, 0.999, Tolerances::Obj, 10000, 28 | 500000, Tolerances::Obj / 10, true); 29 | LogisticRegressionFunctionTest>(optimizer); 30 | } 31 | 32 | TEMPLATE_TEST_CASE("Eve_SphereFunction", "[Eve]", ENS_ALL_TEST_TYPES) 33 | { 34 | Eve optimizer(0.02, 2, 0.9, 0.999, 0.999, Tolerances::Obj, 10000, 35 | 500000, Tolerances::Obj / 10, true); 36 | FunctionTest( 37 | optimizer, 38 | Tolerances::LargeObj, 39 | Tolerances::LargeCoord); 40 | } 41 | 42 | TEMPLATE_TEST_CASE("Eve_StyblinskiTangFunction", "[Eve]", ENS_ALL_TEST_TYPES, 43 | ENS_SPARSE_TEST_TYPES) 44 | { 45 | Eve optimizer(0.16, 2, 0.9, 0.999, 0.999, Tolerances::Obj, 10000, 46 | 500000, Tolerances::Obj / 10, true); 47 | FunctionTest( 48 | optimizer, 49 | Tolerances::LargeObj, 50 | Tolerances::LargeCoord); 51 | } 52 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/holder_table_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file holder_table_function_impl.hpp 3 | * @author Suryoday Basak 4 | * 5 | * Implementation of the Holder table function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_HOLDER_TABLE_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_HOLDER_TABLE_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "holder_table_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline HolderTableFunction::HolderTableFunction() { /* Nothing to do here */ } 22 | 23 | inline void HolderTableFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type HolderTableFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const ElemType objective = -std::abs(std::sin(x1) * std::cos(x2) * 39 | std::exp(std::abs(1 - (std::sqrt(x1 * x1 + x2 * x2) / arma::datum::pi)))); 40 | 41 | return objective; 42 | } 43 | 44 | template 45 | typename MatType::elem_type HolderTableFunction::Evaluate( 46 | const MatType& coordinates) const 47 | { 48 | return Evaluate(coordinates, 0, NumFunctions()); 49 | } 50 | 51 | } // namespace test 52 | } // namespace ens 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /tests/spsa_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file spsa_test.cpp 3 | * @author N Rajiv Vaidyanathan 4 | * @author Marcus Edel 5 | * 6 | * Test file for the SPSA optimizer. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #if defined(ENS_USE_COOT) 14 | #include 15 | #include 16 | #endif 17 | #include 18 | #include "catch.hpp" 19 | #include "test_function_tools.hpp" 20 | 21 | using namespace arma; 22 | using namespace ens; 23 | using namespace ens::test; 24 | 25 | TEMPLATE_TEST_CASE("SPSA_SphereFunction", "[SPSA]", ENS_ALL_TEST_TYPES) 26 | { 27 | SPSA optimizer(0.1, 0.102, 0.16, 0.3, 100000, 0); 28 | FunctionTest( 29 | optimizer, 30 | Tolerances::LargeObj, 31 | Tolerances::LargeCoord); 32 | } 33 | 34 | TEMPLATE_TEST_CASE("SPSA_MatyasFunction", "[SPSA]", ENS_ALL_TEST_TYPES) 35 | { 36 | SPSA optimizer(0.1, 0.102, 0.16, 0.3, 100000, 0); 37 | FunctionTest( 38 | optimizer, 39 | Tolerances::LargeObj, 40 | Tolerances::LargeCoord); 41 | } 42 | 43 | // We don't test on FP16 because SPSA computes a number of values that are too 44 | // large to be represented. 45 | TEMPLATE_TEST_CASE("SPSA_LogisticRegressionFunction", "[SPSA]", ENS_TEST_TYPES) 46 | { 47 | // We allow many trials, because SPSA is definitely not guaranteed to 48 | // converge. 49 | SPSA optimizer(0.5, 0.102, 0.002, 0.3, 5000, 1e-8); 50 | LogisticRegressionFunctionTest( 51 | optimizer, 52 | Tolerances::LRTrainAcc, 53 | Tolerances::LRTestAcc, 54 | 25); 55 | } 56 | -------------------------------------------------------------------------------- /tests/ftml_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ftml_test.cpp 3 | * @author Ryan Curtin 4 | * @author Marcus Edel 5 | * 6 | * Test file for the FTML optimizer. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #if defined(ENS_USE_COOT) 14 | #include 15 | #include 16 | #endif 17 | #include 18 | #include "catch.hpp" 19 | #include "test_function_tools.hpp" 20 | #include "test_types.hpp" 21 | 22 | using namespace ens; 23 | using namespace ens::test; 24 | 25 | TEMPLATE_TEST_CASE("FTML_LogisticRegressionFunction", "[FTML]", 26 | ENS_ALL_TEST_TYPES) 27 | { 28 | FTML optimizer(0.005, 1, 0.9, 0.999, Tolerances::Obj, 100000, 29 | Tolerances::Obj * 10, true); 30 | LogisticRegressionFunctionTest(optimizer); 31 | } 32 | 33 | TEMPLATE_TEST_CASE("FTML_SphereFunction", "[FTML]", ENS_ALL_TEST_TYPES) 34 | { 35 | FTML optimizer(0.06, 2, 0.9, 0.999, Tolerances::Obj / 100, 500000, 36 | Tolerances::Obj / 100, true); 37 | FunctionTest( 38 | optimizer, 39 | Tolerances::LargeObj, 40 | Tolerances::LargeCoord); 41 | } 42 | 43 | TEMPLATE_TEST_CASE("FTML_StyblinskiTangFunction", "[FTML]", ENS_ALL_TEST_TYPES) 44 | { 45 | FTML optimizer(0.8, 2, 0.9, 0.999, Tolerances::Obj, 100000, 46 | Tolerances::Obj / 100, true); 47 | FunctionTest( 48 | optimizer, 49 | 10 * Tolerances::LargeObj, 50 | Tolerances::LargeCoord); 51 | } 52 | 53 | // A test with sp_mat is not done, because FTML uses some parts internally that 54 | // assume the objective is dense. 55 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/grad_clip_by_value.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file grad_clip_by_value.hpp 3 | * @author Marcus Edel 4 | * 5 | * Clips the gradient to a specified min and max. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP 13 | #define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_VALUE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Clip the gradient to a specified min and max. 19 | */ 20 | class GradClipByValue 21 | { 22 | public: 23 | /** 24 | * Set up the gradient clip by value callback class with the min and max 25 | * value. 26 | * 27 | * @param min The minimum value to clip to. 28 | * @param max The maximum value to clip to. 29 | */ 30 | GradClipByValue(const double min, const double max) : lower(min), upper(max) 31 | { /* Nothing to do here. */ } 32 | 33 | /** 34 | * Callback function called at any call to Gradient(). 35 | * 36 | * @param optimizer The optimizer used to update the function. 37 | * @param function Function to optimize. 38 | * @param coordinates Starting point. 39 | * @param gradient Matrix that holds the gradient. 40 | */ 41 | template 42 | bool Gradient(OptimizerType& /* optimizer */, 43 | FunctionType& /* function */, 44 | const MatType& /* coordinates */, 45 | MatType& gradient) 46 | { 47 | gradient = arma::clamp(gradient, lower, upper); 48 | return false; 49 | } 50 | 51 | private: 52 | //! The minimum value to clip to. 53 | const double lower; 54 | 55 | //! The maximum value to clip to. 56 | const double upper; 57 | }; 58 | 59 | } // namespace ens 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/print_loss.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file print_loss.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the print loss callback function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CALLBACKS_PRINT_LOSS_HPP 13 | #define ENSMALLEN_CALLBACKS_PRINT_LOSS_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Print loss function, based on the EndEpoch callback function. 19 | */ 20 | class PrintLoss 21 | { 22 | public: 23 | /** 24 | * Set up the print loss callback class with the width and output stream. 25 | * 26 | * @param ostream Ostream which receives output from this object. 27 | */ 28 | PrintLoss(std::ostream& output = arma::get_cout_stream()) : output(output) 29 | { /* Nothing to do here. */ } 30 | 31 | /** 32 | * Callback function called at the end of a pass over the data. 33 | * 34 | * @param optimizer The optimizer used to update the function. 35 | * @param function Function to optimize. 36 | * @param coordinates Starting point. 37 | * @param epoch The index of the current epoch. 38 | * @param objective Objective value of the current point. 39 | */ 40 | template 41 | bool EndEpoch(OptimizerType& /* optimizer */, 42 | FunctionType& /* function */, 43 | const MatType& /* coordinates */, 44 | const size_t /* epoch */, 45 | const double objective) 46 | { 47 | output << objective << std::endl; 48 | return false; 49 | } 50 | 51 | private: 52 | //! The output stream that all data is to be sent to; example: std::cout. 53 | std::ostream& output; 54 | }; 55 | 56 | } // namespace ens 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fw/proximal/proximal.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file proximal.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * Approximate a vector with another vector on lp ball. Currently support l0 6 | * ball and l1 ball with specific norm. 7 | * It can be used in projected gradient method. 8 | * 9 | * ensmallen is free software; you may redistribute it and/or modify it under 10 | * the terms of the 3-clause BSD license. You should have received a copy of 11 | * the 3-clause BSD license along with ensmallen. If not, see 12 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 13 | */ 14 | #ifndef ENSMALLEN_PROXIMAL_PROXIMAL_HPP 15 | #define ENSMALLEN_PROXIMAL_PROXIMAL_HPP 16 | 17 | namespace ens { 18 | 19 | /** 20 | * Approximate a vector with another vector on lp ball. Currently support l0 21 | * ball and l1 ball with specific norm. 22 | * It can be used in projected gradient method. 23 | */ 24 | class Proximal 25 | { 26 | public: 27 | /** 28 | * Project the vector onto the l1 ball with norm tau. That is, we will solve 29 | * for: 30 | * \f[ 31 | * w = argmin_w ||w - v||_2, \qquad s.t. ~ ||w||_1 \leqslant tau 32 | * \f] 33 | * 34 | * @param v Input vector to be approxmated, the output optimal vector is 35 | * also saved in v. 36 | * @param tau Norm of l1 ball. 37 | */ 38 | template 39 | static void ProjectToL1Ball(MatType& v, double tau); 40 | 41 | /** 42 | * Project the vector onto the l0 ball with norm tau. That is, we try to 43 | * approximate v with sparse vector w: 44 | * \f[ 45 | * w = argmin_w ||w - v||_2, \qquad s.t. ~ ||w||_0 \leqslant tau 46 | * \f] 47 | * 48 | * @param v Input vector to be approxmated, the output optimal vector is 49 | * also saved in v. 50 | * @param tau Norm of l0 ball. 51 | */ 52 | template 53 | static void ProjectToL0Ball(MatType& v, int tau); 54 | }; // class Proximal 55 | 56 | } // namespace ens 57 | 58 | #include "proximal_impl.hpp" 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fw/update_classic.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file update_classic.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * Classic update method for FrankWolfe algorithm. Used as UpdateRuleType. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_FW_UPDATE_CLASSIC_HPP 13 | #define ENSMALLEN_FW_UPDATE_CLASSIC_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Use classic rule in the update step for FrankWolfe algorithm. That is, 19 | * take \f$ \gamma = \frac{2}{k+2} \f$, where \f$ k \f$ is the iteration 20 | * number. The update rule would be: 21 | * \f[ 22 | * x_{k+1} = (1-\gamma) x_k + \gamma s 23 | * \f] 24 | * 25 | */ 26 | class UpdateClassic 27 | { 28 | public: 29 | /** 30 | * Construct the classic update rule for FrankWolfe algorithm. 31 | */ 32 | UpdateClassic() { /* Do nothing. */ } 33 | 34 | /** 35 | * Classic update rule for FrankWolfe. 36 | * 37 | * \f$ x_{k+1} = (1-\gamma)x_k + \gamma s \f$, where \f$ \gamma = 2/(k+2) \f$ 38 | * 39 | * @param function Function to be optimized, not used in this update rule. 40 | * @param oldCoords Previous solution coords. 41 | * @param s Current linear_constr_solution result. 42 | * @param newCoords Output new solution coords. 43 | * @param numIter Current iteration number. 44 | */ 45 | template 46 | void Update(FunctionType& /* function */, 47 | const MatType& oldCoords, 48 | const MatType& s, 49 | MatType& newCoords, 50 | const size_t numIter) 51 | { 52 | typename MatType::elem_type gamma = 2.0 / (numIter + 2.0); 53 | newCoords = (1.0 - gamma) * oldCoords + gamma * s; 54 | } 55 | }; 56 | 57 | } // namespace ens 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/schaffer_function_n4_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file schaffer_function_n4_impl.hpp 3 | * @author Suryoday Basak 4 | * 5 | * Implementation of Schaffer function N.4. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "schaffer_function_n4.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline SchafferFunctionN4::SchafferFunctionN4() { /* Nothing to do here */ } 22 | 23 | inline void SchafferFunctionN4::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type SchafferFunctionN4::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const ElemType objective = ElemType(0.5) + 39 | (std::pow(std::cos(std::sin(std::abs(std::pow(x1, ElemType(2)) - 40 | std::pow(x2, ElemType(2))))), ElemType(2)) - ElemType(0.5)) / 41 | std::pow(1 + ElemType(0.001) * (std::pow(x1, ElemType(2)) + 42 | std::pow(x2, ElemType(2))), ElemType(2)); 43 | 44 | return objective; 45 | } 46 | 47 | template 48 | typename MatType::elem_type SchafferFunctionN4::Evaluate( 49 | const MatType& coordinates) const 50 | { 51 | return Evaluate(coordinates, 0, NumFunctions()); 52 | } 53 | 54 | } // namespace test 55 | } // namespace ens 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/grad_clip_by_norm.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file grad_clip_by_norm.hpp 3 | * @author Marcus Edel 4 | * 5 | * Clip the gradients by multiplying the unit vector of the gradients with the 6 | * threshold. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP 14 | #define ENSMALLEN_CALLBACKS_GRAD_CLIP_BY_NORM_HPP 15 | 16 | namespace ens { 17 | 18 | /** 19 | * Clip the gradients by multiplying the unit vector of the gradients with the 20 | * threshold. 21 | */ 22 | class GradClipByNorm 23 | { 24 | public: 25 | /** 26 | * Set up the gradient clip by norm callback class with the maximum clipping 27 | * value. 28 | * 29 | * @param maxNorm The maximum clipping value. 30 | */ 31 | GradClipByNorm(const double maxNorm) : maxNorm(maxNorm) 32 | { /* Nothing to do here. */ } 33 | 34 | /** 35 | * Callback function called at any call to Gradient(). 36 | * 37 | * @param optimizer The optimizer used to update the function. 38 | * @param function Function to optimize. 39 | * @param coordinates Starting point. 40 | * @param gradient Matrix that holds the gradient. 41 | */ 42 | template 43 | bool Gradient(OptimizerType& /* optimizer */, 44 | FunctionType& /* function */, 45 | const MatType& /* coordinates */, 46 | MatType& gradient) 47 | { 48 | const double gradientNorm = arma::norm(gradient); 49 | if (gradientNorm > maxNorm) 50 | gradient = maxNorm * gradient / gradientNorm; 51 | return false; 52 | } 53 | 54 | private: 55 | //! The maximum clipping value for gradient clipping. 56 | const double maxNorm; 57 | }; 58 | 59 | } // namespace ens 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /include/ensmallen_bits/ens_version.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ens_version.hpp 3 | * @author Conrad Sanderson 4 | * @author Ryan Curtin 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | 12 | 13 | // This follows the Semantic Versioning pattern defined in https://semver.org/. 14 | 15 | #define ENS_VERSION_MAJOR 3 16 | // The minor version is two digits so regular numerical comparisons of versions 17 | // work right. The first minor version of a release is always 10. 18 | #define ENS_VERSION_MINOR 11 19 | #define ENS_VERSION_PATCH 0 20 | // If this is a release candidate, it will be reflected in the version name 21 | // (i.e. the version name will be "RC1", "RC2", etc.). Otherwise the version 22 | // name will typically be a seemingly arbitrary set of words that does not 23 | // contain the capitalized string "RC". 24 | #define ENS_VERSION_NAME "Sunny Day" 25 | // Incorporate the date the version was released. 26 | #define ENS_VERSION_YEAR "2025" 27 | #define ENS_VERSION_MONTH "12" 28 | #define ENS_VERSION_DAY "15" 29 | 30 | namespace ens { 31 | 32 | struct version 33 | { 34 | static const unsigned int major = ENS_VERSION_MAJOR; 35 | static const unsigned int minor = ENS_VERSION_MINOR; 36 | static const unsigned int patch = ENS_VERSION_PATCH; 37 | 38 | static inline std::string as_string() 39 | { 40 | const char* nickname = ENS_VERSION_NAME; 41 | 42 | std::stringstream ss; 43 | ss << version::major << '.' << version::minor << '.' << version::patch 44 | << " (" << nickname << ')'; 45 | 46 | return ss.str(); 47 | } 48 | 49 | static inline std::string date() 50 | { 51 | std::stringstream ss; 52 | ss << ENS_VERSION_YEAR << '-' << ENS_VERSION_MONTH << '-' << ENS_VERSION_DAY; 53 | 54 | return ss.str(); 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/cross_in_tray_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file cross_in_tray_function_impl.hpp 3 | * @author Suryoday Basak 4 | * 5 | * Implementation of the Cross-in-Tray function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_CROSS_IN_TRAY_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_CROSS_IN_TRAY_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "cross_in_tray_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline CrossInTrayFunction::CrossInTrayFunction() { /* Nothing to do here */ } 22 | 23 | inline void CrossInTrayFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type CrossInTrayFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | // Compute objective in higher precision, then cast down. 39 | const double objective = -0.0001 * std::pow(std::abs(std::sin(double(x1)) * 40 | std::sin(double(x2)) * 41 | std::exp(std::abs(100 - (std::sqrt(std::pow(double(x1), 2) + 42 | std::pow(double(x2), 2)) / arma::datum::pi))) + 1), 0.1); 43 | return ElemType(objective); 44 | } 45 | 46 | template 47 | typename MatType::elem_type CrossInTrayFunction::Evaluate( 48 | const MatType& coordinates) const 49 | { 50 | return Evaluate(coordinates, 0, NumFunctions()); 51 | } 52 | 53 | } // namespace test 54 | } // namespace ens 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /tests/yogi_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file yogi_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("Yogi_SphereFunction", "[Yogi]", ENS_ALL_TEST_TYPES) 23 | { 24 | Yogi optimizer(1.0, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false); 25 | FunctionTest( 26 | optimizer, 27 | 10 * Tolerances::LargeObj, 28 | 10 * Tolerances::LargeCoord); 29 | } 30 | 31 | TEMPLATE_TEST_CASE("Yogi_McCormickFunction", "[Yogi]", ENS_ALL_TEST_TYPES) 32 | { 33 | Yogi optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false); 34 | FunctionTest( 35 | optimizer, 36 | Tolerances::LargeObj, 37 | Tolerances::LargeCoord); 38 | } 39 | 40 | TEMPLATE_TEST_CASE("Yogi_LogisticRegressionFunction", "[Yogi]", 41 | ENS_ALL_TEST_TYPES) 42 | { 43 | Yogi optimizer(0.032); 44 | // For low-precision, we need to use a very small step size and some other 45 | // tuning to keep from diverging. 46 | size_t trials = 1; 47 | if (sizeof(typename TestType::elem_type) < 4) 48 | { 49 | optimizer.StepSize() = 5e-4; 50 | optimizer.BatchSize() = 16; 51 | optimizer.Tolerance() = -1.0; // Force maximum number of iterations. 52 | optimizer.MaxIterations() = 1000000; 53 | trials = 5; 54 | } 55 | LogisticRegressionFunctionTest( 56 | optimizer, 57 | Tolerances::LRTrainAcc, 58 | Tolerances::LRTestAcc, 59 | trials); 60 | } 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/moead/weight_init_policies/dirichlet_init.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file dirichlet_init.hpp 3 | * @author Nanubala Gnana Sai 4 | * 5 | * The Dirichlet method of Weight Initialization. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOEAD_DIRICHLET_HPP 13 | #define ENSMALLEN_MOEAD_DIRICHLET_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The Dirichlet method for initializing weights. Sampling a 19 | * Dirichlet distribution with parameters set to one returns 20 | * point lying on unit simplex with uniform distribution. 21 | */ 22 | class Dirichlet 23 | { 24 | public: 25 | /** 26 | * Constructor for Dirichlet policy. 27 | */ 28 | Dirichlet() 29 | { 30 | /* Nothing to do. */ 31 | } 32 | 33 | /** 34 | * Generate the reference direction matrix. 35 | * 36 | * @tparam MatType The type of the matrix used for constructing weights. 37 | * @param numObjectives The dimensionality of objective space. 38 | * @param numPoints The number of reference directions requested. 39 | * @param epsilon Handle numerical stability after weight initialization. 40 | */ 41 | template 42 | MatType Generate(const size_t numObjectives, 43 | const size_t numPoints, 44 | const double epsilon) 45 | { 46 | // TODO: Replace with randg once Bandicoot supports it. Simulate randg using 47 | // inverse transform sampling. 48 | // arma::mat weights = arma::randg(numObjectives, numPoints, 49 | // arma::distr_param(1.0, 1.0)) + epsilon; 50 | MatType weights = -log(1.0 - randu( 51 | numObjectives, numPoints)) + epsilon; 52 | 53 | // Normalize each column. 54 | return normalise(weights, 1, 0); 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fbs/l1_penalty.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file l1_penalty.hpp 3 | * @author Ryan Curtin 4 | * 5 | * An implementation of the proximal operator for the L1 penalty (also known as 6 | * the shrinkage operator). 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_FBS_L1_PENALTY_HPP 14 | #define ENSMALLEN_FBS_L1_PENALTY_HPP 15 | 16 | namespace ens { 17 | 18 | /** 19 | * The L1Penalty applies a non-differentiable L1-norm penalty to the coordinates 20 | * during optimization: 21 | * 22 | * `lambda * || coordinates ||_1` 23 | * 24 | * This class is meant to be used with the FBS optimizer, and any other 25 | * optimizer that uses a proximal operator/step. 26 | */ 27 | class L1Penalty 28 | { 29 | public: 30 | /** 31 | * Construct an L1Penalty object with a given penalty `lambda`. 32 | */ 33 | L1Penalty(const double lambda = 0.0); 34 | 35 | /** 36 | * Evaluate the L1 penalty function: `lambda * || coordinates ||_1`. 37 | */ 38 | template 39 | typename MatType::elem_type Evaluate(const MatType& coordinates) const; 40 | 41 | /** 42 | * After taking a forward step of size `stepSize`, apply a backwards step / 43 | * proximal operator that applies the L1 penalty to `coordinates`. 44 | */ 45 | template 46 | void ProximalStep(MatType& coordinates, const double stepSize) const; 47 | 48 | //! Get the L1 penalty to use when applying the proximal step. 49 | double Lambda() const { return lambda; } 50 | //! Modify the L1 penalty to use when applying the proximal step. 51 | double& Lambda() { return lambda; } 52 | 53 | private: 54 | //! The L1 penalty value to use. 55 | double lambda; 56 | }; 57 | 58 | } // namespace ens 59 | 60 | // Include implementation. 61 | #include "l1_penalty_impl.hpp" 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cmaes/full_selection.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file full_selection.hpp 3 | * @author Marcus Edel 4 | * 5 | * Select the full dataset for use in the Evaluation step. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CMAES_FULL_SELECTION_HPP 13 | #define ENSMALLEN_CMAES_FULL_SELECTION_HPP 14 | 15 | namespace ens { 16 | 17 | /* 18 | * Select the full dataset for use in the Evaluation step. 19 | */ 20 | class FullSelection 21 | { 22 | public: 23 | /** 24 | * Select the full dataset to calculate the objective function. 25 | * 26 | * @tparam SeparableFunctionType Type of the function to be evaluated. 27 | * @param function Function to optimize. 28 | * @param batchSize Batch size to use for each step. 29 | * @param terminate Whether optimization should be terminated after this call. 30 | * @param iterate starting point. 31 | */ 32 | template 35 | double Select(SeparableFunctionType& function, 36 | const size_t batchSize, 37 | const MatType& iterate, 38 | bool& terminate, 39 | CallbackTypes&... callbacks) 40 | { 41 | // Find the number of functions to use. 42 | const size_t numFunctions = function.NumFunctions(); 43 | 44 | typename MatType::elem_type objective = 0; 45 | for (size_t f = 0; f < numFunctions; f += batchSize) 46 | { 47 | const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f); 48 | objective += function.Evaluate(iterate, f, effectiveBatchSize); 49 | 50 | terminate |= Callback::Evaluate(*this, f, iterate, objective, 51 | callbacks...); 52 | } 53 | 54 | return objective; 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/quadratic_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file quadratic_function_impl.hpp 3 | * @author Ryan Curtin 4 | * 5 | * Implementation of QuadraticFunction, f(x) = | x |. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_QUADRATIC_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_QUADRATIC_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "quadratic_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline QuadraticFunction::QuadraticFunction() { /* Nothing to do here */ } 22 | 23 | inline void QuadraticFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type QuadraticFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | return coordinates[0] * coordinates[0]; 32 | } 33 | 34 | template 35 | typename MatType::elem_type QuadraticFunction::Evaluate(const MatType& coordinates) 36 | const 37 | { 38 | return Evaluate(coordinates, 0, NumFunctions()); 39 | } 40 | 41 | template 42 | inline void QuadraticFunction::Gradient(const MatType& coordinates, 43 | const size_t /* begin */, 44 | GradType& gradient, 45 | const size_t /* batchSize */) const 46 | { 47 | gradient.set_size(1, 1); 48 | gradient(0, 0) = 2 * coordinates[0]; 49 | } 50 | 51 | template 52 | inline void QuadraticFunction::Gradient(const MatType& coordinates, 53 | GradType& gradient) 54 | { 55 | Gradient(coordinates, 0, gradient, 1); 56 | } 57 | 58 | } // namespace test 59 | } // namespace ens 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /include/ensmallen_bits/moead/decomposition_policies/weighted_decomposition.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file weighted_decomposition.hpp 3 | * @author Nanubala Gnana Sai 4 | * 5 | * The Weighted Average decomposition policy. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOEAD_WEIGHTED_HPP 13 | #define ENSMALLEN_MOEAD_WEIGHTED_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The Weighted average method of decomposition. The working principle is to 19 | * minimize the dot product between reference direction and the line connecting 20 | * objective vector and ideal point. 21 | * 22 | * For more information, see the following: 23 | * @code 24 | * article{zhang2007moea, 25 | * title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition}, 26 | * author={Zhang, Qingfu and Li, Hui}, 27 | * journal={IEEE Transactions on evolutionary computation}, 28 | * pages={712--731}, 29 | * year={2007} 30 | * @endcode 31 | */ 32 | class WeightedAverage 33 | { 34 | public: 35 | /** 36 | * Constructor for Weighted Average decomposition policy. 37 | */ 38 | WeightedAverage() 39 | { 40 | /* Nothing to do. */ 41 | } 42 | 43 | /** 44 | * Decompose the weight vectors. 45 | * 46 | * @tparam VecType The type of the vector used in the decommposition. 47 | * @param weight The weight vector corresponding to a subproblem. 48 | * @param idealPoint The reference point in the objective space. 49 | * @param candidateFitness The objective vector of the candidate. 50 | */ 51 | template 52 | typename VecType::elem_type Apply(const VecType& weight, 53 | const VecType& /* idealPoint */, 54 | const VecType& candidateFitness) 55 | { 56 | return dot(weight, candidateFitness); 57 | } 58 | }; 59 | 60 | } // namespace ens 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/gradient_descent_test_function.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file gradient_descent_test_function.hpp 3 | * @author Sumedh Ghaisas 4 | * 5 | * Very simple test function for SGD. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_GRADIENT_DESCENT_TEST_FUNCTION_HPP 13 | #define ENSMALLEN_PROBLEMS_GRADIENT_DESCENT_TEST_FUNCTION_HPP 14 | 15 | namespace ens { 16 | namespace test { 17 | 18 | //! Very, very simple test function which is the composite of three other 19 | //! functions. The gradient is not very steep far away from the optimum, so a 20 | //! larger step size may be required to optimize it in a reasonable number of 21 | //! iterations. 22 | class GDTestFunction 23 | { 24 | public: 25 | //! Nothing to do for the constructor. 26 | GDTestFunction() { } 27 | 28 | //! Evaluate a function. 29 | template 30 | typename MatType::elem_type Evaluate(const MatType& coordinates) const; 31 | 32 | //! Evaluate the gradient of a function. 33 | template 34 | void Gradient(const MatType& coordinates, GradType& gradient) const; 35 | 36 | // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not 37 | // required for using ensmallen to optimize this function! They are 38 | // specifically used as a convenience just for ensmallen's testing 39 | // infrastructure. 40 | 41 | //! Get the starting point. 42 | template 43 | MatType GetInitialPoint() const { return MatType("1; 3; 2"); } 44 | 45 | //! Get the final point. 46 | template 47 | MatType GetFinalPoint() const { return MatType("0; 0; 0"); } 48 | 49 | //! Get the final objective. 50 | double GetFinalObjective() const { return 0.0; } 51 | }; 52 | 53 | } // namespace test 54 | } // namespace ens 55 | 56 | #include "gradient_descent_test_function_impl.hpp" 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fw/func_sq.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file func_sq.hpp 3 | * @author Chenzhe Diao 4 | * 5 | * Square loss function: \f$ x-> 0.5 * || Ax - b ||_2^2 \f$. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_FW_FUNC_SQ_HPP 13 | #define ENSMALLEN_FW_FUNC_SQ_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Square loss function \f$ f(x) = 0.5 * ||Ax - b||_2^2 \f$. 19 | * 20 | * Contains matrix \f$ A \f$ and vector \f$ b \f$. 21 | */ 22 | class FuncSq 23 | { 24 | public: 25 | /** 26 | * Construct the square loss function. 27 | * 28 | * @param A matrix A. 29 | * @param b vector b. 30 | */ 31 | FuncSq(const arma::mat& A, const arma::vec& b) : A(A), b(b) 32 | {/* Nothing to do. */} 33 | 34 | /** 35 | * Evaluation of the function. 36 | * \f$ f(x) = 0.5 * ||Ax - b||_2^2 \f$ 37 | * 38 | * @param coords vector x. 39 | * @return \f$ f(x) \f$. 40 | */ 41 | double Evaluate(const arma::mat& coords) 42 | { 43 | arma::vec r = A * coords - b; 44 | return arma::dot(r, r) * 0.5; 45 | } 46 | 47 | /** 48 | * Gradient of square loss function. 49 | * \f$ \nabla f(x) = A^T(Ax - b) \f$ 50 | * 51 | * @param coords input vector x. 52 | * @param gradient output gradient vector. 53 | */ 54 | void Gradient(const arma::mat& coords, arma::mat& gradient) 55 | { 56 | arma::vec r = A * coords - b; 57 | gradient = A.t() * r; 58 | } 59 | 60 | //! Get the matrix A. 61 | arma::mat MatrixA() const {return A;} 62 | //! Modify the matrix A. 63 | arma::mat& MatrixA() {return A;} 64 | 65 | //! Get the vector b. 66 | arma::vec Vectorb() const { return b; } 67 | //! Modify the vector b. 68 | arma::vec& Vectorb() { return b; } 69 | 70 | private: 71 | //! Matrix A in square loss function. 72 | arma::mat A; 73 | 74 | //! Vector b in square loss function. 75 | arma::vec b; 76 | }; 77 | 78 | } // namespace ens 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # The tests that need to be compiled. 2 | set(ENSMALLEN_TESTS_SOURCES 3 | main.cpp 4 | active_cmaes_test.cpp 5 | ada_belief_test.cpp 6 | ada_bound_test.cpp 7 | ada_delta_test.cpp 8 | ada_grad_test.cpp 9 | ada_sqrt_test.cpp 10 | adam_test.cpp 11 | aug_lagrangian_test.cpp 12 | bigbatch_sgd_test.cpp 13 | callbacks_test.cpp 14 | cd_test.cpp 15 | cmaes_test.cpp 16 | cne_test.cpp 17 | de_test.cpp 18 | delta_bar_delta_test.cpp 19 | demon_adam_test.cpp 20 | demon_sgd_test.cpp 21 | eve_test.cpp 22 | fasta_test.cpp 23 | fbs_test.cpp 24 | fista_test.cpp 25 | frankwolfe_test.cpp 26 | ftml_test.cpp 27 | function_test.cpp 28 | gradient_descent_test.cpp 29 | grid_search_test.cpp 30 | iqn_test.cpp 31 | indicators_test.cpp 32 | katyusha_test.cpp 33 | lbfgs_test.cpp 34 | line_search_test.cpp 35 | lookahead_test.cpp 36 | lrsdp_test.cpp 37 | moead_test.cpp 38 | agemoea_test.cpp 39 | momentum_delta_bar_delta_test.cpp 40 | momentum_sgd_test.cpp 41 | nesterov_momentum_sgd_test.cpp 42 | nsga2_test.cpp 43 | parallel_sgd_test.cpp 44 | pop_cmaes_test.cpp 45 | proximal_test.cpp 46 | pso_test.cpp 47 | quasi_hyperbolic_momentum_sgd_test.cpp 48 | rmsprop_test.cpp 49 | sa_test.cpp 50 | sarah_test.cpp 51 | sdp_primal_dual_test.cpp 52 | sgdr_test.cpp 53 | sgd_test.cpp 54 | smorms3_test.cpp 55 | snapshot_ensembles.cpp 56 | spalera_sgd_test.cpp 57 | spsa_test.cpp 58 | svrg_test.cpp 59 | swats_test.cpp 60 | test_types.cpp 61 | wn_grad_test.cpp 62 | yogi_test.cpp 63 | ) 64 | 65 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) 66 | add_executable(ensmallen_tests EXCLUDE_FROM_ALL ${ENSMALLEN_TESTS_SOURCES}) 67 | target_link_libraries(ensmallen_tests PRIVATE ensmallen) 68 | 69 | # Copy test data into place. 70 | add_custom_command(TARGET ensmallen_tests 71 | POST_BUILD 72 | COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/data/ 73 | ${CMAKE_BINARY_DIR}/data/ 74 | ) 75 | 76 | enable_testing() 77 | add_test(NAME ensmallen_tests COMMAND ensmallen_tests 78 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) 79 | -------------------------------------------------------------------------------- /tests/sgdr_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sgdr_test.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | #include "test_types.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | TEMPLATE_TEST_CASE("SGDR_CyclicalResetTest", "[SGDR]", ENS_ALL_CPU_TEST_TYPES) 24 | { 25 | const double stepSize = 0.5; 26 | TestType iterate; 27 | 28 | // Now run cyclical decay policy with a couple of multiplicators and initial 29 | // restarts. 30 | for (size_t restart = 5; restart < 100; restart += 10) 31 | { 32 | for (size_t mult = 2; mult < 5; ++mult) 33 | { 34 | double epochStepSize = stepSize; 35 | 36 | CyclicalDecay cyclicalDecay(restart, double(mult), stepSize); 37 | CyclicalDecay::Policy p(cyclicalDecay); 38 | cyclicalDecay.EpochBatches() = (double) 1000 / 10; 39 | 40 | // Create all restart epochs. 41 | arma::Col nextRestart(1000 / 10 / mult); 42 | nextRestart(0) = restart; 43 | for (size_t j = 1; j < nextRestart.n_elem; ++j) 44 | nextRestart(j) = nextRestart(j - 1) * mult; 45 | 46 | for (size_t i = 0; i < 1000; ++i) 47 | { 48 | p.Update(iterate, epochStepSize, iterate); 49 | if (i <= restart || accu(find(nextRestart == i)) > 0) 50 | { 51 | REQUIRE(epochStepSize == stepSize); 52 | } 53 | } 54 | } 55 | } 56 | } 57 | 58 | TEMPLATE_TEST_CASE("SGDR_LogisticRegressionFunction", "[SGDR]", 59 | ENS_ALL_TEST_TYPES) 60 | { 61 | // Run SGDR with a couple of batch sizes. 62 | for (size_t batchSize = 5; batchSize < 50; batchSize += 5) 63 | { 64 | SGDR<> sgdr(50, 2.0, batchSize, 0.005 * batchSize, 10000, 1e-3); 65 | LogisticRegressionFunctionTest(sgdr); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sa/exponential_schedule.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file exponential_schedule.hpp 3 | * @author Zhihao Lou 4 | * 5 | * Exponential (geometric) cooling schedule used in SA. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SA_EXPONENTIAL_SCHEDULE_HPP 13 | #define ENSMALLEN_SA_EXPONENTIAL_SCHEDULE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The exponential cooling schedule cools the temperature T at every step 19 | * according to the equation 20 | * 21 | * \f[ 22 | * T_{n+1} = (1-\lambda) T_{n} 23 | * \f] 24 | * 25 | * where \f$ 0<\lambda<1 \f$ is the cooling speed. The smaller \f$ \lambda \f$ 26 | * is, the slower the cooling speed, and better the final result will be. Some 27 | * literature uses \f$ \alpha = (-1 \lambda) \f$ instead. In practice, 28 | * \f$ \alpha \f$ is very close to 1 and will be awkward to input (e.g. 29 | * alpha = 0.999999 vs lambda = 1e-6). 30 | */ 31 | class ExponentialSchedule 32 | { 33 | public: 34 | /* 35 | * Construct the ExponentialSchedule with the given parameter. 36 | * 37 | * @param lambda Cooling speed. 38 | */ 39 | ExponentialSchedule(const double lambda = 0.001) : lambda(lambda) { } 40 | 41 | /** 42 | * Returns the next temperature given current status. The current system's 43 | * energy is not used in this calculation. 44 | * 45 | * @param currentTemperature Current temperature of system. 46 | * @param currentEnergy Current energy of system (not used). 47 | */ 48 | template 49 | ElemType NextTemperature( 50 | const double currentTemperature, const ElemType /* currentEnergy */) 51 | { 52 | return ElemType((1 - lambda) * currentTemperature); 53 | } 54 | 55 | //! Get the cooling speed, lambda. 56 | double Lambda() const { return lambda; } 57 | //! Modify the cooling speed, lambda. 58 | double& Lambda() { return lambda; } 59 | 60 | private: 61 | //! The cooling speed. 62 | double lambda; 63 | }; 64 | 65 | } // namespace ens 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /scripts/update-website-after-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This script is used to update the website after a release is made. Push 4 | # access to the ensmallen.org website is needed. Generally, this script will be 5 | # run by mlpack-bot, so it never needs to be run by hand. 6 | # 7 | # Usage: update-website-after-release.sh 8 | 9 | MAJOR=$1; 10 | MINOR=$2; 11 | PATCH=$3; 12 | 13 | # Make sure that the mlpack repository exists. 14 | dest_remote_name=`git remote -v |\ 15 | grep "mlpack/ensmallen (fetch)" |\ 16 | head -1 |\ 17 | awk -F' ' '{ print $1 }'`; 18 | 19 | if [ "a$dest_remote_name" == "a" ]; then 20 | echo "No git remote found for mlpack/ensmallen!"; 21 | echo "Make sure that you've got the ensmallen repository as a remote, and" \ 22 | "that the master branch from that remote is checked out."; 23 | echo "You can do this with a fresh repository via \`git clone" \ 24 | "https://github.com/mlpack/ensmallen\`."; 25 | exit 1; 26 | fi 27 | 28 | # Update the checked out repository, so that we can get the tags. 29 | git fetch $dest_remote_name; 30 | 31 | # Check out a copy of the ensmallen.org repository. 32 | git clone git@github.com:mlpack/ensmallen.org /tmp/ensmallen.org/; 33 | 34 | # Create the release file. 35 | git archive --prefix=ensmallen-$MAJOR.$MINOR.$PATCH/ $MAJOR.$MINOR.$PATCH |\ 36 | gzip > /tmp/ensmallen.org/files/ensmallen-$MAJOR.$MINOR.$PATCH.tar.gz; 37 | 38 | # Now update the website. 39 | wd=`pwd`; 40 | cd /tmp/ensmallen.org/; 41 | git add files/ensmallen-$MAJOR.$MINOR.$PATCH.tar.gz; 42 | 43 | # Update the link to the latest version. 44 | cd files/; 45 | rm ensmallen-latest.tar.gz; 46 | ln -s ensmallen-$MAJOR.$MINOR.$PATCH.tar.gz ensmallen-latest.tar.gz; 47 | cd ../ 48 | 49 | # Update the index page. 50 | sed -i 's/\[ensmallen-[0-9]*\.[0-9]*\.[0-9]\.tar\.gz\](files\/ensmallen-[0-9]*\.[0-9]*\.[0-9]*\.tar.gz)/[ensmallen-'$MAJOR'.'$MINOR'.'$PATCH'.tar.gz](files\/ensmallen-'$MAJOR'.'$MINOR'.'$PATCH'.tar.gz)/' index.md 51 | 52 | git add files/ensmallen-latest.tar.gz; 53 | git add index.md; 54 | git commit -m "Release version $MAJOR.$MINOR.$PATCH."; 55 | 56 | # Finally, push, and we're done. 57 | git push origin; 58 | cd $wd; 59 | 60 | rm -rf /tmp/ensmallen.org; 61 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cd/descent_policies/cyclic_descent.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file cyclic_descent.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Cyclic descent policy for Stochastic Coordinate Descent (SCD). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SCD_DESCENT_POLICIES_CYCLIC_HPP 13 | #define ENSMALLEN_SCD_DESCENT_POLICIES_CYCLIC_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Cyclic descent policy for Stochastic Coordinate Descent(SCD). This 19 | * descent scheme picks a the co-ordinate for the descent in a cyclic manner 20 | * serially. 21 | * 22 | * For more information, see the following. 23 | * @code 24 | * @inproceedings{Shalev-Shwartz2009, 25 | * author = {Shalev-Shwartz, Shai and Tewari, Ambuj}, 26 | * title = {Stochastic Methods for L1 Regularized Loss Minimization}, 27 | * booktitle = {Proceedings of the 26th Annual International Conference on 28 | * Machine Learning}, 29 | * series = {ICML '09}, 30 | * year = {2009}, 31 | * isbn = {978-1-60558-516-1} 32 | * } 33 | * @endcode 34 | */ 35 | class CyclicDescent 36 | { 37 | public: 38 | /** 39 | * The DescentFeature method is used to get the descent coordinate for the 40 | * current iteration. 41 | * 42 | * @tparam ResolvableFunctionType The type of the function to be optimized. 43 | * @param iteration The iteration number for which the feature is to be 44 | * obtained. 45 | * @param iterate The current value of the decision variable. 46 | * @param function The function to be optimized. 47 | * @return The index of the coordinate to be descended. 48 | */ 49 | template 50 | static size_t DescentFeature(const size_t iteration, 51 | const MatType& /* iterate */, 52 | const ResolvableFunctionType& function) 53 | { 54 | return iteration % function.NumFeatures(); 55 | } 56 | }; 57 | 58 | } // namespace ens 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /tests/demon_adam_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file demon_adam_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("DemonAdam_LogisticRegressionFunction", "[DemonAdam]", 23 | ENS_ALL_TEST_TYPES) 24 | { 25 | DemonAdam optimizer(6.4, 32, 0.9, 0.9, 0.999, Tolerances::Obj, 26 | 10000, Tolerances::Obj / 10, true, true, true); 27 | // This may require a few attempts to get right. 28 | LogisticRegressionFunctionTest(optimizer, 29 | Tolerances::LRTrainAcc, 30 | Tolerances::LRTestAcc, 31 | 6); 32 | } 33 | 34 | TEMPLATE_TEST_CASE("DemonAdaMax_LogisticRegressionFunction", "[DemonAdam]", 35 | ENS_ALL_TEST_TYPES) 36 | { 37 | DemonAdamType optimizer(5.0, 10, 0.9, 0.9, 0.999, 38 | Tolerances::Obj, 10000, Tolerances::Obj / 10, true, 39 | true, true); 40 | // This may require a few attempts to get right. 41 | LogisticRegressionFunctionTest(optimizer, 42 | Tolerances::LRTrainAcc, 43 | Tolerances::LRTestAcc, 44 | 6); 45 | } 46 | 47 | TEMPLATE_TEST_CASE("DemonAdam_SphereFunction", "[DemonAdam]", 48 | ENS_ALL_TEST_TYPES, ENS_SPARSE_TEST_TYPES) 49 | { 50 | DemonAdam optimizer(1.0, 2, 0.9); 51 | FunctionTest( 52 | optimizer, 53 | 10 * Tolerances::LargeObj, 54 | Tolerances::LargeCoord); 55 | } 56 | 57 | TEMPLATE_TEST_CASE("DemonAdam_MatyasFunction", "[DemonAdam]", 58 | ENS_ALL_TEST_TYPES) 59 | { 60 | DemonAdam optimizer(0.5, 1, 0.9); 61 | FunctionTest( 62 | optimizer, 63 | 10 * Tolerances::LargeObj, 64 | Tolerances::LargeCoord); 65 | } 66 | -------------------------------------------------------------------------------- /include/ensmallen_bits/moead/decomposition_policies/tchebycheff_decomposition.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file tchebycheff_decomposition.hpp 3 | * @author Nanubala Gnana Sai 4 | * 5 | * The Tchebycheff Weight decomposition policy. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP 13 | #define ENSMALLEN_MOEAD_TCHEBYCHEFF_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The Tchebycheff method works by taking the maximum of element-wise product 19 | * between reference direction and the line connecting objective vector and 20 | * ideal point. 21 | * 22 | * Under mild conditions, for each Pareto Optimal point there exists a reference 23 | * direction such that the given point is also the optimal solution 24 | * to this scalar objective. 25 | * 26 | * For more information, see the following: 27 | * @code 28 | * article{zhang2007moea, 29 | * title={MOEA/D: A multiobjective evolutionary algorithm based on decomposition}, 30 | * author={Zhang, Qingfu and Li, Hui}, 31 | * journal={IEEE Transactions on evolutionary computation}, 32 | * pages={712--731}, 33 | * year={2007} 34 | * @endcode 35 | */ 36 | class Tchebycheff 37 | { 38 | public: 39 | /** 40 | * Constructor for Tchebycheff decomposition policy. 41 | */ 42 | Tchebycheff() 43 | { 44 | /* Nothing to do. */ 45 | } 46 | 47 | /** 48 | * Decompose the weight vectors. 49 | * 50 | * @tparam VecType The type of the vector used in the decommposition. 51 | * @param weight The weight vector corresponding to a subproblem. 52 | * @param idealPoint The reference point in the objective space. 53 | * @param candidateFitness The objective vector of the candidate. 54 | */ 55 | template 56 | typename VecType::elem_type Apply(const VecType& weight, 57 | const VecType& idealPoint, 58 | const VecType& candidateFitness) 59 | { 60 | return max(weight % abs(candidateFitness - idealPoint)); 61 | } 62 | }; 63 | 64 | } // namespace ens 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /tests/cmaes_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file cmaes_test.cpp 3 | * @author Marcus Edel 4 | * @author Kartik Nighania 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #if defined(ENS_USE_COOT) 13 | #include 14 | #include 15 | #endif 16 | #include 17 | #include "catch.hpp" 18 | #include "test_function_tools.hpp" 19 | #include "test_types.hpp" 20 | 21 | using namespace ens; 22 | using namespace ens::test; 23 | 24 | /** 25 | * Run CMA-ES with the full selection policy on logistic regression and 26 | * make sure the results are acceptable. 27 | */ 28 | TEMPLATE_TEST_CASE("CMAES_LogisticRegressionFunction", "[CMAES]", 29 | ENS_FULLPREC_TEST_TYPES) 30 | { 31 | BoundaryBoxConstraint b(-10, 10); 32 | CMAES> cmaes( 33 | 0, b, 32, 500, 1e-3); 34 | cmaes.StepSize() = 0.6; 35 | LogisticRegressionFunctionTest(cmaes, 36 | Tolerances::LRTrainAcc, Tolerances::LRTestAcc, 5); 37 | } 38 | 39 | /** 40 | * Run CMA-ES with the random selection policy on logistic regression and 41 | * make sure the results are acceptable. 42 | */ 43 | TEMPLATE_TEST_CASE("ApproxCMAES_LogisticRegressionFunction", "[CMAES]", 44 | ENS_TEST_TYPES) 45 | { 46 | BoundaryBoxConstraint b(-10, 10); 47 | ApproxCMAES> cmaes(256, b, 16, 500, 1e-3); 48 | cmaes.StepSize() = 0.6; 49 | LogisticRegressionFunctionTest(cmaes, 50 | Tolerances::LRTrainAcc, Tolerances::LRTestAcc, 5); 51 | } 52 | 53 | /** 54 | * Run CMA-ES with the random selection and empty transformation policies 55 | * on logistic regression and make sure the results are acceptable. 56 | * Use arma::fmat. 57 | */ 58 | TEMPLATE_TEST_CASE("ApproxCMAES_EmptyTransformationLogisticRegressionFunction", 59 | "[CMAES]", ENS_TEST_TYPES) 60 | { 61 | ApproxCMAES> 62 | cmaes(0, EmptyTransformation(), 16, 500, 1e-3); 63 | LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); 64 | } 65 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sdp/lrsdp_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file lrsdp.cpp 3 | * @author Ryan Curtin 4 | * 5 | * An implementation of Monteiro and Burer's formulation of low-rank 6 | * semidefinite programs (LR-SDP). 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_SDP_LRSDP_IMPL_HPP 14 | #define ENSMALLEN_SDP_LRSDP_IMPL_HPP 15 | 16 | #include "lrsdp.hpp" 17 | 18 | namespace ens { 19 | 20 | template 21 | LRSDP::LRSDP(const size_t numSparseConstraints, 22 | const size_t numDenseConstraints, 23 | const arma::Mat& initialPoint, 24 | const size_t maxIterations) : 25 | function(numSparseConstraints, numDenseConstraints, initialPoint), 26 | maxIterations(maxIterations) 27 | { } 28 | 29 | template 30 | template 31 | typename MatType::elem_type LRSDP::Optimize( 32 | MatType& coordinates, CallbackTypes&&... callbacks) 33 | { 34 | function.RRTAny().Clean(); 35 | function.RRTAny().template Set( 36 | new MatType(coordinates * coordinates.t())); 37 | 38 | augLag.MaxIterations() = maxIterations; 39 | typename ForwardType::bvec lambda(function.NumConstraints()); 40 | double sigma = 10; 41 | augLag.Optimize(function, coordinates, lambda, sigma, callbacks...); 42 | 43 | return function.Evaluate(coordinates); 44 | } 45 | 46 | template 47 | template 48 | typename MatType::elem_type LRSDP::Optimize( 49 | MatType& coordinates, 50 | VecType& lambda, 51 | double& sigma, 52 | CallbackTypes&&... callbacks) 53 | { 54 | function.RRTAny().Clean(); 55 | function.RRTAny().template Set( 56 | new MatType(coordinates * coordinates.t())); 57 | 58 | augLag.MaxIterations() = maxIterations; 59 | augLag.Optimize(function, coordinates, lambda, sigma, callbacks...); 60 | 61 | return function.Evaluate(coordinates); 62 | } 63 | 64 | } // namespace ens 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/problems.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file problems.hpp 3 | * @author Ryan Curtin 4 | * 5 | * Include each of the test problems for ensmallen. 6 | */ 7 | #ifndef ENSMALLEN_PROBLEMS_PROBLEMS_HPP 8 | #define ENSMALLEN_PROBLEMS_PROBLEMS_HPP 9 | 10 | #include "ackley_function.hpp" 11 | #include "aug_lagrangian_test_functions.hpp" 12 | #include "beale_function.hpp" 13 | #include "booth_function.hpp" 14 | #include "bukin_function.hpp" 15 | #include "colville_function.hpp" 16 | #include "cross_in_tray_function.hpp" 17 | #include "drop_wave_function.hpp" 18 | #include "easom_function.hpp" 19 | #include "eggholder_function.hpp" 20 | #include "fonseca_fleming_function.hpp" 21 | #include "fw_test_function.hpp" 22 | #include "generalized_rosenbrock_function.hpp" 23 | #include "goldstein_price_function.hpp" 24 | #include "gradient_descent_test_function.hpp" 25 | #include "himmelblau_function.hpp" 26 | #include "holder_table_function.hpp" 27 | #include "levy_function_n13.hpp" 28 | #include "logistic_regression_function.hpp" 29 | #include "matyas_function.hpp" 30 | #include "mc_cormick_function.hpp" 31 | #include "quadratic_function.hpp" 32 | #include "rastrigin_function.hpp" 33 | #include "rosenbrock_function.hpp" 34 | #include "rosenbrock_wood_function.hpp" 35 | #include "schaffer_function_n1.hpp" 36 | #include "schaffer_function_n2.hpp" 37 | #include "schaffer_function_n4.hpp" 38 | #include "schwefel_function.hpp" 39 | #include "sgd_test_function.hpp" 40 | #include "softmax_regression_function.hpp" 41 | #include "sparse_test_function.hpp" 42 | #include "sphere_function.hpp" 43 | #include "styblinski_tang_function.hpp" 44 | #include "three_hump_camel_function.hpp" 45 | #include "wood_function.hpp" 46 | #include "zdt/zdt1_function.hpp" 47 | #include "zdt/zdt2_function.hpp" 48 | #include "zdt/zdt3_function.hpp" 49 | #include "zdt/zdt4_function.hpp" 50 | #include "zdt/zdt6_function.hpp" 51 | #include "dtlz/dtlz1_function.hpp" 52 | #include "dtlz/dtlz2_function.hpp" 53 | #include "dtlz/dtlz3_function.hpp" 54 | #include "dtlz/dtlz4_function.hpp" 55 | #include "dtlz/dtlz5_function.hpp" 56 | #include "dtlz/dtlz6_function.hpp" 57 | #include "dtlz/dtlz7_function.hpp" 58 | #include "maf/maf1_function.hpp" 59 | #include "maf/maf2_function.hpp" 60 | #include "maf/maf3_function.hpp" 61 | #include "maf/maf4_function.hpp" 62 | #include "maf/maf5_function.hpp" 63 | #include "maf/maf6_function.hpp" 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /tests/lookahead_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file lookahead_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("LookaheadAdam_SphereFunction", "[Lookahead]", 23 | ENS_ALL_TEST_TYPES) 24 | { 25 | Lookahead<> optimizer(2.5, 5, 100000, 1e-5, NoDecay(), false, true); 26 | optimizer.BaseOptimizer().StepSize() = 0.5; 27 | optimizer.BaseOptimizer().BatchSize() = 2; 28 | optimizer.BaseOptimizer().Beta1() = 0.7; 29 | optimizer.BaseOptimizer().Tolerance() = Tolerances::Obj; 30 | // We allow a few trials. 31 | FunctionTest( 32 | optimizer, 33 | 10 * Tolerances::LargeObj, 34 | 10 * Tolerances::LargeCoord, 35 | 3); 36 | } 37 | 38 | TEMPLATE_TEST_CASE("LookaheadAdaGrad_SphereFunction", "[Lookahead]", 39 | ENS_ALL_TEST_TYPES) 40 | { 41 | AdaGrad adagrad(0.99, 1, 1e-8, 5, Tolerances::Obj / 100, true); 42 | adagrad.ResetPolicy() = false; 43 | Lookahead optimizer(adagrad, 2.5, 5, 5000000, 44 | Tolerances::Obj / 100, NoDecay(), false, true); 45 | FunctionTest( 46 | optimizer, 47 | 10 * Tolerances::LargeObj, 48 | 10 * Tolerances::LargeCoord, 49 | 3); 50 | } 51 | 52 | TEMPLATE_TEST_CASE("LookaheadAdam_LogisticRegressionFunction", "[Lookahead]", 53 | ENS_ALL_TEST_TYPES) 54 | { 55 | Adam adam(0.032, 4, 0.9, 0.999, Tolerances::Obj, 5, 56 | Tolerances::Obj / 10); 57 | adam.ResetPolicy() = false; 58 | Lookahead optimizer(adam, 12.5, 25, 100000, Tolerances::Obj, 59 | NoDecay(), false, true); 60 | LogisticRegressionFunctionTest( 61 | optimizer, 62 | Tolerances::LRTrainAcc, 63 | Tolerances::LRTestAcc); 64 | } 65 | -------------------------------------------------------------------------------- /include/ensmallen_bits/fbs/l1_penalty_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file l1_penalty_impl.hpp 3 | * @author Ryan Curtin 4 | * 5 | * An implementation of the proximal operator for the L1 penalty (also known as 6 | * the shrinkage operator). 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_FBS_L1_PENALTY_IMPL_HPP 14 | #define ENSMALLEN_FBS_L1_PENALTY_IMPL_HPP 15 | 16 | // In case it hasn't been included yet. 17 | #include "l1_penalty.hpp" 18 | 19 | namespace ens { 20 | 21 | inline L1Penalty::L1Penalty(const double lambda) : lambda(lambda) 22 | { 23 | // Nothing to do. 24 | } 25 | 26 | template 27 | typename MatType::elem_type L1Penalty::Evaluate(const MatType& coordinates) 28 | const 29 | { 30 | // Compute the L1 penalty. 31 | return norm(vectorise(coordinates), 1) * typename MatType::elem_type(lambda); 32 | } 33 | 34 | template 35 | void L1Penalty::ProximalStep(MatType& coordinates, 36 | const double stepSize) const 37 | { 38 | // Apply the backwards step coordinate-wise. If `MatType` is sparse, this 39 | // only applies to nonzero elements, which is just fine. 40 | typedef typename MatType::elem_type eT; 41 | 42 | // This is equivalent to the following .transform() implementation (which is 43 | // easier to read but will not work with Bandicoot): 44 | // 45 | //arma::Mat c2 = conv_to>::from(coordinates); 46 | //c2.transform([this, stepSize](eT val) { return (val > eT(0)) ? 47 | // (std::max(eT(0), val - eT(lambda * stepSize))) : 48 | // (std::min(eT(0), val + eT(lambda * stepSize))); }); 49 | // coordinates.transform([this, stepSize](eT val) { return (val > eT(0)) ? 50 | // (std::max(eT(0), val - eT(lambda * stepSize))) : 51 | // (std::min(eT(0), val + eT(lambda * stepSize))); }); 52 | // 53 | coordinates = sign(coordinates) % clamp( 54 | abs(coordinates) - eT(lambda * stepSize), eT(0), 55 | std::numeric_limits::max()); 56 | 57 | //coordinates.print("coordinates"); 58 | //c2.print("c2"); 59 | } 60 | 61 | } // namespace ens 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sarah/sarah_plus_update.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sarah_plus_update.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the SARAH+ update rule which provides an automatic and 6 | * adaptive choice of the inner loop size. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | #ifndef ENSMALLEN_SARAH_SARAH_PLUS_UPDATE_HPP 14 | #define ENSMALLEN_SARAH_SARAH_PLUS_UPDATE_HPP 15 | 16 | namespace ens { 17 | 18 | /** 19 | * SARAH+ provides an automatic and adaptive choice of the inner loop size. 20 | */ 21 | class SARAHPlusUpdate 22 | { 23 | public: 24 | /* 25 | * Construct the SARAH+ update policy. 26 | * 27 | * @param gamma Adaptive parameter for the inner loop. 28 | */ 29 | SARAHPlusUpdate(const double gamma = 0.125) : gamma(gamma) 30 | { 31 | /* Nothing to do here. */ 32 | } 33 | 34 | /** 35 | * Update step for SARAH+. The function parameters are updated in the negative 36 | * direction of the gradient. 37 | * 38 | * @param iterate Parameters that minimize the function. 39 | * @param v Unbiased estimator of the gradient. 40 | * @param gradient The current gradient matrix at time t. 41 | * @param gradient0 The old gradient matrix at time t - 1. 42 | * @param batchSize Batch size to be used for the given iteration. 43 | * @param stepSize Step size to be used for the given iteration. 44 | * @param vNorm The norm of the full gradient. 45 | */ 46 | template 47 | bool Update(MatType& iterate, 48 | GradType& v, 49 | const GradType& gradient, 50 | const GradType& gradient0, 51 | const size_t batchSize, 52 | const double stepSize, 53 | const double vNorm) 54 | { 55 | typedef typename MatType::elem_type ElemType; 56 | 57 | v += (gradient - gradient0) / (ElemType) batchSize; 58 | iterate -= ElemType(stepSize) * v; 59 | 60 | if (norm(v) <= ElemType(gamma * vNorm)) 61 | return true; 62 | 63 | return false; 64 | } 65 | 66 | private: 67 | //! Adaptive parameter for the inner loop. 68 | double gamma; 69 | }; 70 | 71 | } // namespace ens 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /tests/data/r10.txt: -------------------------------------------------------------------------------- 1 | 2.388061771742711503e+00 -7.998843946705324059e-01 0.000000000000000000e+00 -6.055455953973358785e-01 -8.978447303086828368e-01 -8.478705136616015992e-02 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 2 | -7.998843946705324059e-01 3.426145759425113901e+00 -1.638176200441511243e-01 0.000000000000000000e+00 -4.796520189361042075e-01 -7.078326540851567206e-01 -9.383403917548253093e-01 -3.366186799343440228e-01 0.000000000000000000e+00 0.000000000000000000e+00 3 | 0.000000000000000000e+00 -1.638176200441511243e-01 1.048508772479271434e+00 -8.846911524351203093e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 4 | -6.055455953973358785e-01 0.000000000000000000e+00 -8.846911524351203093e-01 2.023397783192608657e+00 -5.331610353601525798e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 5 | -8.978447303086828368e-01 -4.796520189361042075e-01 0.000000000000000000e+00 -5.331610353601525798e-01 2.697683931814737868e+00 -5.616838162810112722e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -2.253423309287867493e-01 6 | -8.478705136616015992e-02 -7.078326540851567206e-01 0.000000000000000000e+00 0.000000000000000000e+00 -5.616838162810112722e-01 2.784761835214184078e+00 -5.704882222059795005e-01 0.000000000000000000e+00 0.000000000000000000e+00 -8.599700912758765359e-01 7 | 0.000000000000000000e+00 -9.383403917548253093e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -5.704882222059795005e-01 1.890408204565853900e+00 -3.815795906050489794e-01 0.000000000000000000e+00 0.000000000000000000e+00 8 | 0.000000000000000000e+00 -3.366186799343440228e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -3.815795906050489794e-01 7.410893491077712447e-01 -2.289107856837824251e-02 0.000000000000000000e+00 9 | 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -2.289107856837824251e-02 2.289107856837824251e-02 0.000000000000000000e+00 10 | 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 -2.253423309287867493e-01 -8.599700912758765359e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.085312422204663285e+00 11 | -------------------------------------------------------------------------------- /.appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | ARMADILLO_DOWNLOAD: "https://sourceforge.net/projects/arma/files/armadillo-10.8.2.tar.xz" 3 | BLAS_LIBRARY: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a" 4 | BLAS_LIBRARY_DLL: "%APPVEYOR_BUILD_FOLDER%/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll" 5 | 6 | matrix: 7 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 8 | VSVER: Visual Studio 15 2017 Win64 9 | MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe 10 | 11 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 12 | VSVER: Visual Studio 16 2019 13 | MSBUILD: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe 14 | 15 | - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 16 | VSVER: Visual Studio 17 2022 17 | MSBUILD: C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Current\Bin\MSBuild.exe 18 | 19 | configuration: Release 20 | 21 | install: 22 | - ps: nuget install OpenBLAS -o "${env:APPVEYOR_BUILD_FOLDER}" 23 | 24 | build_script: 25 | # First, download and build Armadillo. 26 | - cd .. 27 | - appveyor DownloadFile %ARMADILLO_DOWNLOAD% -FileName armadillo.tar.xz 28 | - 7z x armadillo.tar.xz -so -txz | 7z x -si -ttar > nul 29 | - cd armadillo-10.8.2 && mkdir build && cd build 30 | - > 31 | cmake -G "%VSVER%" 32 | -DBLAS_LIBRARY:FILEPATH=%BLAS_LIBRARY% 33 | -DLAPACK_LIBRARY:FILEPATH=%BLAS_LIBRARY% 34 | -DCMAKE_PREFIX:FILEPATH="%APPVEYOR_BUILD_FOLDER%/armadillo" 35 | -DBUILD_SHARED_LIBS=OFF 36 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 37 | -DCMAKE_BUILD_TYPE=Release .. 38 | - > 39 | "%MSBUILD%" "armadillo.sln" 40 | /m /verbosity:quiet /p:Configuration=Release;Platform=x64 41 | - cd ../.. 42 | 43 | # Now build ensmallen. 44 | - cd ensmallen && mkdir build && cd build 45 | - > 46 | cmake -G "%VSVER%" 47 | -DARMADILLO_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%/../armadillo-10.8.2/include/ 48 | -DARMADILLO_LIBRARIES=%BLAS_LIBRARY% 49 | -DLAPACK_LIBRARY=%BLAS_LIBRARY% 50 | -DBLAS_LIBRARY=%BLAS_LIBRARY% 51 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 52 | -DCMAKE_BUILD_TYPE=Release .. 53 | - > 54 | "%MSBUILD%" "ensmallen.sln" 55 | /m /verbosity:minimal /nologo /p:BuildInParallel=false 56 | 57 | # Run tests after copying libraries. 58 | - ps: cp C:\projects\ensmallen\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* C:\projects\ensmallen\build\ 59 | - ctest -C Release -V --output-on-failure . 60 | -------------------------------------------------------------------------------- /tests/snapshot_ensembles.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file snapshot_ensembles.cpp 3 | * @author Marcus Edel 4 | * @author Conrad Sanderson 5 | * 6 | * ensmallen is free software; you may redistribute it and/or modify it under 7 | * the terms of the 3-clause BSD license. You should have received a copy of 8 | * the 3-clause BSD license along with ensmallen. If not, see 9 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 10 | */ 11 | #if defined(ENS_USE_COOT) 12 | #include 13 | #include 14 | #endif 15 | #include 16 | #include "catch.hpp" 17 | #include "test_function_tools.hpp" 18 | #include "test_types.hpp" 19 | 20 | using namespace ens; 21 | using namespace ens::test; 22 | 23 | /* 24 | * Test that the step size resets after a specified number of epochs. 25 | */ 26 | TEMPLATE_TEST_CASE("SnapshotEnsembles_ResetTest", "[SnapshotEnsembles]", 27 | ENS_ALL_CPU_TEST_TYPES) 28 | { 29 | const double stepSize = 0.5; 30 | TestType iterate; 31 | 32 | // Now run cyclical decay policy with a couple of multiplicators and initial 33 | // restarts. 34 | for (size_t restart = 5; restart < 100; restart += 10) 35 | { 36 | for (size_t mult = 2; mult < 5; ++mult) 37 | { 38 | double epochStepSize = stepSize; 39 | 40 | SnapshotEnsembles snapshotEnsembles(restart, 41 | double(mult), stepSize, 1000, 2); 42 | SnapshotEnsembles::Policy p(snapshotEnsembles); 43 | 44 | snapshotEnsembles.EpochBatches() = 10 / (double)1000; 45 | // Create all restart epochs. 46 | arma::Col nextRestart(1000 / 10 / mult); 47 | nextRestart(0) = restart; 48 | for (size_t j = 1; j < nextRestart.n_elem; ++j) 49 | nextRestart(j) = nextRestart(j - 1) * mult; 50 | 51 | for (size_t i = 0; i < 1000; ++i) 52 | { 53 | p.Update(iterate, epochStepSize, iterate); 54 | if (i <= restart || arma::accu(arma::find(nextRestart == i)) > 0) 55 | { 56 | REQUIRE(epochStepSize == stepSize); 57 | } 58 | } 59 | 60 | REQUIRE(p.Snapshots().size() == 2); 61 | } 62 | } 63 | } 64 | 65 | TEMPLATE_TEST_CASE("SnapshotSGDR_LogisticRegressionFunction", "[SnapshotSGDR]", 66 | ENS_ALL_TEST_TYPES) 67 | { 68 | // Run SGDR with snapshot ensembles on a couple of batch sizes. 69 | for (size_t batchSize = 10; batchSize < 50; batchSize += 10) 70 | { 71 | SnapshotSGDR<> sgdr(50, 2.0, batchSize, 0.005 * batchSize, 10000, 1e-3); 72 | LogisticRegressionFunctionTest>(sgdr); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /include/ensmallen_bits/moead/weight_init_policies/bbs_init.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file bbs_init.hpp 3 | * @author Nanubala Gnana Sai 4 | * 5 | * The Bayesian Bootstrap (BBS) method of Weight Initialization. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_MOEAD_BBS_HPP 13 | #define ENSMALLEN_MOEAD_BBS_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * The Bayesian Bootstrap method for initializing weights. Samples are randomly picked from uniform 19 | * distribution followed by sorting and finding adjacent difference. This gives you a list of 20 | * numbers which is guaranteed to sum up to 1. 21 | * 22 | * @code 23 | * @article{rubin1981bayesian, 24 | * title={The bayesian bootstrap}, 25 | * author={Rubin, Donald B}, 26 | * journal={The annals of statistics}, 27 | * pages={130--134}, 28 | * year={1981}, 29 | * @endcode 30 | * 31 | */ 32 | class BayesianBootstrap 33 | { 34 | public: 35 | /** 36 | * Constructor for Bayesian Bootstrap policy. 37 | */ 38 | BayesianBootstrap() 39 | { 40 | /* Nothing to do. */ 41 | } 42 | 43 | /** 44 | * Generate the reference direction matrix. 45 | * 46 | * @tparam MatType The type of the matrix used for constructing weights. 47 | * @param numObjectives The dimensionality of objective space. 48 | * @param numPoints The number of reference directions requested. 49 | * @param epsilon Handle numerical stability after weight initialization. 50 | */ 51 | template 52 | MatType Generate(const size_t numObjectives, 53 | const size_t numPoints, 54 | const double epsilon) 55 | { 56 | typedef typename ForwardType::bvec VecType; 57 | 58 | MatType weights(numObjectives, numPoints); 59 | for (size_t pointIdx = 0; pointIdx < numPoints; ++pointIdx) 60 | { 61 | VecType referenceDirection(numObjectives + 1, 62 | GetFillType::randu); 63 | referenceDirection(0) = 0; 64 | referenceDirection(numObjectives) = 1; 65 | referenceDirection = sort(referenceDirection); 66 | referenceDirection = diff(referenceDirection); 67 | weights.col(pointIdx) = std::move(referenceDirection) + epsilon; 68 | } 69 | 70 | return weights; 71 | } 72 | }; 73 | 74 | } // namespace ens 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/rosenbrock_wood_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file rosenbrock_wood_function_impl.hpp 3 | * @author Ryan Curtin 4 | * @author Marcus Edel 5 | * 6 | * Implementation of the Rosenbrock-Wood function. 7 | * 8 | * ensmallen is free software; you may redistribute it and/or modify it under 9 | * the terms of the 3-clause BSD license. You should have received a copy of 10 | * the 3-clause BSD license along with ensmallen. If not, see 11 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 12 | */ 13 | 14 | #ifndef ENSMALLEN_PROBLEMS_ROSENBROCK_WOOD_FUNCTION_IMPL_HPP 15 | #define ENSMALLEN_PROBLEMS_ROSENBROCK_WOOD_FUNCTION_IMPL_HPP 16 | 17 | // In case it hasn't been included yet. 18 | #include "rosenbrock_wood_function.hpp" 19 | 20 | namespace ens { 21 | namespace test { 22 | 23 | inline RosenbrockWoodFunction::RosenbrockWoodFunction() : rf(4), wf() 24 | { 25 | initialPoint.set_size(4, 2); 26 | initialPoint.col(0) = rf.GetInitialPoint(); 27 | initialPoint.col(1) = wf.GetInitialPoint(); 28 | } 29 | 30 | inline void RosenbrockWoodFunction::Shuffle() { /* Nothing to do here */ } 31 | 32 | template 33 | typename MatType::elem_type RosenbrockWoodFunction::Evaluate( 34 | const MatType& coordinates, 35 | const size_t /* begin */, 36 | const size_t /* batchSize */) const 37 | { 38 | return rf.Evaluate(coordinates.col(0)) + wf.Evaluate(coordinates.col(1)); 39 | } 40 | 41 | template 42 | typename MatType::elem_type RosenbrockWoodFunction::Evaluate( 43 | const MatType& coordinates) const 44 | { 45 | return Evaluate(coordinates, 0, NumFunctions()); 46 | } 47 | 48 | template 49 | inline void RosenbrockWoodFunction::Gradient(const MatType& coordinates, 50 | const size_t /* begin */, 51 | GradType& gradient, 52 | const size_t /* batchSize */) const 53 | { 54 | gradient.set_size(4, 2); 55 | 56 | MatType grf(4, 1); 57 | MatType gwf(4, 1); 58 | 59 | rf.Gradient(coordinates.col(0), grf); 60 | wf.Gradient(coordinates.col(1), gwf); 61 | 62 | gradient.col(0) = grf; 63 | gradient.col(1) = gwf; 64 | } 65 | 66 | template 67 | inline void RosenbrockWoodFunction::Gradient(const MatType& coordinates, 68 | GradType& gradient) const 69 | { 70 | Gradient(coordinates, 0, gradient, 1); 71 | } 72 | 73 | } // namespace test 74 | } // namespace ens 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /tests/active_cmaes_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file active_cmaes_test.cpp 3 | * @author Suvarsha Chennareddy 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | 11 | #include 12 | #include "catch.hpp" 13 | #include "test_function_tools.hpp" 14 | #include "test_types.hpp" 15 | 16 | using namespace ens; 17 | using namespace ens::test; 18 | 19 | /** 20 | * Run Active CMA-ES with the full selection policy on Rosenbrock function and 21 | * make sure the results are acceptable. 22 | */ 23 | TEMPLATE_TEST_CASE("ActiveCMAESRosenbrockFunctionTest", "[ActiveCMAES]", 24 | ENS_TEST_TYPES) 25 | { 26 | BoundaryBoxConstraint b(0, 10); 27 | ActiveCMAES> 28 | activecmaes(0, b, 1, 0, Tolerances::Obj); 29 | activecmaes.StepSize() = 0.01; 30 | FunctionTest(activecmaes, 31 | 100 * Tolerances::LargeObj, 32 | 10 * Tolerances::LargeCoord, 33 | 10); 34 | } 35 | 36 | /** 37 | * Run Active CMA-ES with the random selection policy on Rosenbrock function and 38 | * make sure the results are acceptable. 39 | */ 40 | TEMPLATE_TEST_CASE("ApproxActiveCMAESRosenbrockFunctionTest", "[ActiveCMAES]", 41 | ENS_TEST_TYPES) 42 | { 43 | BoundaryBoxConstraint b(0, 10); 44 | ApproxActiveCMAES> 45 | activecmaes(2048, b, 1, 0, Tolerances::Obj / 100); 46 | activecmaes.StepSize() = 0.01; 47 | FunctionTest(activecmaes, 48 | 100 * Tolerances::LargeObj, 49 | 10 * Tolerances::LargeCoord, 50 | 10); 51 | } 52 | 53 | /** 54 | * Run Active CMA-ES with the random selection and empty transformation policies 55 | * on Rosenbrock function and make sure the results are acceptable. 56 | * Use arma::fmat. 57 | */ 58 | TEMPLATE_TEST_CASE("ApproxActiveCMAESEmptyTransformationLogisticRegressionTest", 59 | "[ActiveCMAES]", ENS_TEST_TYPES) 60 | { 61 | ApproxActiveCMAES> 62 | activecmaes(0, EmptyTransformation(), 16, 0, 63 | 10 * Tolerances::Obj); 64 | 65 | activecmaes.StepSize() = 0.55; 66 | LogisticRegressionFunctionTest(activecmaes, 67 | Tolerances::LRTrainAcc, Tolerances::LRTestAcc, 5); 68 | } 69 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cd/descent_policies/random_descent.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file random_descent.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Random descent policy for Stochastic Coordinate Descent (SCD). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SCD_DESCENT_POLICIES_RANDOM_HPP 13 | #define ENSMALLEN_SCD_DESCENT_POLICIES_RANDOM_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Random descent policy for Stochastic Coordinate Descent(SCD). This 19 | * descent scheme picks a the co-ordinate for the descent uniformly randomly. 20 | * 21 | * For more information, see the following. 22 | * @code 23 | * @inproceedings{ShalevShwartz2009, 24 | * author = {Shalev-Shwartz, Shai and Tewari, Ambuj}, 25 | * title = {Stochastic Methods for L1 Regularized Loss Minimization}, 26 | * booktitle = {Proceedings of the 26th Annual International Conference on 27 | * Machine Learning}, 28 | * series = {ICML '09}, 29 | * year = {2009}, 30 | * isbn = {978-1-60558-516-1} 31 | * } 32 | * @endcode 33 | */ 34 | class RandomDescent 35 | { 36 | public: 37 | /** 38 | * The DescentFeature method is used to get the descent coordinate for the 39 | * current iteration of the SCD optimizer. For more information regarding the 40 | * interface of this policy with the optimizer, have a look at the SCD 41 | * implementation. 42 | * 43 | * @tparam ResolvableFunctionType The type of the function to be optimized. 44 | * @param iteration The iteration number for which the feature is to be 45 | * obtained. 46 | * @param iterate The current value of the decision variable. 47 | * @param function The function to be optimized. 48 | * @return The index of the coordinate to be descended. 49 | */ 50 | template 51 | static size_t DescentFeature(const size_t /* iteration */, 52 | const MatType& /* iterate */, 53 | const ResolvableFunctionType& function) 54 | { 55 | // return randi( 56 | // arma::distr_param(0, function.NumFeatures() - 1)); 57 | 58 | return arma::as_scalar(arma::randi( 59 | 1, arma::distr_param(0, function.NumFeatures() - 1))); 60 | } 61 | }; 62 | 63 | } // namespace ens 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /include/ensmallen_bits/sgd/update_policies/vanilla_update.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file vanilla_update.hpp 3 | * @author Arun Reddy 4 | * 5 | * Vanilla update for Stochastic Gradient Descent. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SGD_EMPTY_UPDATE_HPP 13 | #define ENSMALLEN_SGD_EMPTY_UPDATE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Vanilla update policy for Stochastic Gradient Descent (SGD). The following 19 | * update scheme is used to update SGD in every iteration: 20 | * 21 | * \f[ 22 | * A_{j + 1} = A_j + \alpha \nabla f_i(A) 23 | * \f] 24 | * 25 | * where \f$ \alpha \f$ is a parameter which specifies the step size. \f$ i \f$ 26 | * is chosen according to \f$ j \f$ (the iteration number). 27 | */ 28 | class VanillaUpdate 29 | { 30 | public: 31 | /** 32 | * The UpdatePolicyType policy classes must contain an internal 'Policy' 33 | * template class with two template arguments: MatType and GradType. This is 34 | * instantiated at the start of the optimization. 35 | */ 36 | template 37 | class Policy 38 | { 39 | public: 40 | typedef typename MatType::elem_type ElemType; 41 | 42 | /** 43 | * This is called by the optimizer method before the start of the iteration 44 | * update process. The vanilla update doesn't initialize anything. 45 | * 46 | * @param parent Instantiated parent class. 47 | * @param rows Number of rows in the gradient matrix. 48 | * @param cols Number of columns in the gradient matrix. 49 | */ 50 | Policy(const VanillaUpdate& /* parent */, 51 | const size_t /* rows */, 52 | const size_t /* cols */) 53 | { /* Do nothing. */ } 54 | 55 | /** 56 | * Update step for SGD. The function parameters are updated in the negative 57 | * direction of the gradient. 58 | * 59 | * @param iterate Parameters that minimize the function. 60 | * @param stepSize Step size to be used for the given iteration. 61 | * @param gradient The gradient matrix. 62 | */ 63 | void Update(MatType& iterate, 64 | const double stepSize, 65 | const GradType& gradient) 66 | { 67 | // Perform the vanilla SGD update. 68 | iterate -= ElemType(stepSize) * gradient; 69 | } 70 | }; 71 | }; 72 | 73 | } // namespace ens 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /tests/proximal_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file proximal_test.cpp 3 | * @author Chenzhe Diao 4 | * @author Marcus Edel 5 | * @author Conrad Sanderson 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | 13 | #include 14 | #include "catch.hpp" 15 | 16 | using namespace ens; 17 | using namespace arma; 18 | using namespace ens::test; 19 | 20 | /** 21 | * Approximate vector using a vector with l1 norm small than or equal to tau. 22 | */ 23 | TEST_CASE("ProjectToL1", "[ProximalTest]") 24 | { 25 | int D = 100; // Dimension of the problem. 26 | 27 | // Norm of L1 ball. 28 | double tau1 = 1.5; 29 | double tau2 = 0.5; 30 | 31 | // Vector to be projected, with unit l1 norm. 32 | vec v = randu(D); 33 | v = normalise(v, 1); 34 | 35 | // v is inside the l1 ball, so the projection will not change v. 36 | vec v1 = v; 37 | Proximal::ProjectToL1Ball(v1, tau1); 38 | REQUIRE(norm(v - v1, 2) == Approx(0.0).margin(1e-10)); 39 | 40 | // v is outside the l1 ball, so the projection should find the closest. 41 | vec v2 = v; 42 | Proximal::ProjectToL1Ball(v2, tau2); 43 | double distance = norm(v2 - v, 2); 44 | for (size_t i = 1; i < 1000; i++) 45 | { 46 | // Randomly generate a vector on the surface of the l1 ball with norm tau2. 47 | vec vSurface = randu(D); 48 | vSurface = tau2 * normalise(vSurface, 1); 49 | 50 | double distanceNew = norm(vSurface - v, 2); 51 | 52 | REQUIRE(distanceNew >= distance); 53 | } 54 | } 55 | 56 | /** 57 | * Approximate a vector with a tau-sparse vector. 58 | */ 59 | TEST_CASE("ProjectToL0", "[ProximalTest]") 60 | { 61 | int D = 100; // Dimension of the problem. 62 | int tau = 25; // Sparsity requirement. 63 | 64 | // Vector to be projected. 65 | vec v = randn(D); 66 | 67 | vec v0 = v; 68 | Proximal::ProjectToL0Ball(v0, tau); 69 | double distance = norm(v0 - v, 2); 70 | 71 | for (size_t i = 1; i < 1000; i++) 72 | { 73 | // Randomly find a subset of the support of v, generate a tau-sparse 74 | // vector by restricting v to this support. 75 | uvec indices = ens::linspace(0, D - 1, D); 76 | indices = shuffle(indices); 77 | indices = indices.head(tau); 78 | vec vNew = zeros(D); 79 | vNew.elem(indices) = v.elem(indices); 80 | 81 | double distanceNew = norm(v - vNew, 2); 82 | REQUIRE(distanceNew >= distance); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /include/ensmallen_bits/callbacks/store_best_coordinates.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file store_best_coordinates.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the store best coordinates callback function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_CALLBACKS_STORE_BEST_COORDINATES_HPP 13 | #define ENSMALLEN_CALLBACKS_STORE_BEST_COORDINATES_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Store best coordinates function, based on the Evaluate callback function. 19 | * 20 | * @tparam MatType Type of the model coordinates (arma::colvec, arma::mat, 21 | * arma::sp_mat or arma::cube). 22 | */ 23 | template 24 | class StoreBestCoordinates 25 | { 26 | public: 27 | /** 28 | * Set up the store best model class, which keeps the best-performing 29 | * coordinates and objective. 30 | */ 31 | StoreBestCoordinates() : bestObjective(std::numeric_limits::max()) 32 | { /* Nothing to do here. */ } 33 | 34 | /** 35 | * Callback function called after any call to Evaluate(). 36 | * 37 | * @param optimizer The optimizer used to update the function. 38 | * @param function Function to optimize. 39 | * @param coordinates Starting point. 40 | * @param objective Objective value of the current point. 41 | */ 42 | template 43 | bool Evaluate(OptimizerType& /* optimizer */, 44 | FunctionType& /* function */, 45 | const MatType& coordinates, 46 | const double objective) 47 | { 48 | if (objective < bestObjective) 49 | { 50 | bestObjective = objective; 51 | bestCoordinates = coordinates; 52 | } 53 | return false; 54 | } 55 | 56 | //! Get the best coordinates. 57 | ModelMatType const& BestCoordinates() const { return bestCoordinates; } 58 | //! Modify the best coordinates. 59 | ModelMatType& BestCoordinates() { return bestCoordinates; } 60 | 61 | //! Get the best objective. 62 | double const& BestObjective() const { return bestObjective; } 63 | //! Modify the best objective. 64 | double& BestObjective() { return bestObjective; } 65 | 66 | private: 67 | //! Locally-stored best objective. 68 | double bestObjective; 69 | 70 | //! Locally-stored best model coordinates. 71 | ModelMatType bestCoordinates; 72 | }; 73 | 74 | } // namespace ens 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/schaffer_function_n4.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file schaffer_function_n4.hpp 3 | * @author Suryoday Basak 4 | * 5 | * Definition of Schaffer function N.4. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_HPP 13 | #define ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_HPP 14 | 15 | namespace ens { 16 | namespace test { 17 | 18 | /** 19 | * The Schaffer function N.4, defined by 20 | * 21 | * \f[ 22 | * f(x1, x2) = 0.5 + (cos^2(sin(|x1^2 - x2^2|)) - 0.5) / 23 | * (1 + 0.001 * (x1^2 + x2^2))^2 24 | * \f] 25 | * 26 | * This should optimize to f(x1, x2) = 0.292579, at (x1, x2) = [0, 1.25313], or 27 | * (x1, x2) = [0, -1.25313]. 28 | * 29 | * For more information, please refer to: 30 | * 31 | * @code 32 | * @misc{LevyFunction, 33 | * URL = {http://benchmarkfcns.xyz/benchmarkfcns/schaffern4fcn.html} 34 | * } 35 | * @endcode 36 | */ 37 | class SchafferFunctionN4 38 | { 39 | public: 40 | //! Initialize the SchafferFunctionN4. 41 | SchafferFunctionN4(); 42 | 43 | /** 44 | * Shuffle the order of function visitation. This may be called by the 45 | * optimizer. 46 | */ 47 | void Shuffle(); 48 | 49 | //! Return 1 (the number of functions). 50 | size_t NumFunctions() const { return 1; } 51 | 52 | //! Get the starting point. 53 | template 54 | MatType GetInitialPoint() const { return MatType("-5; 5"); } 55 | 56 | /** 57 | * Evaluate a function for a particular batch-size. 58 | * 59 | * @param coordinates The function coordinates. 60 | * @param begin The first function. 61 | * @param batchSize Number of points to process. 62 | */ 63 | template 64 | typename MatType::elem_type Evaluate(const MatType& coordinates, 65 | const size_t begin, 66 | const size_t batchSize) const; 67 | 68 | /** 69 | * Evaluate a function with the given coordinates. 70 | * 71 | * @param coordinates The function coordinates. 72 | */ 73 | template 74 | typename MatType::elem_type Evaluate(const MatType& coordinates) const; 75 | }; 76 | 77 | } // namespace test 78 | } // namespace ens 79 | 80 | // Include implementation. 81 | #include "schaffer_function_n4_impl.hpp" 82 | 83 | #endif // ENSMALLEN_PROBLEMS_SCHAFFER_FUNCTION_N4_HPP 84 | -------------------------------------------------------------------------------- /tests/ada_bound_test.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ada_bound_test.cpp 3 | * @author Marcus Edel 4 | * 5 | * ensmallen is free software; you may redistribute it and/or modify it under 6 | * the terms of the 3-clause BSD license. You should have received a copy of 7 | * the 3-clause BSD license along with ensmallen. If not, see 8 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 9 | */ 10 | #if defined(ENS_USE_COOT) 11 | #include 12 | #include 13 | #endif 14 | #include 15 | #include "catch.hpp" 16 | #include "test_function_tools.hpp" 17 | #include "test_types.hpp" 18 | 19 | using namespace ens; 20 | using namespace ens::test; 21 | 22 | TEMPLATE_TEST_CASE("AdaBound_SphereFunction", "[AdaBound]", ENS_ALL_TEST_TYPES, 23 | ENS_SPARSE_TEST_TYPES) 24 | { 25 | AdaBound optimizer(0.002, 2, 0.1, 1e-3, 0.9, 0.999, 1e-8, 500000, 26 | 1e-3, false); 27 | FunctionTest( 28 | optimizer, 29 | 10 * Tolerances::LargeObj, 30 | 10 * Tolerances::LargeCoord); 31 | } 32 | 33 | TEMPLATE_TEST_CASE("AMSBound_SphereFunction", "[AdaBound]", ENS_ALL_TEST_TYPES, 34 | ENS_SPARSE_TEST_TYPES) 35 | { 36 | AMSBound optimizer(0.002, 2, 0.1, 1e-3, 0.9, 0.999, 1e-8, 500000, 37 | 1e-3, false); 38 | FunctionTest( 39 | optimizer, 40 | 10 * Tolerances::LargeObj, 41 | 10 * Tolerances::LargeCoord); 42 | } 43 | 44 | TEMPLATE_TEST_CASE("AdaBound_SphereFunctionSpMatDenseGradient", "[AdaBound]", 45 | ENS_SPARSE_TEST_TYPES) 46 | { 47 | typedef typename TestType::elem_type ElemType; 48 | 49 | SphereFunction f(2); 50 | AdaBound optimizer(0.002, 2, 0.1, 1e-3, 0.9, 0.999, 1e-8, 500000, 51 | 1e-3, false); 52 | 53 | TestType coordinates = arma::conv_to::from( 54 | f.GetInitialPoint()); 55 | optimizer.Optimize >( 56 | f, coordinates); 57 | 58 | REQUIRE(coordinates(0) == Approx(0.0).margin(0.1)); 59 | REQUIRE(coordinates(1) == Approx(0.0).margin(0.1)); 60 | } 61 | 62 | TEMPLATE_TEST_CASE("AMSBound_SphereFunctionSpMatDenseGradient", "[AdaBound]", 63 | ENS_SPARSE_TEST_TYPES) 64 | { 65 | typedef typename TestType::elem_type ElemType; 66 | 67 | SphereFunction f(2); 68 | AMSBound optimizer(0.002, 2, 0.1, 1e-3, 0.9, 0.999, 1e-8, 500000, 69 | 1e-3, false); 70 | 71 | arma::sp_mat coordinates = f.GetInitialPoint(); 72 | optimizer.Optimize >( 73 | f, coordinates); 74 | 75 | REQUIRE(coordinates(0) == Approx(0.0).margin(0.1)); 76 | REQUIRE(coordinates(1) == Approx(0.0).margin(0.1)); 77 | } 78 | -------------------------------------------------------------------------------- /include/ensmallen_bits/svrg/svrg_update.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file svrg_update.hpp 3 | * @author Marcus Edel 4 | * 5 | * Vanilla update for stochastic variance reduced gradient (SVRG). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SVRG_SVRG_UPDATE_HPP 13 | #define ENSMALLEN_SVRG_SVRG_UPDATE_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Vanilla update policy for Stochastic variance reduced gradient (SVRG). 19 | * The following update scheme is used to update SGD in every iteration: 20 | */ 21 | class SVRGUpdate 22 | { 23 | public: 24 | /** 25 | * The UpdatePolicyType policy classes must contain an internal 'Policy' 26 | * template class with two template arguments: MatType and GradType. This is 27 | * instantiated at the start of the optimization. 28 | */ 29 | template 30 | class Policy 31 | { 32 | public: 33 | /** 34 | * This is called by the optimizer method before the start of the iteration 35 | * update process. 36 | * 37 | * @param parent Instantiated parent class. 38 | * @param rows Number of rows in the gradient matrix. 39 | * @param cols Number of columns in the gradient matrix. 40 | */ 41 | Policy(SVRGUpdate& /* parent */, 42 | const size_t /* rows */, 43 | const size_t /* cols */) 44 | { /* Do nothing. */ } 45 | 46 | /** 47 | * Update step for SVRG. The function parameters are updated in the negative 48 | * direction of the gradient. 49 | * 50 | * @param iterate Parameters that minimize the function. 51 | * @param fullGradient The computed full gradient. 52 | * @param gradient The current gradient matrix at time t. 53 | * @param gradient0 The old gradient matrix at time t - 1. 54 | * @param batchSize Batch size to be used for the given iteration. 55 | * @param stepSize Step size to be used for the given iteration. 56 | */ 57 | void Update(MatType& iterate, 58 | const GradType& fullGradient, 59 | const GradType& gradient, 60 | const GradType& gradient0, 61 | const size_t batchSize, 62 | const double stepSize) 63 | { 64 | // Perform the vanilla SVRG update. 65 | iterate -= typename MatType::elem_type(stepSize) * 66 | (fullGradient + (gradient - gradient0) / batchSize); 67 | } 68 | }; 69 | }; 70 | 71 | } // namespace ens 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/booth_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file booth_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Booth function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_BOOTH_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_BOOTH_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "booth_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline BoothFunction::BoothFunction() { /* Nothing to do here */ } 22 | 23 | inline void BoothFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type BoothFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const ElemType objective = std::pow(x1 + 2 * x2 - 7, ElemType(2)) + 39 | std::pow(2 * x1 + x2 - 5, ElemType(2)); 40 | 41 | return objective; 42 | } 43 | 44 | template 45 | typename MatType::elem_type BoothFunction::Evaluate( 46 | const MatType& coordinates) const 47 | { 48 | return Evaluate(coordinates, 0, NumFunctions()); 49 | } 50 | 51 | template 52 | inline void BoothFunction::Gradient(const MatType& coordinates, 53 | const size_t /* begin */, 54 | GradType& gradient, 55 | const size_t /* batchSize */) const 56 | { 57 | // Convenience typedef. 58 | typedef typename MatType::elem_type ElemType; 59 | 60 | // For convenience; we assume these temporaries will be optimized out. 61 | const ElemType x1 = coordinates(0); 62 | const ElemType x2 = coordinates(1); 63 | 64 | gradient.set_size(2, 1); 65 | gradient(0) = 10 * x1 + 8 * x2 - 34; 66 | gradient(1) = 8 * x1 + 10 * x2 - 38; 67 | } 68 | 69 | template 70 | inline void BoothFunction::Gradient(const MatType& coordinates, 71 | GradType& gradient) 72 | { 73 | Gradient(coordinates, 0, gradient, 1); 74 | } 75 | 76 | } // namespace test 77 | } // namespace ens 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/sgd_test_function.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sgd_test_function.hpp 3 | * @author Ryan Curtin 4 | * 5 | * Very simple test function for SGD. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SGD_TEST_FUNCTION_HPP 13 | #define ENSMALLEN_PROBLEMS_SGD_TEST_FUNCTION_HPP 14 | 15 | namespace ens { 16 | namespace test { 17 | 18 | //! Very, very simple test function which is the composite of three other 19 | //! functions. The gradient is not very steep far away from the optimum, so a 20 | //! larger step size may be required to optimize it in a reasonable number of 21 | //! iterations. 22 | class SGDTestFunction 23 | { 24 | private: 25 | arma::Col visitationOrder; 26 | 27 | public: 28 | //! Initialize the SGDTestFunction. 29 | SGDTestFunction(); 30 | 31 | /** 32 | * Shuffle the order of function visitation. This may be called by the 33 | * optimizer. 34 | */ 35 | void Shuffle(); 36 | 37 | //! Return 3 (the number of functions). 38 | size_t NumFunctions() const { return 3; } 39 | 40 | //! Evaluate a function for a particular batch-size. 41 | template 42 | typename MatType::elem_type Evaluate(const MatType& coordinates, 43 | const size_t begin, 44 | const size_t batchSize) const; 45 | 46 | //! Evaluate the gradient of a function for a particular batch-size 47 | template 48 | void Gradient(const MatType& coordinates, 49 | const size_t begin, 50 | GradType& gradient, 51 | const size_t batchSize) const; 52 | 53 | // Note: GetInitialPoint(), GetFinalPoint(), and GetFinalObjective() are not 54 | // required for using ensmallen to optimize this function! They are 55 | // specifically used as a convenience just for ensmallen's testing 56 | // infrastructure. 57 | 58 | //! Get the starting point. 59 | template 60 | MatType GetInitialPoint() const { return MatType("6; -45.6; 6.2"); } 61 | 62 | //! Get the final point. 63 | template 64 | MatType GetFinalPoint() const { return MatType("0.0; 0.0; 0.0"); } 65 | 66 | //! Get the final objective. 67 | double GetFinalObjective() const { return -1.0; } 68 | }; 69 | 70 | } // namespace test 71 | } // namespace ens 72 | 73 | // Include implementation. 74 | #include "sgd_test_function_impl.hpp" 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/sphere_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sphere_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Sphere function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_SPHERE_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_SPHERE_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "sphere_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline SphereFunction::SphereFunction(const size_t n) : 22 | n(n), 23 | visitationOrder(arma::linspace >(0, n - 1, n)) 24 | { 25 | initialPoint.set_size(n, 1); 26 | 27 | for (size_t i = 0; i < n; ++i) // Set to [-5 5 -5 5 -5 5...]. 28 | { 29 | if (i % 2 == 1) 30 | initialPoint(i) = 5; 31 | else 32 | initialPoint(i) = -5; 33 | } 34 | } 35 | 36 | inline void SphereFunction::Shuffle() 37 | { 38 | visitationOrder = arma::shuffle( 39 | arma::linspace >(0, n - 1, n)); 40 | } 41 | 42 | template 43 | typename MatType::elem_type SphereFunction::Evaluate( 44 | const MatType& coordinates, 45 | const size_t begin, 46 | const size_t batchSize) const 47 | { 48 | typedef typename MatType::elem_type ElemType; 49 | 50 | ElemType objective = 0; 51 | for (size_t j = begin; j < begin + batchSize; ++j) 52 | { 53 | const size_t p = visitationOrder[j]; 54 | objective += std::pow(coordinates(p), ElemType(2)); 55 | } 56 | 57 | return objective; 58 | } 59 | 60 | template 61 | typename MatType::elem_type SphereFunction::Evaluate( 62 | const MatType& coordinates) const 63 | { 64 | return Evaluate(coordinates, 0, NumFunctions()); 65 | } 66 | 67 | template 68 | void SphereFunction::Gradient(const MatType& coordinates, 69 | const size_t begin, 70 | GradType& gradient, 71 | const size_t batchSize) const 72 | { 73 | gradient.zeros(n, 1); 74 | 75 | for (size_t j = begin; j < begin + batchSize; ++j) 76 | { 77 | const size_t p = visitationOrder[j]; 78 | gradient(p) += 2 * coordinates[p]; 79 | } 80 | } 81 | 82 | template 83 | void SphereFunction::Gradient(const MatType& coordinates, 84 | GradType& gradient) 85 | { 86 | Gradient(coordinates, 0, gradient, NumFunctions()); 87 | } 88 | 89 | } // namespace test 90 | } // namespace ens 91 | 92 | #endif 93 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/matyas_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file matyas_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the Matyas function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_MATYAS_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_MATYAS_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "matyas_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline MatyasFunction::MatyasFunction() { /* Nothing to do here */ } 22 | 23 | inline void MatyasFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type MatyasFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | // Convenience typedef. 32 | typedef typename MatType::elem_type ElemType; 33 | 34 | // For convenience; we assume these temporaries will be optimized out. 35 | const ElemType x1 = coordinates(0); 36 | const ElemType x2 = coordinates(1); 37 | 38 | const ElemType objective = ElemType(0.26) * (std::pow(x1, ElemType(2)) + 39 | std::pow(x2, ElemType(2))) - ElemType(0.48) * x1 * x2; 40 | 41 | return objective; 42 | } 43 | 44 | template 45 | typename MatType::elem_type MatyasFunction::Evaluate( 46 | const MatType& coordinates) const 47 | { 48 | return Evaluate(coordinates, 0, NumFunctions()); 49 | } 50 | 51 | template 52 | inline void MatyasFunction::Gradient(const MatType& coordinates, 53 | const size_t /* begin */, 54 | GradType& gradient, 55 | const size_t /* batchSize */) const 56 | { 57 | // Convenience typedef. 58 | typedef typename MatType::elem_type ElemType; 59 | 60 | // For convenience; we assume these temporaries will be optimized out. 61 | const ElemType x1 = coordinates(0); 62 | const ElemType x2 = coordinates(1); 63 | 64 | gradient.set_size(2, 1); 65 | gradient(0) = ElemType(0.52) * x1 - ElemType(0.48) * x2; 66 | gradient(1) = ElemType(0.52) * x2 - ElemType(0.48) * x1; 67 | } 68 | 69 | template 70 | inline void MatyasFunction::Gradient(const MatType& coordinates, 71 | GradType& gradient) 72 | { 73 | Gradient(coordinates, 0, gradient, NumFunctions()); 74 | } 75 | 76 | } // namespace test 77 | } // namespace ens 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /include/ensmallen_bits/cd/descent_policies/greedy_descent.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file greedy_descent.hpp 3 | * @author Shikhar Bhardwaj 4 | * 5 | * Greedy descent policy for Stochastic Coordinate Descent (SCD). 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_SCD_DESCENT_POLICIES_GREEDY_HPP 13 | #define ENSMALLEN_SCD_DESCENT_POLICIES_GREEDY_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * Greedy descent policy for Stochastic Co-ordinate Descent(SCD). This 19 | * descent scheme picks a the co-ordinate for the descent with the maximum 20 | * guaranteed descent, according to the Gauss-Southwell rule. This is a 21 | * deterministic approach and is generally more expensive to calculate. 22 | * 23 | * For more information, refer to the following. 24 | * @code 25 | * @misc{Nutini2015, 26 | * author = {Julie Nutini and Mark Schmidt and Issam H. 27 | * Laradji and Michael Friedlander and Hoyt Koepke}, 28 | * title = {Coordinate Descent Converges Faster with the Gauss-Southwell Rule 29 | * Than Random Selection}, 30 | * year = {2015}, 31 | * eprint = {arXiv:1506.00552} 32 | * } 33 | * @endcode 34 | */ 35 | class GreedyDescent 36 | { 37 | public: 38 | /** 39 | * The DescentFeature method is used to get the descent coordinate for the 40 | * current iteration. 41 | * 42 | * @tparam ResolvableFunctionType The type of the function to be optimized. 43 | * @param iteration The iteration number for which the feature is to be 44 | * obtained. 45 | * @param iterate The current value of the decision variable. 46 | * @param function The function to be optimized. 47 | * @return The index of the coordinate to be descended. 48 | */ 49 | template 50 | static size_t DescentFeature(const size_t /* iteration */, 51 | const MatType& iterate, 52 | const ResolvableFunctionType& function) 53 | { 54 | typedef typename MatType::elem_type ElemType; 55 | 56 | size_t bestFeature = 0; 57 | ElemType bestDescent = 0; 58 | for (size_t i = 0; i < function.NumFeatures(); ++i) 59 | { 60 | GradType fGrad; 61 | 62 | function.PartialGradient(iterate, i, fGrad); 63 | 64 | ElemType descent = arma::accu(fGrad); 65 | if (descent > bestDescent) 66 | { 67 | bestFeature = i; 68 | bestDescent = descent; 69 | } 70 | } 71 | 72 | return bestFeature; 73 | } 74 | }; 75 | 76 | } // namespace ens 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /include/ensmallen_bits/grid_search/grid_search.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file grid_search.hpp 3 | * @author Kirill Mishchenko 4 | * 5 | * Grid-search optimization. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_GRID_SEARCH_GRID_SEARCH_HPP 13 | #define ENSMALLEN_GRID_SEARCH_GRID_SEARCH_HPP 14 | 15 | namespace ens { 16 | 17 | /** 18 | * An optimizer that finds the minimum of a given function by iterating through 19 | * points on a multidimensional grid. 20 | * 21 | * GridSearch can optimize categorical functions. For more details, see the 22 | * documentation on function types included with this distribution or on the 23 | * ensmallen website. 24 | */ 25 | class GridSearch 26 | { 27 | public: 28 | /** 29 | * Optimize (minimize) the given function by iterating through the all 30 | * possible combinations of values for the parameters specified in 31 | * datasetInfo. 32 | * 33 | * @tparam FunctionType Type of function to optimize. 34 | * @tparam MatType Type of matrix to optimize with. 35 | * @param function Function to optimize. 36 | * @param bestParameters Variable for storing results. 37 | * @param categoricalDimensions Set of dimension types. If a value is true, 38 | * then that dimension is a categorical dimension. 39 | * @param numCategories Number of categories in each categorical dimension. 40 | * @return Objective value of the final point. 41 | */ 42 | template 43 | typename MatType::elem_type Optimize( 44 | FunctionType& function, 45 | MatType& bestParameters, 46 | const std::vector& categoricalDimensions, 47 | const arma::Row& numCategories); 48 | 49 | private: 50 | /** 51 | * Iterate through the last (parameterValueCollections.size() - i) dimensions 52 | * of the grid and change the arguments bestObjective and bestParameters if 53 | * there is something better. The values for the first i dimensions 54 | * (parameters) are specified in the first i rows of the currentParameters 55 | * argument. 56 | */ 57 | template 58 | void Optimize( 59 | FunctionType& function, 60 | typename MatType::elem_type& bestObjective, 61 | MatType& bestParameters, 62 | MatType& currentParameters, 63 | const std::vector& categoricalDimensions, 64 | const arma::Row& numCategories, 65 | size_t i); 66 | }; 67 | 68 | } // namespace ens 69 | 70 | // Include implementation 71 | #include "grid_search_impl.hpp" 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /include/ensmallen_bits/log.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file log.hpp 3 | * @author Marcus Edel 4 | * 5 | * Definition of the Info and Warn log functions. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_LOG_HPP 13 | #define ENSMALLEN_LOG_HPP 14 | 15 | namespace ens { 16 | /** 17 | * This class does nothing and should be optimized out entirely by the compiler. 18 | */ 19 | class NullOutStream 20 | { 21 | public: 22 | /** 23 | * Does nothing. 24 | */ 25 | NullOutStream() { } 26 | 27 | /** 28 | * Does nothing. 29 | */ 30 | NullOutStream(const NullOutStream& /* other */) { } 31 | 32 | //! Does nothing. 33 | NullOutStream& operator<<(bool) { return *this; } 34 | //! Does nothing. 35 | NullOutStream& operator<<(short) { return *this; } 36 | //! Does nothing. 37 | NullOutStream& operator<<(unsigned short) { return *this; } 38 | //! Does nothing. 39 | NullOutStream& operator<<(int) { return *this; } 40 | //! Does nothing. 41 | NullOutStream& operator<<(unsigned int) { return *this; } 42 | //! Does nothing. 43 | NullOutStream& operator<<(long) { return *this; } 44 | //! Does nothing. 45 | NullOutStream& operator<<(unsigned long) { return *this; } 46 | //! Does nothing. 47 | NullOutStream& operator<<(float) { return *this; } 48 | //! Does nothing. 49 | NullOutStream& operator<<(double) { return *this; } 50 | //! Does nothing. 51 | NullOutStream& operator<<(long double) { return *this; } 52 | //! Does nothing. 53 | NullOutStream& operator<<(void*) { return *this; } 54 | //! Does nothing. 55 | NullOutStream& operator<<(const char*) { return *this; } 56 | //! Does nothing. 57 | NullOutStream& operator<<(std::string&) { return *this; } 58 | //! Does nothing. 59 | NullOutStream& operator<<(std::streambuf*) { return *this; } 60 | //! Does nothing. 61 | NullOutStream& operator<<(std::ostream& (*) (std::ostream&)) { return *this; } 62 | //! Does nothing. 63 | NullOutStream& operator<<(std::ios& (*) (std::ios&)) { return *this; } 64 | //! Does nothing. 65 | NullOutStream& operator<<(std::ios_base& (*) (std::ios_base&)) 66 | { return *this; } 67 | 68 | //! Does nothing. 69 | template 70 | NullOutStream& operator<<(const T&) { return *this; } 71 | }; 72 | 73 | #ifdef ENS_PRINT_INFO 74 | static std::ostream& Info = arma::get_cout_stream(); 75 | #else 76 | static NullOutStream Info; 77 | #endif 78 | 79 | #ifdef ENS_PRINT_WARN 80 | static std::ostream& Warn = arma::get_cerr_stream(); 81 | #else 82 | static NullOutStream Warn; 83 | #endif 84 | 85 | } // namespace ens 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /include/ensmallen_bits/problems/mc_cormick_function_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file mc_cormick_function_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of the McCormick function. 6 | * 7 | * ensmallen is free software; you may redistribute it and/or modify it under 8 | * the terms of the 3-clause BSD license. You should have received a copy of 9 | * the 3-clause BSD license along with ensmallen. If not, see 10 | * http://www.opensource.org/licenses/BSD-3-Clause for more information. 11 | */ 12 | #ifndef ENSMALLEN_PROBLEMS_MC_CORMICK_FUNCTION_IMPL_HPP 13 | #define ENSMALLEN_PROBLEMS_MC_CORMICK_FUNCTION_IMPL_HPP 14 | 15 | // In case it hasn't been included yet. 16 | #include "mc_cormick_function.hpp" 17 | 18 | namespace ens { 19 | namespace test { 20 | 21 | inline McCormickFunction::McCormickFunction() { /* Nothing to do here */ } 22 | 23 | inline void McCormickFunction::Shuffle() { /* Nothing to do here */ } 24 | 25 | template 26 | typename MatType::elem_type McCormickFunction::Evaluate( 27 | const MatType& coordinates, 28 | const size_t /* begin */, 29 | const size_t /* batchSize */) const 30 | { 31 | typedef typename MatType::elem_type ElemType; 32 | 33 | // For convenience; we assume these temporaries will be optimized out. 34 | const ElemType x1 = coordinates(0); 35 | const ElemType x2 = coordinates(1); 36 | 37 | const ElemType objective = std::sin(x1 + x2) + 38 | std::pow(x1 - x2, ElemType(2)) - 39 | ElemType(1.5) * x1 + 40 | ElemType(2.5) * x2 + 1; 41 | 42 | return objective; 43 | } 44 | 45 | template 46 | typename MatType::elem_type McCormickFunction::Evaluate( 47 | const MatType& coordinates) const 48 | { 49 | return Evaluate(coordinates, 0, NumFunctions()); 50 | } 51 | 52 | template 53 | inline void McCormickFunction::Gradient(const MatType& coordinates, 54 | const size_t /* begin */, 55 | GradType& gradient, 56 | const size_t /* batchSize */) const 57 | { 58 | typedef typename MatType::elem_type ElemType; 59 | 60 | // For convenience; we assume these temporaries will be optimized out. 61 | const ElemType x1 = coordinates(0); 62 | const ElemType x2 = coordinates(1); 63 | 64 | gradient.set_size(2, 1); 65 | gradient(0) = std::cos(x1 + x2) + 2 * x1 - 2 * x2 - ElemType(1.5); 66 | gradient(1) = std::cos(x1 + x2) - 2 * x1 + 2 * x2 + ElemType(2.5); 67 | } 68 | 69 | template 70 | inline void McCormickFunction::Gradient(const MatType& coordinates, 71 | GradType& gradient) 72 | { 73 | Gradient(coordinates, 0, gradient, NumFunctions()); 74 | } 75 | 76 | } // namespace test 77 | } // namespace ens 78 | 79 | #endif 80 | --------------------------------------------------------------------------------