├── .gitignore ├── LICENSE.md ├── README.md ├── blackbox_opt ├── DFO_src │ ├── README │ ├── __init__.py │ ├── dfo_tr.py │ ├── quad_Frob.py │ └── trust_sub.py ├── __init__.py ├── bb_optimize.py └── test_funcs │ ├── __init__.py │ └── funcs_def.py └── run_test_func.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pyo -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Eclipse Public License - v 1.0 2 | 3 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 4 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 5 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 6 | 7 | 1. DEFINITIONS 8 | 9 | "Contribution" means: 10 | 11 | a) in the case of the initial Contributor, the initial code and documentation 12 | distributed under this Agreement, and 13 | b) in the case of each subsequent Contributor: 14 | i) changes to the Program, and 15 | ii) additions to the Program; 16 | 17 | where such changes and/or additions to the Program originate from and are 18 | distributed by that particular Contributor. A Contribution 'originates' 19 | from a Contributor if it was added to the Program by such Contributor 20 | itself or anyone acting on such Contributor's behalf. Contributions do not 21 | include additions to the Program which: (i) are separate modules of 22 | software distributed in conjunction with the Program under their own 23 | license agreement, and (ii) are not derivative works of the Program. 24 | 25 | "Contributor" means any person or entity that distributes the Program. 26 | 27 | "Licensed Patents" mean patent claims licensable by a Contributor which are 28 | necessarily infringed by the use or sale of its Contribution alone or when 29 | combined with the Program. 30 | 31 | "Program" means the Contributions distributed in accordance with this 32 | Agreement. 33 | 34 | "Recipient" means anyone who receives the Program under this Agreement, 35 | including all Contributors. 36 | 37 | 2. GRANT OF RIGHTS 38 | a) Subject to the terms of this Agreement, each Contributor hereby grants 39 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 40 | reproduce, prepare derivative works of, publicly display, publicly 41 | perform, distribute and sublicense the Contribution of such Contributor, 42 | if any, and such derivative works, in source code and object code form. 43 | b) Subject to the terms of this Agreement, each Contributor hereby grants 44 | Recipient a non-exclusive, worldwide, royalty-free patent license under 45 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 46 | transfer the Contribution of such Contributor, if any, in source code and 47 | object code form. This patent license shall apply to the combination of 48 | the Contribution and the Program if, at the time the Contribution is 49 | added by the Contributor, such addition of the Contribution causes such 50 | combination to be covered by the Licensed Patents. The patent license 51 | shall not apply to any other combinations which include the Contribution. 52 | No hardware per se is licensed hereunder. 53 | c) Recipient understands that although each Contributor grants the licenses 54 | to its Contributions set forth herein, no assurances are provided by any 55 | Contributor that the Program does not infringe the patent or other 56 | intellectual property rights of any other entity. Each Contributor 57 | disclaims any liability to Recipient for claims brought by any other 58 | entity based on infringement of intellectual property rights or 59 | otherwise. As a condition to exercising the rights and licenses granted 60 | hereunder, each Recipient hereby assumes sole responsibility to secure 61 | any other intellectual property rights needed, if any. For example, if a 62 | third party patent license is required to allow Recipient to distribute 63 | the Program, it is Recipient's responsibility to acquire that license 64 | before distributing the Program. 65 | d) Each Contributor represents that to its knowledge it has sufficient 66 | copyright rights in its Contribution, if any, to grant the copyright 67 | license set forth in this Agreement. 68 | 69 | 3. REQUIREMENTS 70 | 71 | A Contributor may choose to distribute the Program in object code form under 72 | its own license agreement, provided that: 73 | 74 | a) it complies with the terms and conditions of this Agreement; and 75 | b) its license agreement: 76 | i) effectively disclaims on behalf of all Contributors all warranties 77 | and conditions, express and implied, including warranties or 78 | conditions of title and non-infringement, and implied warranties or 79 | conditions of merchantability and fitness for a particular purpose; 80 | ii) effectively excludes on behalf of all Contributors all liability for 81 | damages, including direct, indirect, special, incidental and 82 | consequential damages, such as lost profits; 83 | iii) states that any provisions which differ from this Agreement are 84 | offered by that Contributor alone and not by any other party; and 85 | iv) states that source code for the Program is available from such 86 | Contributor, and informs licensees how to obtain it in a reasonable 87 | manner on or through a medium customarily used for software exchange. 88 | 89 | When the Program is made available in source code form: 90 | 91 | a) it must be made available under this Agreement; and 92 | b) a copy of this Agreement must be included with each copy of the Program. 93 | Contributors may not remove or alter any copyright notices contained 94 | within the Program. 95 | 96 | Each Contributor must identify itself as the originator of its Contribution, 97 | if 98 | any, in a manner that reasonably allows subsequent Recipients to identify the 99 | originator of the Contribution. 100 | 101 | 4. COMMERCIAL DISTRIBUTION 102 | 103 | Commercial distributors of software may accept certain responsibilities with 104 | respect to end users, business partners and the like. While this license is 105 | intended to facilitate the commercial use of the Program, the Contributor who 106 | includes the Program in a commercial product offering should do so in a manner 107 | which does not create potential liability for other Contributors. Therefore, 108 | if a Contributor includes the Program in a commercial product offering, such 109 | Contributor ("Commercial Contributor") hereby agrees to defend and indemnify 110 | every other Contributor ("Indemnified Contributor") against any losses, 111 | damages and costs (collectively "Losses") arising from claims, lawsuits and 112 | other legal actions brought by a third party against the Indemnified 113 | Contributor to the extent caused by the acts or omissions of such Commercial 114 | Contributor in connection with its distribution of the Program in a commercial 115 | product offering. The obligations in this section do not apply to any claims 116 | or Losses relating to any actual or alleged intellectual property 117 | infringement. In order to qualify, an Indemnified Contributor must: 118 | a) promptly notify the Commercial Contributor in writing of such claim, and 119 | b) allow the Commercial Contributor to control, and cooperate with the 120 | Commercial Contributor in, the defense and any related settlement 121 | negotiations. The Indemnified Contributor may participate in any such claim at 122 | its own expense. 123 | 124 | For example, a Contributor might include the Program in a commercial product 125 | offering, Product X. That Contributor is then a Commercial Contributor. If 126 | that Commercial Contributor then makes performance claims, or offers 127 | warranties related to Product X, those performance claims and warranties are 128 | such Commercial Contributor's responsibility alone. Under this section, the 129 | Commercial Contributor would have to defend claims against the other 130 | Contributors related to those performance claims and warranties, and if a 131 | court requires any other Contributor to pay any damages as a result, the 132 | Commercial Contributor must pay those damages. 133 | 134 | 5. NO WARRANTY 135 | 136 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN 137 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR 138 | IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each 140 | Recipient is solely responsible for determining the appropriateness of using 141 | and distributing the Program and assumes all risks associated with its 142 | exercise of rights under this Agreement , including but not limited to the 143 | risks and costs of program errors, compliance with applicable laws, damage to 144 | or loss of data, programs or equipment, and unavailability or interruption of 145 | operations. 146 | 147 | 6. DISCLAIMER OF LIABILITY 148 | 149 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 150 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 151 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 152 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 153 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 154 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 155 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 156 | OF SUCH DAMAGES. 157 | 158 | 7. GENERAL 159 | 160 | If any provision of this Agreement is invalid or unenforceable under 161 | applicable law, it shall not affect the validity or enforceability of the 162 | remainder of the terms of this Agreement, and without further action by the 163 | parties hereto, such provision shall be reformed to the minimum extent 164 | necessary to make such provision valid and enforceable. 165 | 166 | If Recipient institutes patent litigation against any entity (including a 167 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 168 | (excluding combinations of the Program with other software or hardware) 169 | infringes such Recipient's patent(s), then such Recipient's rights granted 170 | under Section 2(b) shall terminate as of the date such litigation is filed. 171 | 172 | All Recipient's rights under this Agreement shall terminate if it fails to 173 | comply with any of the material terms or conditions of this Agreement and does 174 | not cure such failure in a reasonable period of time after becoming aware of 175 | such noncompliance. If all Recipient's rights under this Agreement terminate, 176 | Recipient agrees to cease use and distribution of the Program as soon as 177 | reasonably practicable. However, Recipient's obligations under this Agreement 178 | and any licenses granted by Recipient relating to the Program shall continue 179 | and survive. 180 | 181 | Everyone is permitted to copy and distribute copies of this Agreement, but in 182 | order to avoid inconsistency the Agreement is copyrighted and may only be 183 | modified in the following manner. The Agreement Steward reserves the right to 184 | publish new versions (including revisions) of this Agreement from time to 185 | time. No one other than the Agreement Steward has the right to modify this 186 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 187 | Eclipse Foundation may assign the responsibility to serve as the Agreement 188 | Steward to a suitable separate entity. Each new version of the Agreement will 189 | be given a distinguishing version number. The Program (including 190 | Contributions) may always be distributed subject to the version of the 191 | Agreement under which it was received. In addition, after a new version of the 192 | Agreement is published, Contributor may elect to distribute the Program 193 | (including its Contributions) under the new version. Except as expressly 194 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 195 | licenses to the intellectual property of any Contributor under this Agreement, 196 | whether expressly, by implication, estoppel or otherwise. All rights in the 197 | Program not expressly granted under this Agreement are reserved. 198 | 199 | This Agreement is governed by the laws of the State of New York and the 200 | intellectual property laws of the United States of America. No party to this 201 | Agreement will bring a legal action under this Agreement more than one year 202 | after the cause of action arose. Each party waives its rights to a jury trial in 203 | any resulting litigation. 204 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dfo-algorithm 2 | This package provides an implementation of the derivative-free 3 | optimization algorithm, DFO, developed by A. Conn, K. Scheinberg, 4 | L. Vicente. 5 | Using this package, the user can solve a derivative-free 6 | blackbox optimization problem with the DFO method as well 7 | as five derivative free algorithms from the scipy.optimize library. 8 | The scipy algorithms are the Nelder-Mead, Powell, SLSQP, 9 | COBYLA and BFGS algorithms. 10 | 11 | To run a set of sample problems, the user can call the 12 | “run_test_func.py” module. 13 | 14 | To solve a user-defined problem: 15 | ========================================================== 16 | - Write your blackbox optimization function in a new Python 17 | module. For examples of such functions see 18 | blackbox_opt/test_funcs/funcs_def.py 19 | - Write a run file with the module you wrote in the previous 20 | step imported. You can modify the run_test_func.py module and 21 | use it as your run file. Note that you should specify the 22 | algorithms(s) that you wish to solve your problems with, 23 | a starting point and suitable options. 24 | -------------------------------------------------------------------------------- /blackbox_opt/DFO_src/README: -------------------------------------------------------------------------------- 1 | A Python implementation of the DFO algorithm developed by A. Conn, 2 | K. Scheinberg, L. Vicente. 3 | author: Anahita Hassanzadeh 4 | email: Anahita.Hassanzadeh@gmail.com 5 | 6 | The DFO code consisits of the following modules: 7 | 8 | -dfo_tr.py: this is the main module of the algorithm that contains 9 | the trust-region main loop. This module takes the starting point of 10 | the algorithm and reports the iterations stats and solution time. 11 | If the user wishes to specify more than one point, 12 | she should modify the "build_initial_sample" function in this file. 13 | 14 | -quad_Frob.py: this function constructs a quadratic model from a given 15 | set of points. The model is constructed such that the Frobenius norm 16 | of the Hessian is minimized. 17 | 18 | -trust_sub.py: solves the trust region subproblem to find the step 19 | size to move from the current point to a new point. 20 | 21 | The user can run the DFO algorithm with the following options: 22 | - sample_gen # the initial sample set option {"auto" or "manual"} 23 | [default:auto] 24 | - verbosity # the verbosity option set it to 0 or 1 25 | [default:1] 26 | 27 | [default values for the following options are set in the 28 | "params" class in the dfo_tr module] 29 | 30 | - delta #initial delta (i.e. trust region radius) 31 | - tol_delta # smallest delta to stop 32 | - max_delta # max possible delta 33 | 34 | - gamma1 # TR radius shrink factor 35 | - gamma2 # TR radius expansion factor 36 | 37 | - eta0 # step acceptance test (pred/ared) threshold 38 | - eta1 # (pred/ared) level to shrink radius 39 | - eta2 # (pred/ared) level to expand radius 40 | 41 | - tol_f # threshold for abs(fprev - fnew)- used to stop 42 | - tol_norm_g # threshold for norm of gradient- used to stop 43 | - tol_norm_H # threshold for the (frobenius) norm of H 44 | - maxfev # maximum number of iterations 45 | 46 | - min_del_crit # minimum delta for the criticality step 47 | - min_s_crit # minimum step for the criticality step 48 | -------------------------------------------------------------------------------- /blackbox_opt/DFO_src/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /blackbox_opt/DFO_src/dfo_tr.py: -------------------------------------------------------------------------------- 1 | """ 2 | Description: An implementation of the DFO algorithm developed by A. Conn, 3 | K. Scheinberg, L. Vicente. 4 | author: Anahita Hassanzadeh 5 | email: Anahita.Hassanzadeh@gmail.com 6 | """ 7 | import numpy as np 8 | from scipy import linalg as LA 9 | import time 10 | from quad_Frob import quad_Frob 11 | from trust_sub import trust_sub 12 | 13 | class params: 14 | """ 15 | The class of parameters for the trust region algorithms 16 | The parameters include the initial delta, its expansion and contraction 17 | factors and acceptance as well as the maximum number of iterations 18 | """ 19 | def __init__(self): 20 | self.init_delta = 1.0 # initial delta (i.e. trust region radius) 21 | self.tol_delta = 1e-10 # smallest delta to stop 22 | self.max_delta = 100.0 # max possible delta 23 | # TR radius adjustment parameters 24 | self.gamma1 = 0.8 # radius shrink factor 25 | self.gamma2 = 1.5 # radius expansion factor 26 | 27 | self.eta0 = 0.0 # step acceptance test (pred/ared) threshold 28 | self.eta1 = 0.25 # (pred/ared) level to shrink radius 29 | self.eta2 = 0.75 # (pred/ared) level to expand radius 30 | 31 | self.tol_f = 1e-15 # threshold for abs(fprev - fnew)- used to stop 32 | self.tol_norm_g = 1e-15 # threshold for norm of gradient- used to stop 33 | self.tol_norm_H = 1e-10 # threshold for the (frobenius) norm of H 34 | self.maxfev = 1000 # maximum number of iterations 35 | 36 | self.min_del_crit = 1e-8 # minimum delta for the criticality step 37 | self.min_s_crit = 0.1 # minimum step for the criticality step 38 | 39 | class result: 40 | def __init__(self, x, f, iteration, iter_suc, func_eval, delta): 41 | self.x = x 42 | self.fun = f 43 | self.iteration = iteration 44 | self.iter_suc = iter_suc 45 | self.func_eval = func_eval 46 | self.delta = delta 47 | 48 | def _set_parameters(options): 49 | par = params() 50 | if options: 51 | for key in options: 52 | if (key not in par.__dict__.keys() and 53 | key not in ["sample_gen", "verbosity"]): 54 | raise ValueError("{!r} is not a valid option".format(key)) 55 | par.__dict__[key] = options[key] 56 | return par 57 | 58 | def _build_initial_sample(x_initial, delta, options=None): 59 | """ 60 | Given the original point and delta, generate a sample set to start the 61 | method. If nothing is provided here, the algorithm works fine. 62 | Note that if the number of points are greater than (n+1)(n+2)/2, then 63 | a point is only be substituted and no additional points will be added. 64 | It may be a good idea to leave some point generation to the model so it 65 | uses the min Frobenius norm model in the quad_Frob module. 66 | Do not build a sample size more than (n+1)(n+2)/2. We are going to build a 67 | quadratic model after all! 68 | """ 69 | n = x_initial.shape[0] 70 | # if the user hasn't specified any options or no option for sample build 71 | # set it to auto. 72 | if not options or "sample_gen" not in options.keys(): 73 | option = "auto" 74 | else: 75 | option = options["sample_gen"] 76 | 77 | if option == "auto": 78 | Y = np.tile(x_initial, (1, 2*n)) + (0.5 * delta * np.hstack( 79 | (np.eye(n), -np.eye(n)))) 80 | Y = np.hstack((x_initial, Y)) 81 | elif option == "manual": 82 | # othewise the user can specify the starting points manually 83 | # for example, for a function with 4 variables we have: 84 | Y1 = np.array([150.00, 150.00, 230.00, 150.00]).reshape((n, 1)) 85 | Y2 = np.array([150.00, 8.00, 230.00, 250.00]).reshape((n, 1)) 86 | Y3 = np.array([150.00, 50.00, 230.00, 230.00]).reshape((n, 1)) 87 | Y = np.hstack((x_initial, Y1, Y2, Y3)) 88 | else: 89 | raise ValueError("The sample option should be manual or auto.") 90 | 91 | nY = Y.shape[1] 92 | return (Y, nY) 93 | 94 | def _shift_sort_points(x, Y, nY, f_values): 95 | '''shift the points to the origin and sort 96 | the corresponding objective values and norms 97 | ''' 98 | Ynorms = Y - np.tile(x, (1, nY)) 99 | Ynorms = np.sqrt(np.sum(Ynorms**2, axis=0)) 100 | index = np.argsort(Ynorms) 101 | Ynorms = Ynorms[index] 102 | Y = Y[:, index] 103 | f_values = f_values[index] 104 | return (Ynorms, Y, f_values) 105 | 106 | def dfo_tr(bb_func, x_initial, options=None): 107 | """ 108 | This is the main function of the trust region method. It takes the 109 | intialized point as the input and returns the solution x and its 110 | corresponding value. If the maximum number of allowed iterations is 111 | set to a large enough number, the method returns a stationary point of 112 | the optimziation problem. The main functions inside this function are 113 | quad_Frob and trust functions. For the details of the algorithm see 114 | chapter 11 of the Introduction to DFO by Conn, Scheinberg and Vicente 115 | 116 | arguments 117 | --------- 118 | bb_func: function object 119 | The blackbox function to minimize. The function takes a numpy.ndarray 120 | and returns a scalar. For examples of this see test_funcs/funcs_def.py 121 | x_initial: np.ndarray 122 | a starting point for the algorithm 123 | options: dict 124 | See the README file for the supported options. 125 | 126 | returns 127 | --------- 128 | a "result object" with the following attributes 129 | x: np.ndarray 130 | the optimal point found 131 | fun: float 132 | the optimal value 133 | iteration: int 134 | number of iterations 135 | iter_suc: int 136 | number of successful iterations (where the objective decreased) 137 | func_eval: int 138 | number of blackbox function evaluations 139 | delta: float 140 | the radius of the trust region at termination 141 | """ 142 | # start timing and set the paramters 143 | start_time = time.time() 144 | 145 | par = _set_parameters(options) 146 | # see the param class for the description of parameters 147 | delta = par.init_delta 148 | tol_delta = par.tol_delta 149 | max_delta = par.max_delta 150 | gamma1 = par.gamma1 151 | max_iter = par.maxfev 152 | eta0 = par.eta0 153 | eta1 = par.eta1 # this may be used depending on the application 154 | eta2 = par.eta2 155 | tol_f = par.tol_f 156 | gamma2 = par.gamma2 157 | tol_norm_g = par.tol_norm_g 158 | tol_norm_H = par.tol_norm_H 159 | min_del_crit = par.min_del_crit 160 | min_s_crit = par.min_s_crit 161 | 162 | # set the verbosity parameter 163 | if options and "verbosity" in options: 164 | verb = options["verbosity"] 165 | if verb == 0 or verb == 1: 166 | verbosity = verb 167 | else: 168 | raise ValueError("verbosity option should be 0 or 1") 169 | else: 170 | verbosity = 1 171 | # find the dimension of the problem 172 | # find the max and min number of points for the quadratic model 173 | n = x_initial.shape[0] 174 | maxY = (n+1) * (n+2) / 2 175 | minY = n+1 176 | 177 | # iterations counters 178 | iteration = iter_suc = func_eval = 0 179 | 180 | # construct the intial sample set and evaluate the objective at them 181 | # by evaluating the blackbox function 182 | x_initial = x_initial.reshape((n, 1)) 183 | Y, nY = _build_initial_sample(x_initial, delta, options) 184 | f_values = np.empty((nY, 1)) 185 | 186 | for i in range(nY): 187 | f_values[i] = bb_func(Y[:, i].reshape(n, 1)) 188 | func_eval += 1 189 | 190 | # find the point with minimum objective. set it as the first center 191 | ind_f_sort = np.argsort(f_values[:, 0])[0] 192 | x = Y[:, ind_f_sort].reshape(n, 1) 193 | f = f_values[ind_f_sort][0] 194 | 195 | # print out the initial evaluation results 196 | if verbosity: 197 | print ("\n Iteration Report \n") 198 | print ('|it |suc| objective | TR_radius | rho | |Y| \n') 199 | print ("| {} |---| {} | {} | --------- | {} \n".format(iteration, 200 | format(f, '0.6f'), format(delta, '0.6f'), nY)) 201 | 202 | # Start the TR main loop 203 | while True: 204 | success = 0 205 | 206 | # construct the quadratic model 207 | H, g = quad_Frob(Y, f_values) 208 | 209 | # obtain the norm of the "g" (coeff of linear components) of the model 210 | normg = LA.norm(g) 211 | 212 | # Stopping criterion --stop if the TR radius or norm of g is too small 213 | if normg <= tol_norm_g or delta < tol_delta: 214 | break 215 | 216 | # Start the TR iteration 217 | # Minimize the model on the TR, aka: solve the TR subproblem 218 | if LA.norm(H, 'fro') > tol_norm_H: # make sure the trust can work 219 | s, val = trust_sub(g, H, delta) 220 | else: # otherwise take a steepest descent step less than delta 221 | s = -(delta/normg) * g 222 | val = np.dot(g.T, s) + 0.5 * reduce(np.dot, [s.T, H, s]) 223 | 224 | # f: model value at the origin of trust region 225 | # Evaluate the model value at the new point. 226 | fmod = f + val 227 | 228 | # Stop if the reduction is very small or max iterations reached 229 | if abs(val) < tol_f or iteration > max_iter: 230 | break 231 | 232 | # Take the step 233 | xtrial = x + s 234 | 235 | # Evaluate the function at the new point. 236 | ftrue = bb_func(xtrial) 237 | func_eval += 1 238 | 239 | # Calculate ared/pred. 240 | pred = f - fmod 241 | ared = f - ftrue 242 | rho = ared / pred 243 | 244 | # Updating iterate and trust-region radius. 245 | if rho >= eta0 and f - ftrue > 0: 246 | # Set new iterate if there is success. 247 | success = 1 248 | iter_suc += 1 249 | 250 | # Update the center of the trust region 251 | x = xtrial 252 | f = ftrue 253 | if rho >= eta2: # If confident, increase delta 254 | delta = min(gamma2*delta, max_delta) 255 | else: 256 | if nY >= minY: 257 | # decrease the TR radius 258 | delta = gamma1*delta 259 | 260 | iteration += 1 261 | 262 | # print iteration report 263 | if verbosity: 264 | print ("| {} | {} | {:.6f} | {} | {} | {} \n".format(iteration, 265 | success, float(f), format(delta, '0.6f'), 266 | format(rho[0][0], '0.6f'), nY)) 267 | print x.T, "\n" 268 | 269 | # order the sample set according to the distance to the 270 | # current iterate. 271 | Ynorms, Y, f_values = _shift_sort_points(x, Y, nY, f_values) 272 | 273 | # Update the sample set. 274 | if success: 275 | reached_max = 1 276 | if nY < maxY: 277 | reached_max = 0 278 | 279 | # add or substitute the furthest point 280 | if reached_max: # substitute 281 | Y[:, nY-1] = xtrial.T 282 | f_values[nY-1] = ftrue 283 | 284 | else: # add 285 | nY = nY + 1 286 | Y = np.hstack((Y, xtrial)) 287 | f_values = np.vstack((f_values, np.array([ftrue]))) 288 | else: # if not successful 289 | if nY >= maxY: 290 | # add if closer than the furthest point, otherwise 291 | # keep the set as it is 292 | if LA.norm(xtrial-x) <= Ynorms[nY-1]: 293 | # Substitute the furthest point 294 | Y[:, nY-1] = xtrial.T 295 | f_values[nY-1] = ftrue 296 | else: 297 | nY = nY + 1 298 | Y = np.hstack((Y, xtrial)) 299 | f_values = np.vstack((f_values, np.array([ftrue]))) 300 | 301 | # shift and sort the set of points once more 302 | Ynorms, Y, f_values = _shift_sort_points(x, Y, nY, f_values) 303 | 304 | # implementation of an idea similar to the criticality step 305 | # this is, if the gradient is small, then delta should be 306 | # proportional to it -- should not happen often in practice 307 | if delta < min_del_crit and LA.norm(s) < min_s_crit: 308 | # only keep the points that are within 100*delta radius 309 | selection = np.where(Ynorms < delta * 100) 310 | Y = Y[:, selection[0]] 311 | f_values = f_values[selection] 312 | nY = Y.shape[1] 313 | 314 | end_time = time.time() 315 | if verbosity: 316 | # Final Report 317 | print ('*****************REPORT************************\n') 318 | print ("Total time is {} seconds.\n".format(end_time - start_time)) 319 | print ("Norm of the gradient of the model is {}.\n".format(normg)) 320 | print ('***************Final Report**************\n') 321 | print ("|iter | #success| #fevals| final fvalue | final tr_radius|\n") 322 | print ("| {} | {} | {} | {} | {} \n" 323 | .format(iteration, iter_suc, func_eval, f, delta)) 324 | res = result(x, f, iteration, iter_suc, func_eval, delta) 325 | return res -------------------------------------------------------------------------------- /blackbox_opt/DFO_src/quad_Frob.py: -------------------------------------------------------------------------------- 1 | """ 2 | Description: 3 | An implementation of the DFO algorithm developed by A. Conn, 4 | K. Scheinberg, L. Vicente 5 | author: Anahita Hassanzadeh 6 | email: Anahita.Hassanzadeh@gmail.com 7 | """ 8 | import numpy as np 9 | from scipy import linalg as LA 10 | 11 | def _compute_coeffs(W, tol_svd, b, option): 12 | """Compute model coefficients -- Here we are merely solving the 13 | system of equations -- decompose W and use its inverse 14 | """ 15 | if option == 'partial': 16 | U, S, VT = LA.svd(W) 17 | else: 18 | U, S, VT = LA.svd(W, full_matrices=False) 19 | 20 | # Make sure the condition number is not too high 21 | indices = S < tol_svd 22 | S[indices] = tol_svd 23 | Sinv = np.diag(1/S) 24 | V = VT.T 25 | # Get the coefficients 26 | lambda_0 = reduce(np.dot, [V, Sinv, U.T, b]) 27 | return (lambda_0) 28 | 29 | def quad_Frob(X, F_values): 30 | """ 31 | Given a set of points in the trust region 32 | and their values, construct a quadratic model 33 | in the form of g.T x + 1/2 x.T H x + \alpha. 34 | 35 | If the number of points are less than 36 | (n+1) (n+2)/2 then build the model such that the 37 | Frobenius norm is minimized. In this code the KKT 38 | conditions are solved. Otherwise, solve 39 | the system of equations used in polynomial interpolation. 40 | M(\phi, Y) \lambda = f 41 | 42 | arguments: X: the sample points to be interpolated 43 | F_values: the corresponding true solutions to the sample points 44 | outputs: g and H in the quadratic model 45 | """ 46 | # Minimum value accepted for a singular value 47 | eps = np.finfo(float).eps 48 | tol_svd = eps**5 49 | # n = number of variables m = number of points 50 | (n, m) = X.shape 51 | 52 | H = np.zeros((n, n)) 53 | g = np.zeros((n, 1)) 54 | 55 | # Shift the points to the origin 56 | Y = X - np.dot(np.diag(X[:, 0]), np.ones((n, m))) 57 | 58 | if (m < (n+1)*(n+2)/2): 59 | # Construct a quad model by minimizing the Frobenius norm of 60 | # the Hessian -- the following is the solution of the KKT conditions 61 | # of the optimization problem on page 81 of the Intro to DFO book 62 | b = np.vstack((F_values, np.zeros((n+1, 1)))) 63 | A = 0.5 * (np.dot(Y.T, Y)**2) 64 | 65 | # Construct W by augmenting the vector of ones with the linear and 66 | # quadratic terms. The first m rows build the matrix M, which is 67 | # introduced in the slides (monomials of quadratic basis) 68 | top = np.hstack((A, np.ones((m, 1)), Y.T)) 69 | temp = np.vstack((np.ones((1, m)), Y)) 70 | bottom = np.hstack((temp, np.zeros((n+1, n+1)))) 71 | W = np.vstack((top, bottom)) 72 | lambda_0 = _compute_coeffs(W, tol_svd, b, option='partial') 73 | 74 | # Grab the coeffs of linear terms (g) and the ones of quadratic terms 75 | # (H) for g.T s + s.T H s 76 | g = lambda_0[m+1:] 77 | 78 | H = np.zeros((n, n)) 79 | for j in range(m): 80 | H = H + (lambda_0[j] * 81 | np.dot(Y[:, j].reshape(n, 1), Y[:, j].reshape(1, n))) 82 | 83 | else: # Construct a full model 84 | # Here we have enough points. Solve the sys of equations. 85 | b = F_values 86 | phi_Q = np.array([]) 87 | for i in range(m): 88 | y = Y[:, i] 89 | y = y[np.newaxis] # turn y from 1D to a 2D array 90 | aux_H = y * y.T - 0.5 * np.diag(pow(y, 2)[0]) 91 | aux = np.array([]) 92 | for j in range(n): 93 | aux = np.hstack((aux, aux_H[j:n, j])) 94 | 95 | phi_Q = np.vstack((phi_Q, aux)) if phi_Q.size else aux 96 | 97 | W = np.hstack((np.ones((m, 1)), Y.T)) 98 | W = np.hstack((W, phi_Q)) 99 | 100 | lambda_0 = _compute_coeffs(W, tol_svd, b, option='full') 101 | 102 | # Retrieve the model coeffs (g) and (H) 103 | g = lambda_0[1:n+1, :] 104 | cont = n+1 105 | H = np.zeros((n, n)) 106 | 107 | for j in range(n): 108 | H[j:n, j] = lambda_0[cont:cont + n - j, :].reshape((n-j,)) 109 | cont = cont + n - j 110 | 111 | H = H + H.T - np.diag(np.diag(H)) 112 | return (H, g) 113 | -------------------------------------------------------------------------------- /blackbox_opt/DFO_src/trust_sub.py: -------------------------------------------------------------------------------- 1 | """ 2 | Description: This file contains functions to optimize the trust-region 3 | subproblem that is solved sequentially within the DFO method. 4 | Author: Anahita Hassanzadeh 5 | Email: Anahita.Hassanzadeh@gmail.com 6 | """ 7 | from scipy import linalg as LA 8 | import numpy as np 9 | from scipy.sparse import csr_matrix 10 | 11 | def _secular_eqn(lambda_0, eigval, alpha, delta): 12 | """ 13 | The function secular_eqn returns the value of the secular 14 | equation at a set of m points. 15 | """ 16 | m = lambda_0.size 17 | n = len(eigval) 18 | unn = np.ones((n, 1)) 19 | unm = np.ones((m, 1)) 20 | M = np.dot(eigval, unm.T) + np.dot(unn, lambda_0.T) 21 | MC = M.copy() 22 | MM = np.dot(alpha, unm.T) 23 | M[M != 0.0] = MM[M != 0.0] / M[M != 0.0] 24 | M[MC == 0.0] = np.inf * np.ones(MC[MC == 0.0].size) 25 | M = M*M 26 | value = np.sqrt(unm / np.dot(M.T, unn)) 27 | 28 | if len(value[np.where(value == np.inf)]): 29 | inf_arg = np.where(value == np.inf) 30 | value[inf_arg] = np.zeros((len(value[inf_arg]), 1)) 31 | 32 | value = (1.0/delta) * unm - value 33 | 34 | return value 35 | 36 | def rfzero(x, itbnd, eigval, alpha, delta, tol): 37 | """ 38 | This function finds the zero of a function 39 | of one variable to the RIGHT of the starting point x. 40 | The code contanins a small modification of the M-file fzero in matlab, 41 | to ensure a zero to the right of x is searched for. 42 | """ 43 | # start the iteration counter 44 | itfun = 0 45 | 46 | # find the first three points, a, b, and c and their values 47 | if (x != 0.0): 48 | dx = abs(x) / 2 49 | else: 50 | dx = 0.5 51 | 52 | a = x 53 | c = a 54 | fa = _secular_eqn(a, eigval, alpha, delta) 55 | itfun = itfun + 1 56 | 57 | b = x + dx 58 | b = x + 1 59 | fb = _secular_eqn(b, eigval, alpha, delta) 60 | itfun = itfun + 1 61 | 62 | # find change of sign 63 | while ((fa > 0) == (fb > 0)): 64 | 65 | dx = 2*dx 66 | 67 | if ((fa > 0) != (fb > 0)): 68 | break 69 | b = x + dx 70 | fb = _secular_eqn(b, eigval, alpha, delta) 71 | itfun = itfun + 1 72 | 73 | if (itfun > itbnd): 74 | break 75 | 76 | fc = fb 77 | 78 | # main loop, exit from the middle of the loop 79 | while (fb != 0): 80 | # Ensure that b is the best result so far, a is the previous 81 | # value of b, and c is on hte opposit side of 0 from b 82 | if (fb > 0) == (fc > 0): 83 | c = a 84 | fc = fa 85 | d = b - a 86 | e = d 87 | 88 | if abs(fc) < abs(fb): 89 | a = b 90 | b = c 91 | c = a 92 | fa = fb 93 | fb = fc 94 | fc = fa 95 | 96 | # convergence test and possible exit 97 | if itfun > itbnd: 98 | break 99 | 100 | m = 0.5 * (c-b) 101 | rel_tol = 2.0 * tol * max(abs(b), 1.0) 102 | 103 | if (abs(m) <= rel_tol) or (abs(fb) < tol): 104 | break 105 | 106 | # choose bisection or interpolation 107 | if (abs(e) < rel_tol) or (abs(fa) <= abs(fb)): 108 | # bisection 109 | d = e = m 110 | else: 111 | # interpolation 112 | s = float(fb)/fa 113 | if a == c: 114 | # linear interpolation 115 | p = 2.0 * m * s 116 | q = 1.0 - s 117 | else: 118 | # Inverse quad interpolation 119 | q = float(fa)/fc 120 | r = float(fb)/fc 121 | p = s * (2.0 * m * q * (q-r) - (b-a) * (r-1.0)) 122 | q = (q-1.0) * (r-1.0) * (s-1.0) 123 | if p > 0: 124 | q = -q 125 | else: 126 | p = -p 127 | # Check if the interpolated point is acceptable 128 | if (2.0*p < 3.0*m*q - abs(rel_tol*q)) and (p < abs(0.5*e*q)): 129 | e = d 130 | d = float(p)/q 131 | else: 132 | d = m 133 | e = m 134 | # end of iterpolation 135 | 136 | # Next point 137 | a = b 138 | fa = fb 139 | if (abs(d) > rel_tol): 140 | b = b + d 141 | else: 142 | if b > c: 143 | b = b - rel_tol 144 | else: 145 | b = b + rel_tol 146 | 147 | fb = _secular_eqn(b, eigval, alpha, delta) 148 | itfun = itfun + 1 149 | 150 | return (b, c, itfun) 151 | 152 | 153 | def trust_sub(g, H, delta): 154 | """ 155 | This function solves the trust region subproblem when the 156 | Frobenuis norm of H is not very small. 157 | The subproblem is: 158 | min g.T s + 1/2 s.T H s 159 | s.t. || s || <= delta 160 | 161 | Note that any restriction that the problem has 162 | can be added to the constriants in the trust region. 163 | In that case the following algorithm will not work and 164 | another package should be used. The alternative is to 165 | penalize the constraints violations in the objective function 166 | evaluations. 167 | """ 168 | 169 | tol = 10e-12 170 | tol_seqeq = 10e-8 171 | key = 0 172 | itbnd = 50 173 | lambda_0 = 0 174 | s_factor = 0.8 175 | b_factor = 1.2 176 | n = len(g) 177 | coeff = np.zeros((n, 1)) 178 | 179 | # convert H to full matrix if sparse 180 | T = csr_matrix(H) 181 | T = T.todense() 182 | H = np.squeeze(np.asarray(T)) 183 | 184 | # get the eigen value and vector 185 | D, V = LA.eigh(0.5 * (H.T + H)) 186 | count = 0 187 | eigval = D[np.newaxis].T 188 | # find the minimum eigen value 189 | jmin = np.argmin(eigval) 190 | mineig = np.amin(eigval) 191 | 192 | # depending on H, find a step size 193 | alpha = np.dot(-V.T, g) 194 | sig = (np.sign(alpha[jmin]) + (alpha[jmin] == 0).sum())[0] 195 | 196 | # PSD case 197 | if mineig > 0: 198 | lambda_0 = 0 199 | coeff = alpha * (1/eigval) 200 | s = np.dot(V, coeff) 201 | # That is, s = -v (-v.T g./eigval) 202 | nrms = LA.norm(s) 203 | if nrms < b_factor*delta: 204 | key = 1 205 | else: 206 | laminit = np.array([[0]]) 207 | else: 208 | laminit = -mineig 209 | 210 | # Indefinite case 211 | if key == 0: 212 | if _secular_eqn(laminit, eigval, alpha, delta) > 0: 213 | b, c, count = rfzero(laminit, itbnd, eigval, alpha, delta, tol) 214 | 215 | if abs(_secular_eqn(b, eigval, alpha, delta)) <= tol_seqeq: 216 | lambda_0 = b 217 | key = 2 218 | lam = lambda_0 * np.ones((n, 1)) 219 | 220 | coeff, s, nrms, w = compute_step(alpha, eigval, coeff, V, lam) 221 | 222 | if (nrms > b_factor * delta or nrms < s_factor * delta): 223 | key = 5 224 | lambda_0 = -mineig 225 | else: 226 | key = 3 227 | lambda_0 = -mineig 228 | else: 229 | key = 4 230 | lambda_0 = -mineig 231 | 232 | lam = lambda_0 * np.ones((n, 1)) 233 | 234 | if key > 2: 235 | arg = abs(eigval + lam) < 10 * (np.finfo(float).eps * 236 | np.maximum(abs(eigval), np.ones((n,1)))) 237 | alpha[arg] = 0.0 238 | 239 | coeff, s, nrms, w = compute_step(alpha, eigval, coeff, V, lam) 240 | 241 | if key > 2 and nrms < s_factor * delta: 242 | beta = np.sqrt(delta**2 - nrms**2) 243 | s = s + reduce(np.dot, [beta, sig, V[:, jmin]]).reshape(n, 1) 244 | 245 | if key > 2 and nrms > b_factor * delta: 246 | b, c, count = rfzero(laminit, itbnd, eigval, alpha, delta, tol) 247 | lambda_0 = b 248 | lam = lambda_0 * np.ones((n, 1)) 249 | 250 | coeff, s, nrms, w = compute_step(alpha, eigval, coeff, V, lam) 251 | 252 | # return the model prediction of the change in the objective with s 253 | val = np.dot(g.T, s) + reduce(np.dot, [(.5*s).T, H, s]) 254 | 255 | return (s, val) 256 | 257 | def compute_step(alpha, eigval, coeff, V, lam): 258 | w = eigval + lam 259 | arg1 = np.logical_and(w == 0, alpha == 0) 260 | arg2 = np.logical_and(w == 0, alpha != 0) 261 | coeff[w != 0] = alpha[w != 0] / w[w != 0] 262 | coeff[arg1] = 0 263 | coeff[arg2] = np.inf 264 | coeff[np.isnan(coeff)] = 0 265 | s = np.dot(V, coeff) 266 | nrms = LA.norm(s) 267 | return(coeff, s, nrms, w) 268 | -------------------------------------------------------------------------------- /blackbox_opt/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /blackbox_opt/bb_optimize.py: -------------------------------------------------------------------------------- 1 | """ 2 | Description: This module calls five derivative free algorithms 3 | from the scipy.optimize library as well as the DFO algorithm whose 4 | source code is included with this package. The scipy algorithms are the 5 | Nelder-Mead, Powell, SLSQP, COBYLA and BFGS algorithms. 6 | 7 | Output: a result object returned from the algorithm 8 | If the result is from a scipy algorithm, then it contains the following 9 | attributes: 10 | x: ndarray 11 | The solution of the optimization. 12 | success: bool 13 | Whether or not the optimizer exited successfully. 14 | status: int Termination status of the optimizer. 15 | Its value depends on the underlying solver. Refer to message for details. 16 | message: str 17 | Description of the cause of the termination. 18 | fun, jac, hess, hess_inv: ndarray 19 | Values of objective function, 20 | Jacobian, Hessian or its inverse (if available). The Hessians 21 | may be approximations, see the documentation of the function in question. 22 | nfev, njev, nhev: int 23 | Number of evaluations of the objective 24 | functions and of its Jacobian and Hessian. 25 | nit: int 26 | Number of iterations performed by the optimizer. 27 | maxcv: float 28 | The maximum constraint violation. 29 | 30 | For the attributes of the result object returned by the DFO method see 31 | the README file in the DFO_src directory. 32 | 33 | author: Anahita Hassanzadeh 34 | email: Anahita.Hassanzadeh@gmail.com 35 | """ 36 | from scipy.optimize import minimize 37 | from .DFO_src.dfo_tr import dfo_tr 38 | import time 39 | 40 | def bb_optimize(func, x_0, alg, options=None): 41 | # start timing 42 | start = time.time() 43 | 44 | # call the specified algorithm with the given points and options. 45 | if alg.lower() == 'dfo': 46 | res = dfo_tr(func, x_0, options=options) 47 | else: 48 | res = minimize(func, x_0, method=alg, options=options) 49 | 50 | end = time.time() 51 | 52 | # print out the time and return the result object 53 | print "Total time is {} seconds with " .format(end - start) + alg + ( 54 | " method.") 55 | return res 56 | -------------------------------------------------------------------------------- /blackbox_opt/test_funcs/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /blackbox_opt/test_funcs/funcs_def.py: -------------------------------------------------------------------------------- 1 | """ 2 | Description: 3 | This module contains the definitions of Ackley's, sphere, 4 | Rosenbrock, Booth's, Bukin, Beale's functions. These functions are 5 | called as blackbox functions from the run_test_func.py module. 6 | 7 | These functions are popular global optimization test problems. 8 | 9 | Functions and their optimal solution: 10 | 11 | -Rosenbrock: n-dimensional 12 | smooth, non-convex function. The minumum lies on a narrow flat 13 | valley. Convergence to global optimum is non-trivial. 14 | global optimum: f(x*) = 0, x* = all variables equal to 1 15 | 16 | - sphere: n-dimensional 17 | smooth function with no local minumum. Convex. 18 | global optimum: f(x*) = 0, x* = all variables equal to 0 19 | 20 | - Ackley's: 2-dimensional 21 | highly nonconvex-nonconcave, nonsmooth problem with many 22 | local solutions. 23 | global optimum: f(x*) = 0, x* = (0,0) 24 | 25 | - Beale: 2-dimensional 26 | non-convex, problem with multiple local solutions. Sharp corners. 27 | global optimum: f(x*) = 0, x* = (3,0.5) 28 | 29 | - Booth's: 2-dimensional 30 | smooth convex function. 31 | global optimum: f(x*) = 0, x* = (1,3) 32 | 33 | - Bukin: 2-dimensional 34 | non-smooth, nonconvex function with deep and steep valleys. 35 | global optimum: f(x*) = 0, x* = (-10,1) 36 | 37 | 38 | input: 39 | x: numpy.ndarray 40 | the input to each function 41 | output: 42 | f(x): scalar 43 | the evaluation of the function at the given point x 44 | author: Anahita Hassanzadeh 45 | email: Anahita.Hassanzadeh@gmail.com 46 | """ 47 | import numpy as np 48 | def rosen(x): 49 | """The Rosenbrock function""" 50 | return sum(100.0 * (x[1:] - x[:-1]**2.0)**2.0 + (1 - x[:-1])**2.0) 51 | 52 | def sphere(x): 53 | """The sphere function""" 54 | res = sum(x**2) 55 | return res 56 | 57 | def ackley(x): 58 | """The Ackley's function""" 59 | res = -20 * np.exp(-0.2 * np.sqrt(0.5 * sum(x**2))) - ( 60 | - np.exp(0.5 * (sum(np.cos(2*np.pi*x))))) + np.e + 20 61 | return res 62 | 63 | def beale(x_0): 64 | """The Beale function""" 65 | x = x_0[0] 66 | y = x_0[1] 67 | res = (1.5 - x + x*y)**2 + (2.25 - x + x*y**2)**2 + ( 68 | (2.625 - x + x*y**3)**2) 69 | return res 70 | 71 | def booth(x_0): 72 | """The Booth function""" 73 | x = x_0[0] 74 | y = x_0[1] 75 | return ((x+2*y-7)**2 + (2*x+y-5)**2) 76 | 77 | def bukin(x_0): 78 | """The Bukin function""" 79 | x = x_0[0] 80 | y = x_0[1] 81 | return (100 * np.sqrt(abs(y - 0.01*x**2)) + 0.01 * (abs(x+10))) 82 | -------------------------------------------------------------------------------- /run_test_func.py: -------------------------------------------------------------------------------- 1 | """ 2 | Description: 3 | This module runs six popular test problems for global optimization 4 | that are defined and described in blackbox_opt/test_funcs/funcs_def.py. 5 | 6 | The user is required to import a blackbox function, specify the 7 | algorithm, a starting point and options. 8 | func: an imported blackbox function object 9 | x_0: starting point: numpy array with shape (n,1) --n is the 10 | dimension of x_0 11 | alg: selected algorithm: string 12 | options : a dictionary of options customized for each algorithm 13 | For a full list of options available for DFO see the README file in 14 | DFO_src directory. 15 | For scipy algorithms, options for each alg are available by a call to 16 | scipy.optimize.show_options. These options are available at 17 | http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.optimize.show_options.html 18 | author: Anahita Hassanzadeh 19 | email: Anahita.Hassanzadeh@gmail.com 20 | """ 21 | import numpy as np 22 | from blackbox_opt.bb_optimize import bb_optimize 23 | # import the function to be optimized 24 | from blackbox_opt.test_funcs.funcs_def import (ackley, sphere, rosen, 25 | booth, bukin, beale) 26 | def get_results(func, x_0, alg, options): 27 | """ 28 | This function calls the main blackbox optimization 29 | code with the specified algorithm, options and starting point and 30 | prints the best point found along with its optimal value. 31 | input: func: an imported blackbox function object 32 | x_0: starting point: numpy array with shape (n,1) 33 | alg: selected algorithm: string 34 | options : a dictionary of options customized for each algorithm 35 | """ 36 | res = bb_optimize(func, x_0, alg, options) 37 | 38 | print "Printing result for function " + func.__name__ + ":" 39 | print "best point: {}, with obj: {:.6f}".format( 40 | np.around(res.x.T, decimals=5), float(res.fun)) 41 | print "-------------" + alg + " Finished ----------------------\n" 42 | 43 | 44 | if __name__ == "__main__": 45 | # separate the functions based on their dimension. This is merely 46 | # done to ensure the starting point x_0 will later have the 47 | # correct dimension 48 | nd_func_names = [sphere, rosen] # functions from R^n -> R 49 | td_func_names = [ackley, booth, bukin, beale] # functions from R^2 -> R 50 | all_func_names = td_func_names + nd_func_names 51 | 52 | # Run all the algorithms and problems with given starting points 53 | # Specify the starting point and options. For example, try the following 54 | # options. 55 | for func in all_func_names: 56 | if func in td_func_names: 57 | x_0 = np.array([1.3, 0.7]) 58 | else: 59 | x_0 = np.array([1.3, 0.7, 0.8, 1.9, 1.2]) 60 | 61 | print "\n\n********* Function " + func.__name__ + "********" 62 | alg = "DFO" 63 | options = {"maxfev": 100, "init_delta": 20, 64 | "tol_delta": 1e-25, "tol_f": 1e-26, "tol_norm_g": 1e-5, 65 | "sample_gen": "auto", "verbosity": 0} 66 | get_results(func, x_0, alg, options) 67 | 68 | alg = "Powell" 69 | options = {"disp": True, "maxfev": 100, "ftol": 1e-26} 70 | get_results(func, x_0, alg, options) 71 | 72 | alg = "Nelder-Mead" 73 | options = {"disp": True, "maxfev": 100, "ftol": 1e-26} 74 | get_results(func, x_0, alg, options) 75 | 76 | alg = 'COBYLA' 77 | options = {"disp": True, "tol": 1e-25} 78 | get_results(func, x_0, alg, options) 79 | 80 | alg = 'BFGS' 81 | options = {"maxiter": 8, "disp": True, "gtol": 1e-5} 82 | get_results(func, x_0, alg, options) 83 | 84 | alg = 'SLSQP' 85 | options = {"maxiter": 20, "disp": True, "ftol": 1e-26} 86 | get_results(func, x_0, alg, options) 87 | --------------------------------------------------------------------------------