├── .gitattributes ├── .gitignore ├── Davidson-Fletcher-Powell.py ├── One-dimesional.py ├── PenaltyFunction.py ├── README.md ├── RosenbrockClassy.py ├── UsingDerivativesMethodsFirstAndSecondOrder.py ├── hook-jeeves.py └── rosenbrock.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | *.sln merge=union 7 | *.csproj merge=union 8 | *.vbproj merge=union 9 | *.fsproj merge=union 10 | *.dbproj merge=union 11 | 12 | # Standard to msysgit 13 | *.doc diff=astextplain 14 | *.DOC diff=astextplain 15 | *.docx diff=astextplain 16 | *.DOCX diff=astextplain 17 | *.dot diff=astextplain 18 | *.DOT diff=astextplain 19 | *.pdf diff=astextplain 20 | *.PDF diff=astextplain 21 | *.rtf diff=astextplain 22 | *.RTF diff=astextplain 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################# 2 | ## Eclipse 3 | ################# 4 | 5 | *.pydevproject 6 | .project 7 | .metadata 8 | bin/ 9 | tmp/ 10 | *.tmp 11 | *.bak 12 | *.swp 13 | *~.nib 14 | local.properties 15 | .classpath 16 | .settings/ 17 | .loadpath 18 | 19 | # External tool builders 20 | .externalToolBuilders/ 21 | 22 | # Locally stored "Eclipse launch configurations" 23 | *.launch 24 | 25 | # CDT-specific 26 | .cproject 27 | 28 | # PDT-specific 29 | .buildpath 30 | 31 | 32 | ################# 33 | ## Visual Studio 34 | ################# 35 | 36 | ## Ignore Visual Studio temporary files, build results, and 37 | ## files generated by popular Visual Studio add-ons. 38 | 39 | # User-specific files 40 | *.suo 41 | *.user 42 | *.sln.docstates 43 | 44 | # Build results 45 | 46 | [Dd]ebug/ 47 | [Rr]elease/ 48 | x64/ 49 | build/ 50 | [Bb]in/ 51 | [Oo]bj/ 52 | 53 | # MSTest test Results 54 | [Tt]est[Rr]esult*/ 55 | [Bb]uild[Ll]og.* 56 | 57 | *_i.c 58 | *_p.c 59 | *.ilk 60 | *.meta 61 | *.obj 62 | *.pch 63 | *.pdb 64 | *.pgc 65 | *.pgd 66 | *.rsp 67 | *.sbr 68 | *.tlb 69 | *.tli 70 | *.tlh 71 | *.tmp 72 | *.tmp_proj 73 | *.log 74 | *.vspscc 75 | *.vssscc 76 | .builds 77 | *.pidb 78 | *.log 79 | *.scc 80 | 81 | # Visual C++ cache files 82 | ipch/ 83 | *.aps 84 | *.ncb 85 | *.opensdf 86 | *.sdf 87 | *.cachefile 88 | 89 | # Visual Studio profiler 90 | *.psess 91 | *.vsp 92 | *.vspx 93 | 94 | # Guidance Automation Toolkit 95 | *.gpState 96 | 97 | # ReSharper is a .NET coding add-in 98 | _ReSharper*/ 99 | *.[Rr]e[Ss]harper 100 | 101 | # TeamCity is a build add-in 102 | _TeamCity* 103 | 104 | # DotCover is a Code Coverage Tool 105 | *.dotCover 106 | 107 | # NCrunch 108 | *.ncrunch* 109 | .*crunch*.local.xml 110 | 111 | # Installshield output folder 112 | [Ee]xpress/ 113 | 114 | # DocProject is a documentation generator add-in 115 | DocProject/buildhelp/ 116 | DocProject/Help/*.HxT 117 | DocProject/Help/*.HxC 118 | DocProject/Help/*.hhc 119 | DocProject/Help/*.hhk 120 | DocProject/Help/*.hhp 121 | DocProject/Help/Html2 122 | DocProject/Help/html 123 | 124 | # Click-Once directory 125 | publish/ 126 | 127 | # Publish Web Output 128 | *.Publish.xml 129 | *.pubxml 130 | 131 | # NuGet Packages Directory 132 | ## TODO: If you have NuGet Package Restore enabled, uncomment the next line 133 | #packages/ 134 | 135 | # Windows Azure Build Output 136 | csx 137 | *.build.csdef 138 | 139 | # Windows Store app package directory 140 | AppPackages/ 141 | 142 | # Others 143 | sql/ 144 | *.Cache 145 | ClientBin/ 146 | [Ss]tyle[Cc]op.* 147 | ~$* 148 | *~ 149 | *.dbmdl 150 | *.[Pp]ublish.xml 151 | *.pfx 152 | *.publishsettings 153 | 154 | # RIA/Silverlight projects 155 | Generated_Code/ 156 | 157 | # Backup & report files from converting an old project file to a newer 158 | # Visual Studio version. Backup files are not needed, because we have git ;-) 159 | _UpgradeReport_Files/ 160 | Backup*/ 161 | UpgradeLog*.XML 162 | UpgradeLog*.htm 163 | 164 | # SQL Server files 165 | App_Data/*.mdf 166 | App_Data/*.ldf 167 | 168 | ############# 169 | ## Windows detritus 170 | ############# 171 | 172 | # Windows image file caches 173 | Thumbs.db 174 | ehthumbs.db 175 | 176 | # Folder config file 177 | Desktop.ini 178 | 179 | # Recycle Bin used on file shares 180 | $RECYCLE.BIN/ 181 | 182 | # Mac crap 183 | .DS_Store 184 | 185 | 186 | ############# 187 | ## Python 188 | ############# 189 | 190 | *.py[co] 191 | 192 | # Packages 193 | *.egg 194 | *.egg-info 195 | dist/ 196 | build/ 197 | eggs/ 198 | parts/ 199 | var/ 200 | sdist/ 201 | develop-eggs/ 202 | .installed.cfg 203 | 204 | # Installer logs 205 | pip-log.txt 206 | 207 | # Unit test / coverage reports 208 | .coverage 209 | .tox 210 | 211 | #Translations 212 | *.mo 213 | 214 | #Mr Developer 215 | .mr.developer.cfg 216 | -------------------------------------------------------------------------------- /Davidson-Fletcher-Powell.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Apr 21 13:46:36 2014 4 | @author: Alex 5 | 6 | Davidson-Fletcher-Powell optimization algorithm 7 | Uses Davies-Svenn-Campy-Powell for one-dimensional search 8 | """ 9 | import math 10 | import numpy as np 11 | import matplotlib.pylab as pl 12 | 13 | count = 0 14 | 15 | def incCount(): 16 | global count 17 | count = count+1 18 | 19 | #done 20 | def calcQuasiX(x0, A, grad, lmb): 21 | ''' 22 | x0 - gradient*lambda 23 | ''' 24 | lmbA = np.dot(lmb, A) 25 | lmbAgrad = np.dot(lmbA, grad) 26 | calc = x0 - lmbAgrad 27 | return calc 28 | 29 | #done 30 | def svennQuasi(x0, grad, lmb, delta, A): 31 | """ 32 | One-dimensional Svenn search 33 | """ 34 | #print "Svenn stage..." 35 | f0 = fun(calcQuasiX(x0, A, grad, lmb)) 36 | if f0 < fun(calcQuasiX(x0, A, grad, lmb+delta)): 37 | delta = -delta 38 | x1 = lmb + delta 39 | f1 = fun(calcQuasiX(x0, A, grad, x1)) 40 | while f1 < f0: 41 | delta *= 2 42 | lmb = x1 43 | x1 = lmb + delta 44 | f0 = f1 45 | f1 = fun(calcQuasiX(x0, A, grad, x1)) 46 | a = lmb + delta/2 47 | b = lmb - delta/2 48 | f0 = fun(calcQuasiX(x0, A, grad, lmb)) 49 | f1 = fun(calcQuasiX(x0, A, grad, b)) 50 | 51 | if f0 < f1: 52 | if a < b: 53 | return [a, b] 54 | else: 55 | return [b, a] 56 | elif f1 < f0: 57 | if lmb < x1: 58 | return [lmb, x1] 59 | else: 60 | return [x1, lmb] 61 | else: 62 | if lmb < b: 63 | return [lmb, b] 64 | else: 65 | return [b, lmb] 66 | 67 | 68 | def dscQuasi(x0, grad, lmb, delta, A): 69 | svenn_res = svennQuasi(x0, grad, lmb, delta, A) 70 | x1 = svenn_res[0] 71 | x3 = svenn_res[1] 72 | x2 = (x1 + x3)/2 73 | f1 = fun(calcQuasiX(x0, A, grad, x1)) 74 | f2 = fun(calcQuasiX(x0, A, grad, x2)) 75 | f3 = fun(calcQuasiX(x0, A, grad, x3)) 76 | xApprox = x2 + ((x3 - x2) * (f1 - f3)) / (2 * (f1 - 2 * f2 + f3)) 77 | return [x1, x2, x3, xApprox] 78 | 79 | 80 | def dscPowellQuasi(x0, grad, eps, lmb, delta, A): 81 | dsc_res = dscQuasi(x0, grad, lmb, delta, A) 82 | a = dsc_res[0] 83 | xmin = dsc_res[1] 84 | b = dsc_res[2] 85 | xApprox = dsc_res[3] 86 | 87 | while abs(xmin-xApprox) >= eps or abs(fun(calcQuasiX(x0, A, grad, xmin)) - fun(calcQuasiX(x0, A, grad, xApprox))) >= eps: 88 | if xApprox < xmin: 89 | b = xmin 90 | else: 91 | a = xmin 92 | xmin = xApprox 93 | funcRes = [fun(calcQuasiX(x0, A, grad, a)), fun(calcQuasiX(x0, A, grad, xmin)), fun(calcQuasiX(x0, A, grad, b))] 94 | a1 = (funcRes[1] - funcRes[0]) / (xmin - a) 95 | a2 = ((funcRes[2] - funcRes[0]) / (b - a) - a1) / (b - xmin) 96 | xApprox = (a + xmin) / 2 - a1 / (2 * a2) 97 | return xmin 98 | 99 | #done 100 | def goldQuasi(a, b, eps, x0, grad, A): 101 | """ 102 | One-dimensional gold search 103 | """ 104 | l = sub(b,a) 105 | x1 = add(a, mults(l, 0.382)) 106 | x2 = add(a, mults(l, 0.618)) 107 | while l > eps: 108 | if fun(calcQuasiX(x0, A, grad, x1)) < fun(calcQuasiX(x0, A, grad, x2)): 109 | b = x2 110 | x2 = x1 111 | l = sub(b,a) 112 | x1 = add(a, mults(l, 0.382)) 113 | print "gold a: " + str(a) 114 | print "gold b: " + str(b) 115 | else: 116 | a = x1 117 | x1 = x2 118 | l = sub(b,a) 119 | x2 = add(a, mults(l, 0.618)) 120 | print "gold a: " + str(a) 121 | print "gold b: " + str(b) 122 | 123 | return [a, b] 124 | 125 | #done 126 | def calcLambdaQuasi(x0, grad, eps, lmb, A): 127 | line = svennQuasi(x0, grad, lmb, 0.1, A) 128 | line = goldQuasi(line[0], line[1], eps, x0, grad, A) 129 | lmb = (line[0] + line[1])/2 130 | return lmb 131 | 132 | 133 | def plot(points, col): 134 | n = 256 135 | x = np.linspace(-12, 12, n) 136 | y = np.linspace(-12, 12, n) 137 | X, Y = np.meshgrid(x, y) 138 | 139 | xs = [] 140 | ys = [] 141 | 142 | pl.contourf(X, Y, fun([X, Y]), 8, alpha=.75, cmap='jet') 143 | pl.contour(X, Y, fun([X, Y]), 8, colors='black', linewidth=.5) 144 | 145 | for i in range(len(points)): 146 | xs.append(points[i][0]) 147 | ys.append(points[i][1]) 148 | 149 | pl.plot(xs, ys, marker='o', linestyle='--', color=str(col), label='Square') 150 | 151 | def add(x, y): 152 | res = [] 153 | for i in xrange(len(x)): 154 | res.append(x[i] + y[i]) 155 | return res 156 | 157 | def sub(x, y): 158 | res = [] 159 | for i in xrange(len(x)): 160 | res.append(x[i] - y[i]) 161 | return res 162 | 163 | def mults(x, n): 164 | res = [] 165 | for i in xrange(len(x)): 166 | res.append(x[i]*n) 167 | return res 168 | 169 | def derivative(x, n): 170 | h = [] 171 | for i in xrange(len(x)): 172 | if i == n: 173 | h.append(1e-5) 174 | else: 175 | h.append(0) 176 | return (fun([x[0] + h[0], x[1] + h[1]]) - fun([x[0] - h[0], x[1] - h[1]]))/(2*h[n]) 177 | #return (fun([x[0] + h[0], x[1] + h[1]]) - fun(x)/(h[n])) 178 | #return (fun(x) - fun([x[0] - h[0], x[1] - h[1]]))/(h[n]) # разностная схема назад - ок 179 | 180 | 181 | 182 | def derivative2(x, a, b): 183 | ai = [] 184 | aj = [] 185 | for i in xrange(len(x)): 186 | if i == a: 187 | ai.append(0.001) 188 | else: 189 | ai.append(0) 190 | for j in xrange(len(x)): 191 | if j == b: 192 | aj.append(0.001) 193 | else: 194 | aj.append(0) 195 | return (fun(add(x, add(ai, aj))) - fun(add(x, ai)) - fun(add(x, aj)) + fun(x))/ (ai[a]**2) 196 | 197 | def gradient(x): 198 | grad = [] 199 | for i in xrange(len(x)): 200 | grad.append(derivative(x, i)) 201 | return grad 202 | 203 | def norm(s1): 204 | normas = 0 205 | for i in xrange(len(s1)): 206 | normas += s1[i]**2 207 | return math.sqrt(normas) 208 | 209 | def hesse(x): 210 | h = [] 211 | for i in xrange(len(x)): 212 | for j in xrange(len(x)): 213 | h.append(derivative2(x, i, j)) 214 | return h 215 | 216 | def fun(x): 217 | incCount() 218 | #return (x[0] - 6)**2 - x[0]*x[1] + 3*x[1]**2 219 | ##return 4*(x[0]-5)**2 + (x[1] - 6)**2 220 | #return (x[0]-1)**2 + 100*(x[0]**2 - x[1])**2 221 | #return (10*(x[0] - x[1])**2 + (x[0] - 1)**2)**4 222 | #return (10*(x[0] - x[1])**2 + (x[0] - 1)**2)**(1/4) 223 | #return 8*x[0]**2 + 4*x[0]*x[1] + 5*x[1]**4 224 | #return (x[1]**3 + x[0]**2-3*x[1])**4 - (x[0]**3-x[1]**2-3*x[0]) - x[1]**2 + 3 225 | return x[0]**3 + x[1]**3 - 3*x[0]*x[1] 226 | 227 | def dfp(x0, eps1, eps2): 228 | restart = 0 229 | iteration = 0 230 | xs = [] 231 | xs.append(x0) 232 | lmb = 0.01 233 | A = np.eye(len(x0)) 234 | 235 | print x0 236 | print "------------------------" 237 | while True: 238 | 239 | grad = gradient(x0) 240 | 241 | if iteration > 0: 242 | if norm(grad) < eps1: 243 | print "=========================" 244 | print "break" 245 | print "FUNCTIONS COUNT" 246 | print count 247 | print "ITERATIONS" 248 | print iteration 249 | print "RESTARTS" 250 | print restart 251 | plot(xs, 'red') 252 | break 253 | 254 | lmb = dscPowellQuasi(x0, grad, eps2, lmb, 0.01, A) 255 | 256 | if lmb < 0: 257 | print "RESTART" 258 | A = np.eye(len(x0)) 259 | restart = restart+1 260 | 261 | x1 = calcQuasiX(x0, A, grad, lmb) 262 | 263 | deltag = np.array(sub(gradient(x1), gradient(x0)))[np.newaxis] 264 | deltax = sub(x1, x0) 265 | 266 | 267 | deltax = np.array(deltax)[np.newaxis] 268 | deltaxT = np.array(deltax).T 269 | deltagT = np.array(deltag).T 270 | 271 | # А теперь делаем так, чтобы работало на питоне :) 272 | 273 | deltax, deltaxT = deltaxT, deltax 274 | deltag, deltagT = deltagT, deltag 275 | 276 | first = np.dot(deltax, deltaxT) 277 | second = np.dot(deltaxT,deltag) 278 | resOne = first/second 279 | #print resOne 280 | 281 | first = np.dot(A, deltag) 282 | second = np.dot(first, deltagT) 283 | third = np.dot(second, A) 284 | temp = third 285 | 286 | first = np.dot(deltagT, A) 287 | second = np.dot(first, deltag) 288 | 289 | resTwo = temp/second 290 | #print resTwo 291 | 292 | A = A + resOne - resTwo 293 | #print "MATRIX A" 294 | #print A 295 | 296 | x0 = x1 297 | xs.append(x0) 298 | print "POINT" 299 | print x1 300 | print "FUNCTIONS COUNT" 301 | print count 302 | print "LAMBDA" 303 | print lmb 304 | print "INV A" 305 | print np.linalg.inv(A) 306 | #print "HESSE" 307 | #print hesse(x1) 308 | print "------------------------" 309 | iteration +=1 310 | 311 | print "==============================" 312 | print x1 313 | 314 | 315 | def main(): 316 | dfp([11, 5], 0.01, 0.01) 317 | 318 | 319 | 320 | 321 | if __name__ == '__main__': 322 | main() 323 | 324 | 325 | -------------------------------------------------------------------------------- /One-dimesional.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Little pack for 1-dimensional optimization algorithms 3 | NOTE: Svenn algorithm is a first step of DSC-algoritm 4 | ''' 5 | 6 | __author__ = 'Alex Gonchar' 7 | 8 | import numpy as np 9 | import matplotlib.pylab as plt 10 | from sympy import * 11 | 12 | 13 | def fun(x): 14 | ''' 15 | Input your symbolic function here 16 | ''' 17 | #return x*(2*x - 3) 18 | return x**2 + 2/x 19 | #return (100-x)**2 20 | #return x**2 - 6*x 21 | 22 | 23 | def deriv(point): 24 | ''' 25 | Computes derivative in a point 26 | ''' 27 | x = Symbol('x') 28 | y = fun(x) 29 | #print str(y) 30 | d = y.diff() 31 | #print str(d) 32 | f = lambdify(x, d, 'numpy') 33 | return f(point) 34 | 35 | 36 | def deriv2(point): 37 | x = Symbol('x') 38 | y = deriv(x) 39 | d = y.diff() 40 | #print str(d) 41 | f = lambdify(x, d, 'numpy') 42 | return f(point) 43 | 44 | 45 | def plotFun(): 46 | ''' 47 | Helpful function for plotting your function 48 | ''' 49 | x = np.arange(-5,5) 50 | plt.plot(x, fun(x)) 51 | 52 | 53 | def svenn(x0, delta): 54 | print "Starting Svenn algorithm..." 55 | x1 = x0 56 | x2 = x1+delta 57 | if fun(x1) < fun(x2): 58 | delta = -delta 59 | x1 = x0 60 | x2 = x1+delta 61 | while fun(x2) < fun(x1): 62 | delta*=2 63 | x1 = x2 64 | x2 = x1+delta 65 | print "x1: " + str(x1) + " , f(x1): " + str(fun(x1)) 66 | print "x2: " + str(x2) + " , f(x2): " + str(fun(x2)) 67 | a = x2 - 3*delta/2 68 | b = x2 - delta/2 69 | if a= eps and abs(fun(xmin) - fun(xs)) >= eps: 106 | if xs < xmin: 107 | b = xmin 108 | xmin = xs 109 | elif xs > xmin: 110 | a = xmin 111 | xmin = xs 112 | else: 113 | break 114 | 115 | f = [fun(a), fun(xmin), fun(b)] 116 | print "a = %8.5f, min = %8.5f b = %8.5f" % (a, xmin, b) 117 | a1 = (f[1] - f[0]) / (xmin - a) 118 | a2 = ((f[2] - f[0]) / (b - a) - a1) / (b - xmin) 119 | 120 | xs = (a + xmin)/2 - a1/(2*a2) 121 | print "x* = %8.5f" % xs 122 | 123 | f_min = min(f) 124 | if f[0] == f_min: 125 | xmin = a 126 | elif f[1] == f_min: 127 | xmin = xmin 128 | else: 129 | xmin = b 130 | print "a = %8.5f, min = %8.5f b = %8.5f" % (a, xmin, b) 131 | 132 | 133 | def dix(a, b, delta): 134 | print "Starting dichotomy method..." 135 | while round(abs(b-a), 3) > abs(delta): 136 | x = (a + b - delta)/2 137 | y = (a + b + delta)/2 138 | print "i x: %8.5f, y: %8.5f, L: %8.5f" % (x, y, abs(b-a)) 139 | if fun(x) < fun(y): 140 | b = y 141 | else: 142 | a = x 143 | print "[%8.5f; %8.5f] = %8.5f" % (a, b, (a+b)/2) 144 | 145 | 146 | def gold(a, b, delta): 147 | print "Starting gold search..." 148 | l = b - a 149 | x = a + 0.382*l 150 | y = a + 0.618*l 151 | while abs(b-a) >= delta: 152 | if (fun(x) < fun(y)): 153 | print "fun(x1) in %8.5f: %8.5f < fun(x2) in %8.5f: %8.5f" % (x, fun(x), y, fun(y)) 154 | b = y 155 | y = x 156 | x = a + b - y 157 | print "a: %8.5f; b: %8.5f; L: %8.5f" % (a, b, (b-a)) 158 | else: 159 | print "fun(x1) in %8.5f: %8.5f > fun(x2) in %8.5f: %8.5f" % (x, fun(x), y, fun(y)) 160 | a = x 161 | x = y 162 | y = a + b - x 163 | print "a: %8.5f; b: %8.5f; L: %8.5f" % (a, b, (b-a)) 164 | print "[%8.5f; %8.5f] => %8.5f" % (a, b, (a+b)/2) 165 | 166 | 167 | def bolzano(a, b, delta): 168 | print "Starting Bolzano method..." 169 | c = (a+b)/2 170 | print "a = %8.5f, b = %8.5f, c = %8.5f, f'(%8.5f) = %8.5f" % (a, b, c, c, deriv(c)) 171 | while (abs(deriv(c)) > delta): 172 | if deriv(c) < 0: 173 | a = c 174 | c = (a+b)/2 175 | print "a = %8.5f, b = %8.5f, c = %8.5f, f'(%8.5f) = %8.5f" % (a, b, c, c, deriv(c)) 176 | else: 177 | b = c 178 | c = (a+b)/2 179 | print "a = %8.5f, b = %8.5f, c = %8.5f, f'(%8.5f) = %8.5f" % (a, b, c, c, deriv(c)) 180 | print "Answer: a = %8.5f, b = %8.5f" % (a, b) 181 | 182 | 183 | def newton(a, delta): 184 | xn = a 185 | xn1 = xn - deriv(xn)/deriv2(xn) 186 | while abs(deriv(xn)) >= delta: 187 | xn = xn1 188 | xn1 = xn - deriv(xn)/deriv2(xn) 189 | print "xk = %8.5f, f'(xk) = %8.5f" % (xn, deriv(xn)) 190 | 191 | 192 | def chords(a, b, eps): 193 | ''' 194 | thanks to @FlyingPirate 195 | ''' 196 | xApprox = b - ((deriv(b) * (b - a)) / (deriv(b) - deriv(a))) 197 | while deriv(xApprox) > eps: 198 | if signum(deriv(xApprox)) * signum(deriv(a)) > 0: 199 | a = xApprox 200 | else: 201 | b = xApprox 202 | print "[%8.5f, %8.5f]" % (a, b) 203 | xApprox = b - ((deriv(b) * (b - a)) / (deriv(b) - deriv(a))) 204 | return xApprox 205 | 206 | 207 | def signum(x): 208 | if x < 0: 209 | return -1 210 | elif x == 0: 211 | return 0 212 | else: 213 | return 1 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | -------------------------------------------------------------------------------- /PenaltyFunction.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sat May 17 16:35:36 2014 4 | 5 | @author: Alex 6 | """ 7 | 8 | import math 9 | import numpy as np 10 | import matplotlib.pylab as pl 11 | 12 | 13 | count = 0 14 | r = 1 15 | 16 | def incCount(): 17 | global count 18 | count = count+1 19 | 20 | def zeroCount(): 21 | global count 22 | count = 0 23 | 24 | 25 | def changeR(value): 26 | global r 27 | r = value 28 | 29 | 30 | def calcX(x0, grad, lmb): 31 | ''' 32 | x0 - gradient*lambda 33 | ''' 34 | return sub(x0, mults(grad, lmb)) 35 | 36 | def svenn(x0, grad, lmb, delta): 37 | """ 38 | One-dimensional Svenn search 39 | """ 40 | #print "Svenn stage..." 41 | f0 = fun(calcX(x0, grad, lmb)) 42 | if f0 < fun(calcX(x0, grad, lmb+delta)): 43 | delta = -delta 44 | x1 = lmb + delta 45 | f1 = fun(calcX(x0, grad, x1)) 46 | while f1 < f0: 47 | delta *= 2 48 | lmb = x1 49 | x1 = lmb + delta 50 | f0 = f1 51 | f1 = fun(calcX(x0, grad, x1)) 52 | a = lmb + delta/2 53 | b = lmb - delta/2 54 | if a > b: 55 | temp = b 56 | b = a 57 | a = temp 58 | #print "svenn a: " + str(a) 59 | #print "svenn b: " + str(b) 60 | return [a , b] 61 | 62 | 63 | def dsc(x0, grad, lmb, delta): 64 | svenn_res = svenn(x0, grad, lmb, delta) 65 | x1 = svenn_res[0] 66 | x3 = svenn_res[1] 67 | x2 = (x1 + x3)/2 68 | f1 = fun(calcX(x0, grad, x1)) 69 | f2 = fun(calcX(x0, grad, x2)) 70 | f3 = fun(calcX(x0, grad, x3)) 71 | xApprox = x2 + ((x3 - x2) * (f1 - f3)) / (2 * (f1 - 2 * f2 + f3)) 72 | return [x1, x2, x3, xApprox] 73 | 74 | 75 | def dscPowell(x0, grad, eps, lmb, delta): 76 | dsc_res = dsc(x0, grad, lmb, delta) 77 | a = dsc_res[0] 78 | xmin = dsc_res[1] 79 | b = dsc_res[2] 80 | xApprox = dsc_res[3] 81 | 82 | while abs(xmin-xApprox) >= eps or abs(fun(calcX(x0, grad, xmin)) - fun(calcX(x0, grad, xApprox))) >= eps: 83 | if xApprox < xmin: 84 | b = xmin 85 | else: 86 | a = xmin 87 | xmin = xApprox 88 | funcRes = [fun(calcX(x0, grad, a)), fun(calcX(x0, grad, xmin)), fun(calcX(x0, grad, b))] 89 | a1 = (funcRes[1] - funcRes[0]) / (xmin - a) 90 | a2 = ((funcRes[2] - funcRes[0]) / (b - a) - a1) / (b - xmin) 91 | xApprox = (a + xmin) / 2 - a1 / (2 * a2) 92 | return xmin 93 | 94 | def gold(a, b, eps, x0, grad): 95 | """ 96 | One-dimensional gold search 97 | """ 98 | l = b - a 99 | x1 = a + 0.382*l 100 | x2 = a + 0.618*l 101 | while l > eps: 102 | if fun(calcX(x0, grad, x1)) < fun(calcX(x0, grad, x2)): 103 | b = x2 104 | x2 = x1 105 | l = b - a 106 | x1 = a + 0.382*l 107 | else: 108 | a = x1 109 | x1 = x2 110 | l = b - a 111 | x2 = a + 0.618*l 112 | print "gold a: " + str(a) 113 | print "gold b: " + str(b) 114 | return [a, b] 115 | 116 | 117 | def calcLambda(x0, grad, eps, lmb): 118 | line = svenn(x0, grad, lmb, 0.1) 119 | line = gold(line[0], line[1], eps, x0, grad) 120 | lmb = (line[0] + line[1])/2 121 | return lmb 122 | 123 | def plot3D(points, col): 124 | n = 256 125 | x = np.linspace(-100, 100, n) 126 | y = np.linspace(-100, 100, n) 127 | z = np.linspace(-100, 100, n) 128 | X, Y, Z = np.meshgrid(x, y, z) 129 | 130 | xs = [] 131 | ys = [] 132 | zs = [] 133 | 134 | #pl.contourf(X, Y, Z, fun([X, Y, Z]), 8, alpha=.75, cmap='jet') 135 | #C = pl.contour(X, Y, Z, fun([X, Y, Z]), 8, colors='black', linewidth=.5) 136 | 137 | for i in range(len(points)): 138 | xs.append(points[i][0]) 139 | ys.append(points[i][1]) 140 | zs.append(points[i][2]) 141 | 142 | pl.plot(xs, ys, marker='o', linestyle='--', color=str(col), label='Square') 143 | 144 | def plot(points, col): 145 | n = 256 146 | x = np.linspace(-100, 100, n) 147 | y = np.linspace(-100, 100, n) 148 | X, Y = np.meshgrid(x, y) 149 | 150 | xs = [] 151 | ys = [] 152 | 153 | pl.contourf(X, Y, fun([X, Y]), 8, alpha=.75, cmap='jet') 154 | pl.contour(X, Y, fun([X, Y]), 8, colors='black', linewidth=.5) 155 | 156 | for i in range(len(points)): 157 | xs.append(points[i][0]) 158 | ys.append(points[i][1]) 159 | 160 | pl.plot(xs, ys, marker='o', linestyle='--', color=str(col), label='Square') 161 | 162 | def add(x, y): 163 | res = [] 164 | for i in xrange(len(x)): 165 | res.append(x[i] + y[i]) 166 | return res 167 | 168 | def sub(x, y): 169 | res = [] 170 | for i in xrange(len(x)): 171 | res.append(x[i] - y[i]) 172 | return res 173 | def mults(x, n): 174 | res = [] 175 | for i in xrange(len(x)): 176 | res.append(x[i]*n) 177 | return res 178 | 179 | def derivative(x, n): 180 | h = [] 181 | for i in xrange(len(x)): 182 | if i == n: 183 | h.append(0.000000000001) 184 | else: 185 | h.append(0) 186 | return (fun([x[0] + h[0], x[1] + h[1]]) - fun([x[0] - h[0], x[1] - h[1]]))/(2*h[n]) 187 | 188 | def derivative2(x, a, b): 189 | ai = [] 190 | aj = [] 191 | for i in xrange(len(x)): 192 | if i == a: 193 | ai.append(0.001) 194 | else: 195 | ai.append(0) 196 | for j in xrange(len(x)): 197 | if j == b: 198 | aj.append(0.001) 199 | else: 200 | aj.append(0) 201 | return (fun(add(x, add(ai, aj))) - fun(add(x, ai)) - fun(add(x, aj)) + fun(x))/ (ai[a]**2) 202 | 203 | def gradient(x): 204 | grad = [] 205 | for i in xrange(len(x)): 206 | grad.append(derivative(x, i)) 207 | return grad 208 | 209 | def norm(s1): 210 | normas = 0 211 | for i in xrange(len(s1)): 212 | normas += s1[i]**2 213 | return math.sqrt(normas) 214 | 215 | def hesse(x): 216 | h = [] 217 | for i in xrange(len(x)): 218 | for j in xrange(len(x)): 219 | h.append(derivative2(x, i, j)) 220 | return h 221 | 222 | 223 | def fun(x): 224 | incCount() 225 | return ourFun(x) + constraintFun(x, r) 226 | 227 | def ourFun(x): 228 | """ 229 | Целевая функция для минимизации 230 | """ 231 | #return (x[0] - 4)**2 + (x[1] - 4)**2 232 | #return 4*(x[0]-5)**2 + (x[1] - 6)**2 233 | #return (1-x[0])**2 + 100*(x[1] - x[0]**2)**2 234 | return (10*(x[0] - x[1])**2 + (x[0] - 1)**2)**4 235 | #return 4*(x[0]-4)**2 + x[0]*x[1] + 3*x[1]**2 236 | 237 | 238 | def constraintFun(x, r): 239 | """ 240 | Часть функция для минимизации, где мы работаем с ограничениями 241 | """ 242 | line1 = 3*(x[1] - 1)**2 + (x[0] - 1)**2 - 0.5 243 | line2 = (x[1] - 1)**2 244 | return r*sqrtCut(line1)**2 + r*sqrtCut(line2)**2 245 | #return 0.5*(line + abs(line))**2 246 | #return r*(1/line) 247 | 248 | def sqrtCut(x): 249 | """ 250 | Квадрат срезки 251 | """ 252 | if x <= 0: 253 | return x 254 | else: 255 | return 0 256 | 257 | def conjugatedDirectionPowell(x0, eps, eps2): 258 | zeroCount() 259 | iteration = 0 260 | si = np.eye(len(x0)) 261 | xn = mults(x0, -1) 262 | xs = [] 263 | xs.append(x0) 264 | while norm(sub(xn, x0)) > 0.1 * eps: 265 | iteration+=1 266 | x0 = xn 267 | xi = x0 268 | fi = [] 269 | 270 | for i in xrange(len(x0)): 271 | lmb = dscPowell(xi, si[i], eps2, 0, 0.1) 272 | xn = calcX(xi, si[i], lmb) 273 | fi.append(fun(xi) - fun(xn)) 274 | xi = xn 275 | for i in xrange(len(si)-1): 276 | si[i] = si[i+1] 277 | 278 | si[len(si)-1] = sub(xn, x0) 279 | lmb = dscPowell(xn, si[len(si) - 1], eps2, 0, 0.1) 280 | xn = calcX(xn, si[len(si) - 1], lmb) 281 | #print xn 282 | xs.append(xn) 283 | #plot(xs, 'red') 284 | #print "calculations: " + str(count) 285 | return xn 286 | 287 | def main(): 288 | point = [0.8,0.7] 289 | minimum = conjugatedDirectionPowell(point, 0.01, 0.01) 290 | for i in range(10): 291 | minimum = conjugatedDirectionPowell(point, 0.001, 0.001) 292 | print "penalty: " + str(constraintFun(minimum, r) ) + " R = " + str(r) 293 | changeR(r*10) 294 | point = minimum 295 | print point 296 | print ourFun(minimum) 297 | #print "Count " + str(count) 298 | 299 | #line = [] 300 | #for i in xrange(-10, 10): 301 | # line.append([i, cf([i, i])]) 302 | #plot(line, 'green') 303 | 304 | if __name__ == "__main__": 305 | main() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Optimization Algorithms 2 | =================== 3 | 4 | Hello! 5 | Here you can find a collection of optimization methods implemented in Python. 6 | It was created while studying at KPI, so as university task it works with 2D functions only, but you can easily change it. 7 | 8 | One-dimensional methods: 9 | - Svenn algorithm 10 | - Davies-Svenn-Campy algorithm 11 | - DSC-Powell algorithm 12 | - Gold search 13 | - Dichotomy method 14 | - Bolzano method 15 | - Newton search 16 | - Chords method 17 | 18 | Zero-dimensional methods (no derivative required): 19 | - Hook-Jeeves algorithm 20 | - Rozenbrock algorithm 21 | 22 | Methods using derivative: 23 | - Gradient descent 24 | - Fastest descent (Cauchy method) 25 | - Advanced partan (parallel tangents) algorithm 26 | - Newton method (requires second order derivative) 27 | - Fletcher-Reeves method 28 | - Conjugated directions algorithm 29 | 30 | Other methods: 31 | - Davidson-Fletcher-Powell method 32 | -------------------------------------------------------------------------------- /RosenbrockClassy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Mar 13 19:25:20 2014 4 | 5 | Rosenbrock's optimization algorithm with visualization 6 | Some helpful outputs for better intuition are commented ('#') 7 | 8 | @author: Alex 9 | """ 10 | 11 | import numpy as np 12 | import math 13 | import matplotlib.pylab as pl 14 | 15 | count = 0 16 | 17 | def norm(s1): 18 | normas = 0 19 | for i in xrange(len(s1)): 20 | normas += s1[i]**2 21 | return math.sqrt(normas) 22 | 23 | def incCount(): 24 | global count 25 | count = count+1 26 | 27 | def perpendicular(a): 28 | b = np.empty_like(a) 29 | b[0] = -a[1] 30 | b[1] = a[0] 31 | return b 32 | 33 | 34 | def fun(x): 35 | incCount() 36 | #return 4*(x[0]-5)**2 + (x[1]-6)**2 37 | #return (x[0] - variant)**2 - x[0]*x[1] + 3*x[1]**2 38 | #return (x[0] - 3)**2 + x[0] * x[1] + 3 *x[1]**2 39 | #return (1-x[0])**2 + 100*(x[1] - x[0]**2)**2 40 | return (10*(x[0] - x[1])**2 + (x[0] - 1)**2)**4 41 | 42 | def constraintFunCircle(x): 43 | ''' 44 | Все ограничения подаем в виде >=0 и меняем знаки 45 | ''' 46 | return -(x[0]+1.2)**2 - (x[1])**2 + 1 47 | 48 | 49 | def constraintFunLine(x): 50 | """ 51 | Не меняем знак??? 52 | """ 53 | return 1 - x[0] - 2*x[1] 54 | 55 | 56 | def constraintFunFignya(x): 57 | return -(x[1]**3 + x[0]**2-3*x[1])**4 - (x[0]**3-x[1]**2-3*x[0]) - x[1]**2 + 3 58 | 59 | def outCircle(x): 60 | return -(x[0]-5)**2 - (x[1]-5)**2 + 10 61 | 62 | def innerCircle(x): 63 | return 2*(x[0]+0.5)**2 + (x[1]+0.7)**2 - 1 64 | 65 | def plot(points): 66 | ''' 67 | Plotting 2D function and way search 68 | ''' 69 | n = 256 70 | x = np.linspace(-12, 12, n) 71 | y = np.linspace(-12, 12, n) 72 | X, Y = np.meshgrid(x, y) 73 | 74 | xs = [] 75 | ys = [] 76 | 77 | pl.contourf(X, Y, fun([X, Y]), 8, alpha=.75, cmap='jet') 78 | C = pl.contour(X, Y, fun([X, Y]), 8, colors='black', linewidth=.5) 79 | 80 | for i in range(len(points)): 81 | xs.append(points[i][0]) 82 | ys.append(points[i][1]) 83 | 84 | pl.plot(xs, ys, marker='o', linestyle='--', color='r', label='Square') 85 | 86 | 87 | def rozenbrock(x0, lmb, epsilon1, epsilon2, alpha, beta): 88 | """ 89 | x0 - start point, define as [x, y] 90 | lmb - delta x, define as [0.1, 0.1] 91 | epsilon1, epsilon2 - estimates errors 92 | alpha, beta - coefficients for increasing\decreasing step 93 | success = array for saving YES\NO answers 94 | """ 95 | 96 | start = x0 97 | s1 = [1, 0] # s1 vector 98 | s2 = [0, 1] #s2 vector 99 | N = 0 # iterations count 100 | success1 = [] 101 | success2 = [] 102 | points = [] 103 | points.append(x0) 104 | iterations = 0 105 | 106 | temp = [] 107 | 108 | while True: 109 | temp = x0 110 | x1 = [x0[0] + lmb[0]*s1[0], x0[1] + lmb[0]*s1[1]] 111 | 112 | if fun(x1) < fun(x0): 113 | success1.append("Y") 114 | lmb[0] *= alpha 115 | x0 = x1[:] 116 | 117 | elif fun(x1) > fun(x0): 118 | success1.append("N") 119 | lmb[0] *= beta 120 | 121 | x2 = [x0[0] + lmb[1]*s2[0], x0[1] + lmb[1]*s2[1]] 122 | 123 | 124 | if fun(x2) < fun(x0): 125 | success2.append("Y") 126 | lmb[1] *= alpha 127 | x0 = x2[:] 128 | elif fun(x2) > fun(x0): 129 | success2.append("N") 130 | lmb[1] *= beta 131 | 132 | if constraintFunCircle(x0) >= 0: 133 | lmb[0] /= 2 134 | lmb[1] /= 2 135 | print "lmb reduced cause of constraint" 136 | print lmb 137 | x0 = temp[:] 138 | continue 139 | 140 | if success1[len(success1)-1] == "N" and success2[len(success2)-1] == "N" and "Y" in success1 and "Y" in success2: 141 | print "Our stop point: [%8.5f, %8.5f]" % (x0[0], x0[1]) 142 | s1 = [x0[0] - start[0], x0[1] - start[1]] 143 | print "S1" + str(s1) 144 | norma = norm(s1) 145 | s1[0] /= norma 146 | s1[1] /= norma 147 | s2 = perpendicular(s1) 148 | print "F(X1) = %8.5f" % fun(x0) 149 | 150 | error_point = norm([x0[0] - start[0], x0[1] - start[1]])/norm(x0) 151 | error_func = abs(fun(x0) - fun(start))/abs(fun(start)) 152 | 153 | print "||x^2 - x^1|| / ||x^2|| = %8.5f" % error_point 154 | print "|f(x^2) - f(x^1)| / |x^2| = %8.5f" % error_func 155 | 156 | if error_point < epsilon1 or error_func < epsilon2 and N>1: 157 | print "DONE" 158 | print "N = %3d" % N 159 | break 160 | 161 | points.append(x0) 162 | start = x0[:] 163 | success1 = [] 164 | success2 = [] 165 | N+=1 166 | 167 | print "___________________________________________________________________________" 168 | iterations+=1 169 | 170 | 171 | #xOne = dsc_pauell(x0, 0.1, epsilon2) 172 | #print xOne 173 | plot(points) 174 | print "FUNCTIONS COUNT = " + str(count) 175 | print "RESULT " + str(x0) 176 | 177 | 178 | def main(): 179 | x0 = [4, 4] 180 | rozenbrock(x0, [0.3, 0.1], 0.001, 0.001, 2.9, -0.5) 181 | 182 | 183 | if __name__ == '__main__': 184 | main() 185 | 186 | -------------------------------------------------------------------------------- /UsingDerivativesMethodsFirstAndSecondOrder.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sun Apr 06 14:01:22 2014 4 | 5 | @author: Alex 6 | """ 7 | import math 8 | import numpy as np 9 | import matplotlib.pylab as pl 10 | 11 | count = 0 12 | 13 | def incCount(): 14 | global count 15 | count = count+1 16 | 17 | def zeroCount(): 18 | global count 19 | count = 0 20 | 21 | 22 | def calcX(x0, grad, lmb): 23 | ''' 24 | x0 - gradient*lambda 25 | ''' 26 | return sub(x0, mults(grad, lmb)) 27 | 28 | def svenn(x0, grad, lmb, delta): 29 | """ 30 | One-dimensional Svenn search 31 | """ 32 | #print "Svenn stage..." 33 | f0 = fun(calcX(x0, grad, lmb)) 34 | if f0 < fun(calcX(x0, grad, lmb+delta)): 35 | delta = -delta 36 | x1 = lmb + delta 37 | f1 = fun(calcX(x0, grad, x1)) 38 | while f1 < f0: 39 | delta *= 2 40 | lmb = x1 41 | x1 = lmb + delta 42 | f0 = f1 43 | f1 = fun(calcX(x0, grad, x1)) 44 | a = lmb + delta/2 45 | b = lmb - delta/2 46 | if f0 < f1: 47 | if a < b: 48 | return [a, b] 49 | else: 50 | return [b, a] 51 | elif f1 < f0: 52 | if lmb < x1: 53 | return [lmb, x1] 54 | else: 55 | return [x1, lmb] 56 | else: 57 | if lmb < b: 58 | return [lmb, b] 59 | else: 60 | return [b, lmb] 61 | 62 | 63 | def dsc(x0, grad, lmb, delta): 64 | svenn_res = svenn(x0, grad, lmb, delta) 65 | x1 = svenn_res[0] 66 | x3 = svenn_res[1] 67 | x2 = (x1 + x3)/2 68 | f1 = fun(calcX(x0, grad, x1)) 69 | f2 = fun(calcX(x0, grad, x2)) 70 | f3 = fun(calcX(x0, grad, x3)) 71 | xApprox = x2 + ((x3 - x2) * (f1 - f3)) / (2 * (f1 - 2 * f2 + f3)) 72 | return [x1, x2, x3, xApprox] 73 | 74 | 75 | def dscPowell(x0, grad, eps, lmb, delta): 76 | dsc_res = dsc(x0, grad, lmb, delta) 77 | a = dsc_res[0] 78 | xmin = dsc_res[1] 79 | b = dsc_res[2] 80 | xApprox = dsc_res[3] 81 | 82 | while abs(xmin-xApprox) >= eps or abs(fun(calcX(x0, grad, xmin)) - fun(calcX(x0, grad, xApprox))) >= eps: 83 | if xApprox < xmin: 84 | b = xmin 85 | else: 86 | a = xmin 87 | xmin = xApprox 88 | funcRes = [fun(calcX(x0, grad, a)), fun(calcX(x0, grad, xmin)), fun(calcX(x0, grad, b))] 89 | a1 = (funcRes[1] - funcRes[0]) / (xmin - a) 90 | a2 = ((funcRes[2] - funcRes[0]) / (b - a) - a1) / (b - xmin) 91 | xApprox = (a + xmin) / 2 - a1 / (2 * a2) 92 | return xmin 93 | 94 | def gold(a, b, eps, x0, grad): 95 | """ 96 | One-dimensional gold search 97 | """ 98 | l = b - a 99 | x1 = a + 0.382*l 100 | x2 = a + 0.618*l 101 | while l > eps: 102 | if fun(calcX(x0, grad, x1)) < fun(calcX(x0, grad, x2)): 103 | b = x2 104 | x2 = x1 105 | l = b - a 106 | x1 = a + 0.382*l 107 | else: 108 | a = x1 109 | x1 = x2 110 | l = b - a 111 | x2 = a + 0.618*l 112 | print "gold a: " + str(a) 113 | print "gold b: " + str(b) 114 | return [a, b] 115 | 116 | 117 | def calcLambda(x0, grad, eps, lmb): 118 | line = svenn(x0, grad, lmb, 0.1) 119 | line = gold(line[0], line[1], eps, x0, grad) 120 | lmb = (line[0] + line[1])/2 121 | return lmb 122 | 123 | def plot3D(points, col): 124 | n = 256 125 | x = np.linspace(-100, 100, n) 126 | y = np.linspace(-100, 100, n) 127 | z = np.linspace(-100, 100, n) 128 | X, Y, Z = np.meshgrid(x, y, z) 129 | 130 | xs = [] 131 | ys = [] 132 | zs = [] 133 | 134 | #pl.contourf(X, Y, Z, fun([X, Y, Z]), 8, alpha=.75, cmap='jet') 135 | #C = pl.contour(X, Y, Z, fun([X, Y, Z]), 8, colors='black', linewidth=.5) 136 | 137 | for i in range(len(points)): 138 | xs.append(points[i][0]) 139 | ys.append(points[i][1]) 140 | zs.append(points[i][2]) 141 | 142 | pl.plot(xs, ys, marker='o', linestyle='--', color=str(col), label='Square') 143 | 144 | def plot(points, col): 145 | n = 256 146 | x = np.linspace(-5, 5, n) 147 | y = np.linspace(-5, 5, n) 148 | X, Y = np.meshgrid(x, y) 149 | 150 | xs = [] 151 | ys = [] 152 | 153 | pl.contourf(X, Y, fun([X, Y]), 8, alpha=.75, cmap='jet') 154 | C = pl.contour(X, Y, fun([X, Y]), 8, colors='black', linewidth=.5) 155 | 156 | for i in range(len(points)): 157 | xs.append(points[i][0]) 158 | ys.append(points[i][1]) 159 | 160 | pl.plot(xs, ys, marker='o', linestyle='--', color=str(col), label='Square') 161 | 162 | def add(x, y): 163 | res = [] 164 | for i in xrange(len(x)): 165 | res.append(x[i] + y[i]) 166 | return res 167 | 168 | def sub(x, y): 169 | res = [] 170 | for i in xrange(len(x)): 171 | res.append(x[i] - y[i]) 172 | return res 173 | 174 | def mults(x, n): 175 | res = [] 176 | for i in xrange(len(x)): 177 | res.append(x[i]*n) 178 | return res 179 | 180 | def derivative(x, n): 181 | h = [] 182 | for i in xrange(len(x)): 183 | if i == n: 184 | h.append(0.000000000001) 185 | else: 186 | h.append(0) 187 | return (fun([x[0] + h[0], x[1] + h[1]]) - fun([x[0] - h[0], x[1] - h[1]]))/(2*h[n]) 188 | 189 | def derivative2(x, a, b): 190 | ai = [] 191 | aj = [] 192 | for i in xrange(len(x)): 193 | if i == a: 194 | ai.append(0.001) 195 | else: 196 | ai.append(0) 197 | for j in xrange(len(x)): 198 | if j == b: 199 | aj.append(0.001) 200 | else: 201 | aj.append(0) 202 | return (fun(add(x, add(ai, aj))) - fun(add(x, ai)) - fun(add(x, aj)) + fun(x))/ (ai[a]**2) 203 | 204 | def gradient(x): 205 | grad = [] 206 | for i in xrange(len(x)): 207 | grad.append(derivative(x, i)) 208 | return grad 209 | 210 | def norm(s1): 211 | normas = 0 212 | for i in xrange(len(s1)): 213 | normas += s1[i]**2 214 | return math.sqrt(normas) 215 | 216 | def hesse(x): 217 | h = [] 218 | for i in xrange(len(x)): 219 | for j in xrange(len(x)): 220 | h.append(derivative2(x, i, j)) 221 | return h 222 | 223 | def fun(x): 224 | incCount() 225 | 226 | #return 4*(x[0]-5)**2 + (x[1] - 6)**2 227 | #return (1-x[0])**2 + 100*(x[1] - x[0]**2)**2 228 | #return (10*(x[0] - x[1])**2 + (x[0] - 1)**2)**(1/4) 229 | #return 4*(x[0]-4)**2 + x[0]*x[1] + 3*x[1]**2 230 | #return 2*x[0]**2 + x[0]*x[1] + 3*x[1]**2 231 | return x[0]**3 + x[1]**3 - 3*x[0]*x[1] 232 | 233 | 234 | def fixedGradient(x0, eps1, eps2): 235 | zeroCount() 236 | """ 237 | Just gradient descent with fixed lamba 238 | Lambda will be reduced, if next iteration gives worse (bigger) value 239 | """ 240 | print "fixed gradient goes!" 241 | xs = [] 242 | xs.append(x0) 243 | x0 = mults(x0, -1) 244 | x1 = [-x0[0], -x0[1]] 245 | lmb = 1.0 246 | while norm(sub(x1, x0))/norm(x0) >= eps2: 247 | x0 = x1 248 | grad = gradient(x0) 249 | if norm(grad) < eps1: 250 | break; 251 | x1 = sub(x0, mults(mults(grad, (1 / norm(grad))), lmb)) 252 | if fun(x1) > fun(x0): 253 | lmb /= 2.0 254 | print "lmb reduced = " + str(lmb) 255 | print x1 256 | xs.append(x1) 257 | plot(xs, 'r') 258 | return x1 259 | 260 | 261 | def fastestDescent(x0, eps1, eps2): 262 | zeroCount() 263 | """ 264 | Gradient descent with lambda. calculated with one-dimensional search 265 | """ 266 | print "fastest descent goes!" 267 | xs = [] 268 | xs.append(x0) 269 | x0 = mults(x0, -1) 270 | x1 = [-x0[0], -x0[1]] 271 | lmb = 0.1 272 | while norm(sub(x1, x0))/norm(x0) >= eps1: 273 | x0 = x1 274 | grad = gradient(x0) 275 | if norm(grad) < eps1: 276 | break 277 | lmb = calcLambda(x0, grad, eps2, lmb) 278 | x1 = calcX(x0, grad, lmb) 279 | print x1 280 | xs.append(x1) 281 | print "end fastest descent" 282 | plot(xs, 'g') 283 | return x1 284 | 285 | 286 | def partan(x0, eps1, eps2): 287 | zeroCount() 288 | """ 289 | Parallel tangents method (advanced) 290 | """ 291 | print "SPartan goes!" 292 | grad = gradient(x0) 293 | lmb = 0.1 294 | xs = [] 295 | xs.append(x0) 296 | while (norm(grad) >= eps1): 297 | print "x0 = " + str(x0) 298 | lmb = calcLambda(x0, grad, eps2, lmb) 299 | x1 = calcX(x0, grad, lmb) 300 | print "x1 = " + str(x1) 301 | grad = gradient(x1) 302 | lmb = calcLambda(x1, grad, eps2, lmb) 303 | x2 = calcX(x1, grad, lmb) 304 | print "x2 = " + str(x2) 305 | grad = sub(x2, x0) 306 | lmb = calcLambda(x2, grad, eps2, lmb) 307 | x2 = calcX(x2, grad, lmb) 308 | print "x3 = " + str(x2) 309 | grad = gradient(x2) 310 | lmb = calcLambda(x2, grad, eps2, lmb) 311 | x0 = calcX(x2, grad, lmb) 312 | print "x4 = " + str(x0) 313 | grad = sub(x0, x1) 314 | lmb = calcLambda(x0, grad, eps2, lmb) 315 | x0 = calcX(x0, grad, lmb) 316 | print "x5 = " + str(x0) 317 | grad = gradient(x0) 318 | print "-------------------------" 319 | xs.append(x0) 320 | plot(xs, 'orange') 321 | return x0 322 | 323 | def newton(x0, eps): 324 | zeroCount() 325 | """ 326 | Newton second-order optimization method 327 | """ 328 | xs = [] 329 | xs.append(x0) 330 | lmb = 0.1 331 | while norm(gradient(x0)) > eps: 332 | hesseMatrix = hesse(x0) 333 | hesseMatrix = np.reshape(hesseMatrix, (-1, len(x0))) 334 | #print "hesse: " 335 | #print hesseMatrix 336 | grad = gradient(x0) 337 | #print "grad" + str(grad) 338 | invHesse = np.linalg.pinv(np.array(hesseMatrix)) 339 | #print "hesse inv " 340 | #print invHesse 341 | transGrad = np.transpose(np.array([grad])) 342 | #print "transpose grad" 343 | #print transGrad 344 | res = np.dot(invHesse, transGrad) 345 | #print "res " 346 | #print res 347 | normNow = norm(np.transpose(res)[0]) 348 | #print "norm" 349 | #print normNow 350 | fin = mults(np.transpose(res)[0], 1/normNow) 351 | #print "fin " 352 | #print fin 353 | #lmb = calcLambda(x0, fin, eps, lmb) 354 | lmb = dscPowell(x0, fin, eps, 0, 0.1) 355 | #print "lambda " + str(lmb) 356 | x0 = calcX(x0, fin, lmb) 357 | print x0 358 | xs.append(x0) 359 | plot(xs, 'green') 360 | print "calculations: " + str(count) 361 | return x0 362 | 363 | def fletcherReeves(x0, eps): 364 | zeroCount() 365 | s = gradient(x0) 366 | i = 0 367 | xs = [] 368 | xs.append(x0) 369 | while norm(s) > eps: 370 | i+=1 371 | lmb = dscPowell(x0, s, 0.00000001, 0, 0.1 * norm(x0) / norm(s)) 372 | print "LMB" 373 | print lmb 374 | print "S" 375 | print s 376 | x0 = calcX(x0, s, lmb) 377 | print "X" 378 | print x0 379 | xs.append(x0) 380 | gradK = gradient(x0) 381 | s = sub(mults(s, -(norm(gradK)**2)/(norm(s)**2)), gradK) 382 | if norm(s) < eps: 383 | s = gradK 384 | print "iterations: " + str(i) 385 | print "calculations: " + str(count) 386 | plot(xs, 'yellow') 387 | return x0 388 | 389 | def conjugatedDirectionPowell(x0, eps, eps2): 390 | zeroCount() 391 | iteration = 0 392 | si = np.eye(len(x0)) 393 | xn = mults(x0, -1) 394 | xs = [] 395 | xs.append(x0) 396 | while norm(sub(xn, x0)) > 0.1 * eps: 397 | iteration+=1 398 | x0 = xn 399 | xi = x0 400 | fi = [] 401 | 402 | for i in xrange(len(x0)): 403 | lmb = dscPowell(xi, si[i], eps2, 0, 0.1) 404 | xn = calcX(xi, si[i], lmb) 405 | fi.append(fun(xi) - fun(xn)) 406 | xi = xn 407 | for i in xrange(len(si)-1): 408 | si[i] = si[i+1] 409 | 410 | si[len(si)-1] = sub(xn, x0) 411 | lmb = dscPowell(xn, si[len(si) - 1], eps2, 0, 0.1) 412 | xn = calcX(xn, si[len(si) - 1], lmb) 413 | print xn 414 | xs.append(xn) 415 | plot(xs, 'red') 416 | print "calculations: " + str(count) 417 | return xn 418 | 419 | 420 | def conjugatedDirection(x0, eps): 421 | xs = [] 422 | s = np.array(gradient(x0))[np.newaxis].T 423 | while norm(gradient(x0)) > eps: 424 | print "START S" 425 | print s 426 | hesseM = ([[hesse(x0)[0], hesse(x0)[1]], [hesse(x0)[2], hesse(x0)[3]]]) 427 | grad = np.array(gradient(x0))[np.newaxis].T 428 | 429 | up = np.dot(grad.T, s) 430 | down = np.dot(s.T, hesseM) 431 | down = np.dot(down, s) 432 | lmb = up/down 433 | 434 | xn = calcX(x0, s, lmb) 435 | 436 | if math.isnan(xn[0]) or math.isnan(xn[1]): 437 | print "NAN" 438 | return x0 439 | 440 | calc = (hesseM[0][0]*s[0])/(hesseM[1][1]*s[1]) 441 | 442 | s[0] = math.sqrt(1/(1+calc)) 443 | s[1] = s[0]*calc[0] 444 | 445 | s = np.array(s)[np.newaxis].T 446 | 447 | print "END S" 448 | print s 449 | 450 | news = [s[0][0][0] , s[0][1][0]] 451 | newxn = [xn[0][0][0], xn[1][0][0]] 452 | 453 | s = np.array(news)[np.newaxis].T 454 | xn = newxn 455 | x0 = xn 456 | print xn 457 | xs.append(xn) 458 | plot(xs, "red") 459 | print count 460 | 461 | 462 | 463 | 464 | def main(): 465 | point = [3,5] 466 | #fixedGradient(point, 0.01, 0.01) 467 | #fastestDescent(point, 0.01, 0.01) 468 | #partan(point, 0.01, 0.01) 469 | #newton(point, 0.01) 470 | #fletcherReeves(point, 0.001) 471 | #conjugatedDirectionPowell(point, 0.01, 0.001) 472 | 473 | #conjugatedDirection(point, 0.00001) 474 | 475 | if __name__ == '__main__': 476 | main() 477 | -------------------------------------------------------------------------------- /hook-jeeves.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Mar 14 21:12:42 2014 4 | 5 | @author: Alex (Python code), original implementation idea: @FlyingPirate 6 | Hook-Jeevse optimization method 7 | 8 | """ 9 | 10 | import matplotlib.pylab as pl 11 | import numpy as np 12 | import math 13 | 14 | 15 | def norm(s1): 16 | return math.sqrt(s1[0]**2 + s1[1]**2) 17 | 18 | 19 | def fun(dot): 20 | return (dot[0] - 2)**2 + (dot[1]-2)**2 21 | 22 | def pointError(x, y): 23 | return norm([x[0] - y[0], x[1] - y[1]])/norm(y) 24 | 25 | 26 | def funcError(x, y): 27 | return abs(fun(x) - fun(y))/abs(fun(y)) 28 | 29 | def h(x0, deltax, epsilon1, epsilon2): 30 | print "Starting Hook-Jeeves..." 31 | xTB = [0,0] 32 | points = [] 33 | 34 | n = 1 35 | 36 | while round(pointError(xTB, x0),3) > epsilon1: #or round(funcError(xTB, x0),3) > epsilon2: 37 | xTB = search(x0, deltax) 38 | 39 | while fun(xTB) < fun(x0): 40 | print "x%d: fun[%8.5f, %8.5f] = %8.5f" %(n, xTB[0], xTB[1], fun(xTB)) 41 | points.append(xTB) 42 | 43 | if round(pointError(x0, xTB),3) < epsilon1: #and round(funcError(x0, xTB),3) < epsilon2: 44 | print pointError(x0, xTB) 45 | print funcError(x0, xTB) 46 | break 47 | 48 | xR = [2*xTB[0] - x0[0], 2*xTB[1] - x0[1]] 49 | print "xR" + str(xR) + "fun(xR)" + str(fun(xR)) 50 | 51 | if fun(xTB) < fun(x0): 52 | x0 = xTB[:] 53 | xTB = search(xR, deltax) 54 | 55 | n+=1 56 | 57 | deltax[0] /= 2 58 | deltax[1] /= 2 59 | 60 | print "END" 61 | plot(points) 62 | print points 63 | 64 | 65 | def search(x0, deltax): 66 | x1 = [x0[0] + deltax[0], x0[1]] 67 | if fun(x0) > fun(x1): 68 | x0 = x1[:] 69 | x2 = [x0[0] - deltax[0], x0[1]] 70 | if fun(x0) > fun(x2): 71 | x0 = x2[:] 72 | x3 = [x0[0], x0[1] + deltax[1]] 73 | if fun(x0) > fun(x3): 74 | x0 = x3[:] 75 | x4 = [x0[0], x0[1] - deltax[1]] 76 | if fun(x0) > fun(x4): 77 | x0 = x4[:] 78 | return x0 79 | 80 | 81 | def plot(points): 82 | ''' 83 | Plotting 2D function and way search 84 | ''' 85 | n = 256 86 | x = np.linspace(-12, 12, n) 87 | y = np.linspace(-12, 12, n) 88 | X, Y = np.meshgrid(x, y) 89 | 90 | xs = [] 91 | ys = [] 92 | 93 | pl.contourf(X, Y, fun([X, Y]), 8, alpha=.75, cmap='jet') 94 | C = pl.contour(X, Y, fun([X, Y]), 8, colors='black', linewidth=.5) 95 | 96 | for i in range(len(points)): 97 | xs.append(points[i][0]) 98 | ys.append(points[i][1]) 99 | 100 | pl.plot(xs, ys, marker='o', linestyle='--', color='r', label='Square') 101 | 102 | def main(): 103 | h([-2, 2], [0.5, 0.5], 0.01, 0.01) 104 | #plot([[1,1]], 'yellow') 105 | 106 | 107 | 108 | if __name__ == '__main__': 109 | main() 110 | -------------------------------------------------------------------------------- /rosenbrock.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Mar 13 19:25:20 2014 4 | 5 | Rosenbrock's optimization algorithm with visualization 6 | Some helpful outputs for better intuition are commented ('#') 7 | 8 | @author: Alex 9 | """ 10 | 11 | import numpy as np 12 | import math 13 | import matplotlib.pylab as pl 14 | 15 | count = 0 16 | 17 | def incCount(): 18 | global count 19 | count = count+1 20 | 21 | def perpendicular(a): 22 | b = np.empty_like(a) 23 | b[0] = -a[1] 24 | b[1] = a[0] 25 | return b 26 | 27 | 28 | def fun(x): 29 | incCount() 30 | #return 4*(x[0]-5)**2 + (x[1]-6)**2 31 | #return (x[0] - variant)**2 - x[0]*x[1] + 3*x[1]**2 32 | #return (x[0] - 3)**2 + x[0] * x[1] + 3 *x[1]**2 33 | return (1-x[0])**2 + 100*(x[1] - x[0]**2)**2 34 | #return (10*(x[0] - x[1])**2 + (x[0] - 1)**2)**4 35 | 36 | def constraintFunCircle(x): 37 | ''' 38 | Все ограничения подаем в виде >=0 и меняем знаки 39 | ''' 40 | return -(x[0]+1.2)**2 - (x[1])**2 + 1 41 | 42 | 43 | def constraintFunLine(x): 44 | """ 45 | Не меняем знак??? 46 | """ 47 | return 1 - x[0] - x[1] 48 | 49 | def plot(points): 50 | ''' 51 | Plotting 2D function and way search 52 | ''' 53 | n = 256 54 | x = np.linspace(-12, 12, n) 55 | y = np.linspace(-12, 12, n) 56 | X, Y = np.meshgrid(x, y) 57 | 58 | xs = [] 59 | ys = [] 60 | 61 | pl.contourf(X, Y, fun([X, Y]), 8, alpha=.75, cmap='jet') 62 | C = pl.contour(X, Y, fun([X, Y]), 8, colors='black', linewidth=.5) 63 | 64 | for i in range(len(points)): 65 | xs.append(points[i][0]) 66 | ys.append(points[i][1]) 67 | 68 | pl.plot(xs, ys, marker='o', linestyle='--', color='r', label='Square') 69 | 70 | 71 | 72 | def norm(s1): 73 | return math.sqrt(s1[0]**2 + s1[1]**2) 74 | 75 | 76 | def rozenbrock(x0, lmb, epsilon1, epsilon2, alpha, beta): 77 | """ 78 | x0 - start point, define as [x, y] 79 | lmb - delta x, define as [0.1, 0.1] 80 | epsilon1, epsilon2 - estimates errors 81 | alpha, beta - coefficients for increasing\decreasing step 82 | success = array for saving YES\NO answers 83 | """ 84 | 85 | start = x0 86 | s1 = [1, 0] # s1 vector 87 | s2 = [0, 1] #s2 vector 88 | N = 0 # iterations count 89 | success1 = [] 90 | success2 = [] 91 | points = [] 92 | points.append(x0) 93 | iterations = 0 94 | 95 | temp = [] 96 | 97 | while True: 98 | temp = x0 99 | x1 = [x0[0] + lmb[0]*s1[0], x0[1] + lmb[0]*s1[1]] 100 | 101 | if fun(x1) < fun(x0): 102 | success1.append("Y") 103 | lmb[0] *= alpha 104 | x0 = x1[:] 105 | 106 | elif fun(x1) > fun(x0): 107 | success1.append("N") 108 | lmb[0] *= beta 109 | 110 | x2 = [x0[0] + lmb[1]*s2[0], x0[1] + lmb[1]*s2[1]] 111 | 112 | 113 | if fun(x2) < fun(x0): 114 | success2.append("Y") 115 | lmb[1] *= alpha 116 | x0 = x2[:] 117 | elif fun(x2) > fun(x0): 118 | success2.append("N") 119 | lmb[1] *= beta 120 | 121 | if success1[len(success1)-1] == "N" and success2[len(success2)-1] == "N" and "Y" in success1 and "Y" in success2: 122 | print "Our stop point: [%8.5f, %8.5f]" % (x0[0], x0[1]) 123 | s1 = [x0[0] - start[0], x0[1] - start[1]] 124 | norma = norm(s1) 125 | s1[0] /= norma 126 | s1[1] /= norma 127 | s2 = perpendicular(s1) 128 | print "F(X1) = %8.5f" % fun(x0) 129 | 130 | error_point = norm([x0[0] - start[0], x0[1] - start[1]])/norm(x0) 131 | error_func = abs(fun(x0) - fun(start))/abs(fun(start)) 132 | 133 | print "||x^2 - x^1|| / ||x^2|| = %8.5f" % error_point 134 | print "|f(x^2) - f(x^1)| / |x^2| = %8.5f" % error_func 135 | 136 | if error_point < epsilon1 or error_func < epsilon2 and N>1: 137 | print "DONE" 138 | print "N = %3d" % N 139 | break 140 | 141 | points.append(x0) 142 | start = x0[:] 143 | success1 = [] 144 | success2 = [] 145 | N+=1 146 | 147 | print "___________________________________________________________________________" 148 | iterations+=1 149 | 150 | 151 | plot(points) 152 | print "FUNCTIONS COUNT = " + str(count) 153 | 154 | 155 | def main(): 156 | x0 = [-1.2, 0] 157 | rozenbrock(x0, [0.3, 0.1], 0.01, 0.01, 3, -0.5) 158 | 159 | 160 | if __name__ == '__main__': 161 | main() 162 | 163 | --------------------------------------------------------------------------------