├── 4 ├── hw41.py ├── hw41test.py ├── hw41test2.py ├── hw42.in └── hw42_autodiff.py ├── 11 └── 11.py ├── 12 └── 12.ipynb ├── 13 ├── 131.ipynb ├── 133.ipynb ├── 134.ipynb ├── 135.ipynb ├── bfgs.py ├── lbfgs.py └── rs1.ipynb ├── 17 └── 17.ipynb ├── 18 ├── 1.ipynb └── 2.ipynb ├── 19 ├── .vscode │ └── settings.json ├── 1.ipynb ├── 2.ipynb └── 3.ipynb ├── 20 ├── 1.py └── 2.py ├── .gitattributes ├── .vscode └── settings.json ├── LICENSE └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | *.ipynb linguist-documentation 4 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.suggest.snippetsPreventQuickSuggestions": false 3 | } -------------------------------------------------------------------------------- /11/11.py: -------------------------------------------------------------------------------- 1 | # HW11 给定函数的梯度下降优化 2 | # By ZincCat 3 | import numpy as np 4 | from matplotlib import pyplot as plt 5 | 6 | # 设置随机种子 7 | np.random.seed(19890817) 8 | 9 | # 初始化问题 10 | m = 10 11 | n = 15 12 | 13 | a = np.random.normal(5, 5, [n, m]) 14 | x = np.ones(n) 15 | 16 | 17 | def f(x): 18 | # 计算函数值 19 | return np.sum(np.exp(a.T@x)) + np.sum(np.exp(-a.T@x)) 20 | 21 | 22 | def gradient_f(x): 23 | # 计算函数梯度 24 | return a@(np.exp(a.T@x) - np.exp(-a.T@x)) 25 | 26 | 27 | def descent(x, grad, grad_norm, mode='2'): 28 | # 梯度下降函数 29 | # 输入目前x取值, 梯度, 梯度的范数, 下降模式 30 | # 输出下降后x取值, 步长t 31 | # 下降模式为'2'时采用2范数, 为'inf'时采用无穷范数 32 | normalized_grad = grad/grad_norm 33 | t = 1.0 34 | if mode == '2': 35 | # l_2 norm 36 | while f(x - t*normalized_grad) > value - alpha*t*np.dot(grad, normalized_grad): 37 | t *= beta 38 | x -= t*normalized_grad 39 | elif mode == 'inf': 40 | # l_infty norm 41 | while f(x - t*np.sign(normalized_grad)) > value - alpha*t*np.dot(grad, np.sign(normalized_grad)): 42 | t *= beta 43 | x -= t*np.sign(normalized_grad) 44 | return x, t 45 | 46 | 47 | minValue = f(np.zeros(n)) # 函数最小值 48 | 49 | alpha_list = [0.22] 50 | beta_list = [0.62] 51 | maxIter = 1000 # 最大迭代次数 52 | eta = 0.01 # 停止条件 53 | 54 | result = [] # 记录 参数-结果 对 55 | time = [] # 记录时间步, 用于绘图 56 | values = [] # 记录某一时间步下函数值, 用于绘图 57 | stepsize = [] # 记录某一时间步下步长, 用于绘图 58 | Plot = True # 是否绘图, 请保证此时alpha, beta均为单一取值 59 | 60 | t = 0 # 用于绘图 61 | # 实验 62 | for alpha in alpha_list: 63 | for beta in beta_list: 64 | timestep = 0 65 | x = np.ones(n) 66 | while True: 67 | value = f(x) 68 | # print("Iteration:", timestep, "Error", value - minValue) 69 | if Plot: 70 | time.append(timestep) 71 | stepsize.append(t) 72 | values.append(value) 73 | grad = gradient_f(x) 74 | grad_norm = np.linalg.norm(grad) 75 | if grad_norm <= eta or timestep > maxIter: 76 | break 77 | x, t = descent(x, grad, grad_norm, mode='inf') # 此时使用无穷范数 78 | timestep += 1 79 | result.append((alpha, beta, f(x)-minValue, timestep)) 80 | for i in result: 81 | print(i) 82 | 83 | # 绘图 84 | if Plot: 85 | # f − p^* versus iteration 86 | plt.plot(time, values) 87 | plt.xlabel('Iterations', fontsize=14) 88 | plt.ylabel('Value', fontsize=14) 89 | plt.savefig('alpha'+str(alpha)+'beta'+str(beta)+'value.pdf') 90 | plt.show() 91 | 92 | # step length versus iteration number 93 | del(time[0]) 94 | del(stepsize[0]) 95 | plt.plot(time, stepsize) 96 | plt.xlabel('Iterations', fontsize=14) 97 | plt.ylabel('Step length', fontsize=14) 98 | plt.savefig('alpha'+str(alpha)+'beta'+str(beta)+'step.pdf') 99 | plt.show() 100 | -------------------------------------------------------------------------------- /13/131.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# HW13.1 DFP\n", 8 | "\n", 9 | "By ZincCat\n", 10 | "\n", 11 | "用DFP优化\n", 12 | "$f(\\mathbf{x})=\\frac{x_{1}^{4}}{4}+\\frac{x_{2}^{2}}{2}-x_{1} x_{2}+x_{1}-x_{2}$" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "%matplotlib inline\n", 22 | "import numpy as np\n", 23 | "from matplotlib import pyplot as plt\n", 24 | "from matplotlib.colors import LogNorm\n", 25 | "\n", 26 | "# 设置随机种子\n", 27 | "np.random.seed(19890817)" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "DFP" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": { 41 | "tags": [] 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "# 定义一些辅助函数\n", 46 | "def f(x):\n", 47 | " # 计算函数值\n", 48 | " return x[0]**4/4 + x[1]*x[1]/2 - x[0]*x[1] + x[0] - x[1]\n", 49 | "\n", 50 | "def gradient_f(x):\n", 51 | " # 计算函数梯度\n", 52 | " return np.array([x[0]*x[0]*x[0] - x[1] + 1, x[1] - x[0] - 1])\n", 53 | "\n", 54 | "def linesearch(f, x, g, d, alpha=0.4, beta=0.8):\n", 55 | " # backtrack linesearch\n", 56 | " t = 1.0\n", 57 | " value = f(x)\n", 58 | " while f(x + t*d) > value + alpha*t*np.dot(g, d):\n", 59 | " t *= beta\n", 60 | " return t" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "alpha = 0.4\n", 70 | "beta = 0.8\n", 71 | "maxIter = 100\n", 72 | "\n", 73 | "# 设置变量初值\n", 74 | "# x = np.array([0, 0], dtype=np.float64) #x0\n", 75 | "x = np.array([1.5, 1], dtype=np.float64) #x0\n", 76 | "H = np.eye(2) #H_0\n", 77 | "g = gradient_f(x)\n", 78 | "d = -g\n", 79 | "eta = 1e-5\n", 80 | "eps = 1e-18\n", 81 | "timestep = 0\n", 82 | "\n", 83 | "Plot = True\n", 84 | "xlog1 = []\n", 85 | "xlog2 = []\n", 86 | "\n", 87 | "while True:\n", 88 | " print(timestep, \"th iteration, x=\", x, \", f(x)=\", f(x))\n", 89 | " print(\"H=\", H)\n", 90 | " print(\"g=\", g)\n", 91 | " if Plot:\n", 92 | " xlog1.append(x[0])\n", 93 | " xlog2.append(x[1])\n", 94 | " if np.linalg.norm(g) < eta: # 算法终止条件\n", 95 | " break\n", 96 | " alpha = linesearch(f, x, g, d, alpha, beta)\n", 97 | " dx = alpha*d\n", 98 | " x += dx\n", 99 | " dg = gradient_f(x) - g\n", 100 | " g += dg\n", 101 | " t = H@dg\n", 102 | " if np.abs(np.dot(dg, t)) < eps: #避免除0\n", 103 | " break\n", 104 | " H = H + np.outer(dx, dx)/np.dot(dx, dg) - np.outer(t, t)/np.dot(dg, t) # DFP\n", 105 | " d = -H@g\n", 106 | " timestep += 1\n", 107 | " if timestep >= maxIter:\n", 108 | " break\n", 109 | "print(\"result: x:\", x, \"\\nf(x):\",f(x))" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "后面的代码仅用于绘图" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "xmin, xmax, xstep = -4.5, 4.5, .2\n", 126 | "ymin, ymax, ystep = -4.5, 4.5, .2\n", 127 | "x = np.meshgrid(np.arange(xmin, xmax + xstep, xstep), np.arange(ymin, ymax + ystep, ystep))\n", 128 | "z = f(x)+1\n", 129 | "zmin, zmax = np.min(z), np.max(z)\n", 130 | "minima1 = np.array([1, 2]).reshape(-1, 1)\n", 131 | "minima2 = np.array([-1, 0]).reshape(-1, 1)" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "fig = plt.figure(figsize=(8, 5))\n", 141 | "ax = plt.axes(projection='3d', elev=50, azim=-50)\n", 142 | "ax.plot_surface(*x, z, edgecolor='none', norm = LogNorm(), alpha=.8, cmap=plt.get_cmap('rainbow'))\n", 143 | "ax.plot(*minima1, f(minima1)+1, 'r*', markersize=10)\n", 144 | "ax.plot(*minima2, f(minima2)+1, 'r*', markersize=10)\n", 145 | "\n", 146 | "ax.set_xlabel('$x_1$')\n", 147 | "ax.set_ylabel('$x_2$')\n", 148 | "ax.set_zlabel('$f$')\n", 149 | "\n", 150 | "ax.set_xlim((xmin, xmax))\n", 151 | "ax.set_ylim((ymin, ymax))\n", 152 | "plt.savefig('131func.pdf')\n", 153 | "plt.show()" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "a = plt.contourf(*x, z, 10, cmap=plt.get_cmap('rainbow'))\n", 163 | "b = plt.contour(*x, z, levels = np.logspace(-2,3,100))\n", 164 | "plt.xlabel('$x_1$'); plt.ylabel(\"$x_2$\")\n", 165 | "plt.title(\"Contour plot of loss function for $x_1 = 1.5, x_2=1$\")\n", 166 | "plt.plot(xlog1, xlog2, 'r--')\n", 167 | "plt.savefig('131cont1.51.pdf')" 168 | ] 169 | } 170 | ], 171 | "metadata": { 172 | "interpreter": { 173 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 174 | }, 175 | "kernelspec": { 176 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 177 | "name": "python3" 178 | }, 179 | "language_info": { 180 | "name": "python", 181 | "version": "" 182 | }, 183 | "orig_nbformat": 2 184 | }, 185 | "nbformat": 4, 186 | "nbformat_minor": 2 187 | } -------------------------------------------------------------------------------- /13/133.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# HW13.3 DFP & BFGS\n", 8 | "By ZincCat\n", 9 | "\n", 10 | "$\\min_{\\mathbf{x}}f(\\mathbf{x})=(3−x_1)^2+7(x_2−x_1^2)^2+ (x_3−x_1-x_2^2)^2$\n" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "%matplotlib inline\n", 20 | "import numpy as np\n", 21 | "from matplotlib import pyplot as plt\n", 22 | "\n", 23 | "# 设置随机种子\n", 24 | "np.random.seed(19890817)" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "# 定义一些辅助函数\n", 34 | "\n", 35 | "def f(x):\n", 36 | " # 计算函数值\n", 37 | " return (3-x[0])**2 + 7*(x[1]-x[0]*x[0])**2 + 9*(x[2]-x[0]-x[1]*x[1])**2\n", 38 | "\n", 39 | "def gradient_f(x):\n", 40 | " # 计算函数梯度\n", 41 | " g = np.zeros(3)\n", 42 | " g[0] = 2*x[0] - 6 - 28*x[0]*(x[1]-x[0]*x[0]) - 18*(x[2]-x[0]-x[1]*x[1])\n", 43 | " g[1] = 14*(x[1]-x[0]*x[0]) - 36*x[1]*(x[2]-x[0]-x[1]*x[1])\n", 44 | " g[2] = 18*(x[2]-x[0]-x[1]*x[1])\n", 45 | " return g\n", 46 | "\n", 47 | "def linesearch_Armijo(f, x, g, d, alpha=0.4, beta=0.8):\n", 48 | " # backtrack linesearch using Armijo rules\n", 49 | " t = 0.5\n", 50 | " value = f(x)\n", 51 | " while f(x + t*d) > value + alpha*t*np.dot(g, d):\n", 52 | " t *= beta\n", 53 | " return t\n", 54 | "\n", 55 | "def linesearch_Wolfe(f, x, g, d, start=0,end=1e10,rho=0.1,sigma=0.8):\n", 56 | " # linesearch using strong Wolfe rules\n", 57 | " value = f(x)\n", 58 | " reg1 = np.dot(g, d)\n", 59 | " reg2 = sigma*g\n", 60 | " t = 0\n", 61 | " while t < 50:\n", 62 | " alpha = (start + end)/2\n", 63 | " x_new = x + alpha*d\n", 64 | " cond1 = (f(x_new) < value + rho*alpha*reg1)\n", 65 | " cond2 = (np.abs(np.dot(gradient_f(x_new), d)) < np.abs(np.dot(reg2, d)))\n", 66 | " if (cond1 and cond2):\n", 67 | " break\n", 68 | " if not cond1:\n", 69 | " end = alpha\n", 70 | " else:\n", 71 | " start = alpha\n", 72 | " t += 1\n", 73 | " return alpha\n", 74 | "\n", 75 | "def DFP(H, dx, dg):\n", 76 | " t = H@dg\n", 77 | " return H + np.outer(dx, dx)/np.dot(dx, dg) - np.outer(t, t)/np.dot(dg, t)\n", 78 | "\n", 79 | "def BFGS(H, dx, dg, eps=1e-30):\n", 80 | " t1 = H@dg\n", 81 | " t2 = np.dot(dg, dx)\n", 82 | " return H + (1+np.dot(dg, t1)/t2)*np.outer(dx, dx)/t2 - (np.outer(t1, dx)+np.outer(dx, t1))/t2" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 3, 88 | "metadata": { 89 | "tags": [] 90 | }, 91 | "outputs": [ 92 | { 93 | "name": "stdout", 94 | "output_type": "stream", 95 | "text": [ 96 | "0 th iteration, x= [0. 0. 0.] , f(x)= 9.0\n", 97 | "1 th iteration, x= [0.46815885 0. 0. ] , f(x)= 8.719030691382894\n", 98 | "2 th iteration, x= [0.95935802 0.24763874 0.68009537] , f(x)= 8.376169675917236\n", 99 | "3 th iteration, x= [0.97807705 0.53665697 0.72202827] , f(x)= 7.986750127517768\n", 100 | "4 th iteration, x= [0.66429007 0.4524167 0.90015991] , f(x)= 5.465163604781935\n", 101 | "5 th iteration, x= [0.97798628 0.66692536 1.27784077] , f(x)= 4.864395396482811\n", 102 | "6 th iteration, x= [0.90311931 0.69087695 1.33533122] , f(x)= 4.524147585688882\n", 103 | "7 th iteration, x= [0.9693146 0.87271233 1.57159107] , f(x)= 4.383506357917406\n", 104 | "8 th iteration, x= [1.30010547 1.3570212 3.04848064] , f(x)= 3.7451055720928808\n", 105 | "9 th iteration, x= [1.17063637 1.2905051 2.98814978] , f(x)= 3.5994789912954928\n", 106 | "10 th iteration, x= [1.20499954 1.36494848 2.963248 ] , f(x)= 3.3740166182024143\n", 107 | "11 th iteration, x= [1.42098792 1.97943184 5.19208444] , f(x)= 2.6989770716336503\n", 108 | "12 th iteration, x= [1.43740359 1.95376093 5.3146299 ] , f(x)= 2.5625418839937195\n", 109 | "13 th iteration, x= [1.43596422 1.98011995 5.30971498] , f(x)= 2.5131168398922537\n", 110 | "14 th iteration, x= [1.61037794 2.4214066 7.28413775] , f(x)= 2.460944230208893\n", 111 | "15 th iteration, x= [1.5040341 2.29372208 6.81912921] , f(x)= 2.2710854488974\n", 112 | "16 th iteration, x= [1.52810689 2.31196248 6.81471446] , f(x)= 2.2010867670879906\n", 113 | "17 th iteration, x= [1.64435852 2.57010807 8.23430305] , f(x)= 1.9652590718526628\n", 114 | "18 th iteration, x= [1.68640396 2.73089315 9.04673602] , f(x)= 1.9004810428012624\n", 115 | "19 th iteration, x= [1.67625017 2.71270974 9.05072419] , f(x)= 1.820531869099901\n", 116 | "20 th iteration, x= [ 1.72399894 2.94832485 10.35586102] , f(x)= 1.6653827159882684\n", 117 | "21 th iteration, x= [ 1.77460219 3.11316071 11.36255438] , f(x)= 1.6077006623245291\n", 118 | "22 th iteration, x= [ 1.77185254 3.08785293 11.36680335] , f(x)= 1.559514675953722\n", 119 | "23 th iteration, x= [ 1.84088723 3.32785777 12.84877132] , f(x)= 1.409700195745528\n", 120 | "24 th iteration, x= [ 1.90546392 3.70085302 15.6106141 ] , f(x)= 1.2330711965781271\n", 121 | "25 th iteration, x= [ 1.93314014 3.70174675 15.60989074] , f(x)= 1.1530724774500491\n", 122 | "26 th iteration, x= [ 1.94553572 3.76135253 16.03851546] , f(x)= 1.1428661274896377\n", 123 | "27 th iteration, x= [ 2.02578374 4.05901996 18.39576563] , f(x)= 1.0636124617987326\n", 124 | "28 th iteration, x= [ 2.02294899 4.04217977 18.39792784] , f(x)= 0.9837388180340672\n", 125 | "29 th iteration, x= [ 2.06168853 4.21181759 19.76706404] , f(x)= 0.9013585224850673\n", 126 | "30 th iteration, x= [ 2.08528809 4.31361851 20.62535273] , f(x)= 0.8858699848799768\n", 127 | "31 th iteration, x= [ 2.0836814 4.30230166 20.6267287 ] , f(x)= 0.860469645964415\n", 128 | "32 th iteration, x= [ 2.12124119 4.47322352 22.0960129 ] , f(x)= 0.7881087186595049\n", 129 | "33 th iteration, x= [ 2.14097827 4.55763705 22.85459317] , f(x)= 0.7734431309736383\n", 130 | "34 th iteration, x= [ 2.13995331 4.54715226 22.85578908] , f(x)= 0.7608192988676439\n", 131 | "35 th iteration, x= [ 2.18352119 4.7396841 24.60445691] , f(x)= 0.6893206874779909\n", 132 | "36 th iteration, x= [ 2.25207451 5.05142596 27.65689259] , f(x)= 0.6753792179096714\n", 133 | "37 th iteration, x= [ 2.25104625 5.04000201 27.65803943] , f(x)= 0.5663731594907137\n", 134 | "38 th iteration, x= [ 2.28611517 5.20707571 29.37279081] , f(x)= 0.5187671075645074\n", 135 | "39 th iteration, x= [ 2.30250218 5.28204651 30.15718471] , f(x)= 0.5076522399439221\n", 136 | "40 th iteration, x= [ 2.30211779 5.27730144 30.15764855] , f(x)= 0.4908502776310041\n", 137 | "41 th iteration, x= [ 2.34072906 5.45945284 32.11216568] , f(x)= 0.4478360110513104\n", 138 | "42 th iteration, x= [ 2.37523631 5.62223707 33.90938853] , f(x)= 0.44415731392549385\n", 139 | "43 th iteration, x= [ 2.37443755 5.61371775 33.91015998] , f(x)= 0.39975472422812974\n", 140 | "44 th iteration, x= [ 2.39986279 5.74395451 35.37271701] , f(x)= 0.36547950582823985\n", 141 | "45 th iteration, x= [ 2.4207975 5.84440765 36.52754649] , f(x)= 0.3600524463453233\n", 142 | "46 th iteration, x= [ 2.42032998 5.83851185 36.52806168] , f(x)= 0.3421012454375568\n", 143 | "47 th iteration, x= [ 2.44452803 5.96181855 37.96931276] , f(x)= 0.3129801561762211\n", 144 | "48 th iteration, x= [ 2.46516616 6.06267906 39.17303168] , f(x)= 0.3084112130037878\n", 145 | "49 th iteration, x= [ 2.46471727 6.056812 39.17352498] , f(x)= 0.29391388203137603\n", 146 | "50 th iteration, x= [ 2.48724504 6.17338008 40.58027184] , f(x)= 0.26688826783290814\n", 147 | "51 th iteration, x= [ 2.50294801 6.2514404 41.54540809] , f(x)= 0.26132861959820786\n", 148 | "52 th iteration, x= [ 2.50259363 6.24667907 41.54579738] , f(x)= 0.2537092910072152\n", 149 | "53 th iteration, x= [ 2.52832153 6.38018194 43.21347997] , f(x)= 0.2277119346135504\n", 150 | "54 th iteration, x= [ 2.55216285 6.5008325 44.760926 ] , f(x)= 0.22607984775062254\n", 151 | "55 th iteration, x= [ 2.55162332 6.49400801 44.76145867] , f(x)= 0.21579955047604363\n", 152 | "56 th iteration, x= [ 2.57191098 6.60529039 46.18335611] , f(x)= 0.18693577821774585\n", 153 | "57 th iteration, x= [ 2.5872011 6.68110654 47.19445121] , f(x)= 0.17956187674768154\n", 154 | "58 th iteration, x= [ 2.58702518 6.67910997 47.19460436] , f(x)= 0.1719181971250397\n", 155 | "59 th iteration, x= [ 2.62151279 6.86411111 49.70399983] , f(x)= 0.15384628620503868\n", 156 | "60 th iteration, x= [ 2.62376503 6.87490002 49.90796457] , f(x)= 0.1457325434985123\n", 157 | "61 th iteration, x= [ 2.62388796 6.87634008 49.90786251] , f(x)= 0.14195989625683178\n", 158 | "62 th iteration, x= [ 2.66916139 7.1076931 53.13347276] , f(x)= 0.13862829824048095\n", 159 | "63 th iteration, x= [ 2.65412248 7.03227553 52.09117436] , f(x)= 0.12291476829925095\n", 160 | "64 th iteration, x= [ 2.65398264 7.03118338 52.09125543] , f(x)= 0.12081201321196944\n", 161 | "65 th iteration, x= [ 2.67238878 7.14665125 53.73331366] , f(x)= 0.10919240485626873\n", 162 | "66 th iteration, x= [ 2.69467226 7.25808099 55.33394695] , f(x)= 0.10803240657241088\n", 163 | "67 th iteration, x= [ 2.69457066 7.25508857 55.33415397] , f(x)= 0.09360478766273336\n", 164 | "68 th iteration, x= [ 2.7329274 7.45760565 58.30283031] , f(x)= 0.09124617781264992\n", 165 | "69 th iteration, x= [ 2.72021156 7.39041855 57.3247572 ] , f(x)= 0.0805646457779455\n", 166 | "70 th iteration, x= [ 2.72010261 7.38941585 57.3248275 ] , f(x)= 0.07899419350132612\n", 167 | "71 th iteration, x= [ 2.73758889 7.49272308 58.86552582] , f(x)= 0.07039129752308566\n", 168 | "72 th iteration, x= [ 2.75253584 7.57319772 60.07861452] , f(x)= 0.06799336271876542\n", 169 | "73 th iteration, x= [ 2.75246581 7.57109952 60.0787539 ] , f(x)= 0.061648194875798694\n", 170 | "74 th iteration, x= [ 2.7713384 7.67404904 61.65101369] , f(x)= 0.05372119627716382\n", 171 | "75 th iteration, x= [ 2.78723241 7.76225803 63.01074581] , f(x)= 0.05319768271224509\n", 172 | "76 th iteration, x= [ 2.7870622 7.75996946 63.01089487] , f(x)= 0.04616735607208302\n", 173 | "77 th iteration, x= [ 2.81406169 7.91587216 65.44154867] , f(x)= 0.04476648406259706\n", 174 | "78 th iteration, x= [ 2.79665853 7.81654923 63.90151555] , f(x)= 0.04187606097843844\n", 175 | "79 th iteration, x= [ 2.79670123 7.8170812 63.90148273] , f(x)= 0.04150459208853409\n", 176 | "80 th iteration, x= [ 2.81912384 7.94082257 65.86241895] , f(x)= 0.03463282498713069\n", 177 | "81 th iteration, x= [ 2.83466019 8.03070177 67.30435796] , f(x)= 0.03203052845013933\n", 178 | "82 th iteration, x= [ 2.83453551 8.02887347 67.30447293] , f(x)= 0.028064672587024324\n", 179 | "83 th iteration, x= [ 2.85167055 8.12886755 68.91907412] , f(x)= 0.023177117010038033\n", 180 | "84 th iteration, x= [ 2.86119915 8.18328891 69.8117755 ] , f(x)= 0.0215378707619593\n", 181 | "85 th iteration, x= [ 2.86112582 8.18199191 69.81185552] , f(x)= 0.0196971359770597\n", 182 | "86 th iteration, x= [ 2.87994071 8.29103647 71.60844082] , f(x)= 0.015949424870018575\n", 183 | "87 th iteration, x= [ 2.88155907 8.30111558 71.79386824] , f(x)= 0.01419346183189707\n", 184 | "88 th iteration, x= [ 2.88159379 8.30144645 71.79384885] , f(x)= 0.01407979428864567\n", 185 | "89 th iteration, x= [ 2.90826592 8.45512486 74.37390537] , f(x)= 0.013442417313111696\n", 186 | "90 th iteration, x= [ 2.89185231 8.35999182 72.78677846] , f(x)= 0.012020108434132249\n", 187 | "91 th iteration, x= [ 2.89187689 8.36047032 72.78675051] , f(x)= 0.011794106203199458\n", 188 | "92 th iteration, x= [ 2.91069406 8.4694034 74.63272859] , f(x)= 0.00871850890378609\n", 189 | "93 th iteration, x= [ 2.92896103 8.5776313 76.49013499] , f(x)= 0.006970763026402732\n", 190 | "94 th iteration, x= [ 2.92889926 8.57635597 76.4902096 ] , f(x)= 0.005582693118588413\n", 191 | "95 th iteration, x= [ 2.9402065 8.64330502 77.64126247] , f(x)= 0.003880112105218761\n", 192 | "96 th iteration, x= [ 2.94958105 8.69901288 78.61460547] , f(x)= 0.0030969535597560414\n", 193 | "97 th iteration, x= [ 2.94954596 8.69832261 78.61464538] , f(x)= 0.0027264498225982965\n", 194 | "98 th iteration, x= [ 2.96155552 8.76980025 79.86515402] , f(x)= 0.0017876709914927847\n", 195 | "99 th iteration, x= [ 2.96182039 8.77186547 79.91000113] , f(x)= 0.0015183795012128597\n", 196 | "100 th iteration, x= [ 2.96184304 8.77209701 79.90998805] , f(x)= 0.0014785416262937633\n", 197 | "101 th iteration, x= [ 2.9743052 8.84572899 81.21600457] , f(x)= 0.000909718479756839\n", 198 | "102 th iteration, x= [ 2.97662182 8.8593872 81.46943194] , f(x)= 0.0007010715951328817\n", 199 | "103 th iteration, x= [ 2.97663484 8.85975955 81.46941112] , f(x)= 0.0006075330813888108\n", 200 | "104 th iteration, x= [ 2.98744519 8.92438619 82.62831045] , f(x)= 0.0002891961681161045\n", 201 | "105 th iteration, x= [ 2.98746174 8.92488017 82.64250227] , f(x)= 0.00017897077683002276\n", 202 | "106 th iteration, x= [ 2.98747569 8.92502231 82.64249432] , f(x)= 0.00016594225430377802\n", 203 | "107 th iteration, x= [ 2.99722628 8.98264838 83.68317937] , f(x)= 4.7974577753456634e-05\n", 204 | "108 th iteration, x= [ 2.99518304 8.97113571 83.47752351] , f(x)= 3.3404637862856576e-05\n", 205 | "109 th iteration, x= [ 2.99519157 8.97123337 83.47751806] , f(x)= 2.757822360834381e-05\n", 206 | "110 th iteration, x= [ 2.99964568 8.99773676 83.95913465] , f(x)= 7.019294745613007e-07\n", 207 | "111 th iteration, x= [ 3.00001947 9.00011496 84.00209693] , f(x)= 9.9858323132762e-10\n", 208 | "112 th iteration, x= [ 3.00001945 9.00011572 84.00209689] , f(x)= 6.588867547066142e-10\n", 209 | "113 th iteration, x= [ 3.00000227 9.00001563 84.00028453] , f(x)= 4.2468315935036195e-11\n", 210 | "114 th iteration, x= [ 3.00000034 9.00000115 84.00002119] , f(x)= 5.865008472997876e-12\n", 211 | "115 th iteration, x= [ 3.00000032 9.00000117 84.00002119] , f(x)= 4.109601552844873e-12\n", 212 | "116 th iteration, x= [ 3.00000001 9.00000018 84.00000332] , f(x)= 9.821514741278904e-14\n", 213 | "117 th iteration, x= [ 3. 8.99999999 83.99999986] , f(x)= 2.59655997488727e-15\n", 214 | "result: x: [ 3. 8.99999999 83.99999986] \n", 215 | "f(x): 2.59655997488727e-15\n", 216 | "245.29560587024054\n" 217 | ] 218 | } 219 | ], 220 | "source": [ 221 | "alpha = 0.4\n", 222 | "beta = 0.8\n", 223 | "maxIter = 3000\n", 224 | "\n", 225 | "# 设置变量初值\n", 226 | "x = np.array([0, 0, 0], dtype=np.float64) #x0\n", 227 | "H = np.eye(3) #H_0\n", 228 | "g = gradient_f(x)\n", 229 | "d = -g\n", 230 | "eta = 1e-5\n", 231 | "eps = 1e-20\n", 232 | "timestep = 0\n", 233 | "lam = []\n", 234 | "while True:\n", 235 | " print(timestep, \"th iteration, x=\", x, \", f(x)=\", f(x))\n", 236 | " # print(\"H=\", np.linalg.svd(H)[1][2])\n", 237 | " # print(\"g=\", g)\n", 238 | " lam.append(np.mean(np.linalg.svd(H)[1]))\n", 239 | " if np.linalg.norm(g) < eta:\n", 240 | " break\n", 241 | " # alpha = linesearch_Armijo(f, x, g, d, alpha, beta)\n", 242 | " alpha = linesearch_Wolfe(f, x, g, d)\n", 243 | " dx = alpha*d\n", 244 | " x += dx\n", 245 | " dg = gradient_f(x) - g\n", 246 | " g += dg\n", 247 | " # H = DFP(H, dx, dg)\n", 248 | " H = BFGS(H, dx, dg)\n", 249 | " d = -H@g\n", 250 | " timestep += 1\n", 251 | " if timestep%3 == 0:\n", 252 | " d = -g\n", 253 | " # print(g)\n", 254 | "print(\"result: x:\", x, \"\\nf(x):\",f(x))\n", 255 | "print(np.mean(lam))" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [] 264 | } 265 | ], 266 | "metadata": { 267 | "interpreter": { 268 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 269 | }, 270 | "kernelspec": { 271 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 272 | "name": "python3" 273 | }, 274 | "language_info": { 275 | "name": "python", 276 | "version": "" 277 | }, 278 | "orig_nbformat": 2 279 | }, 280 | "nbformat": 4, 281 | "nbformat_minor": 2 282 | } -------------------------------------------------------------------------------- /13/134.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# HW13.4 L-BFGS\n", 8 | "By ZincCat\n", 9 | "\n", 10 | "$\\displaystyle\\min_{\\mathbf{x}}f(\\mathbf{x})=\\sum_{i=1}^{n/2}\\left[\\alpha\\left(x_{2i}-x_{2i-1}^{2}\\right)^{2}+\\left(1-x_{2 i-1}\\right)^{2}\\right]$\n", 11 | "\n" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import numpy as np\n", 21 | "from matplotlib import pyplot as plt\n", 22 | "\n", 23 | "# 设置随机种子\n", 24 | "np.random.seed(19890817)" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "# 定义一些辅助函数\n", 34 | "n = 500\n", 35 | "alpha = 100\n", 36 | "def f(x):\n", 37 | " # 计算函数值\n", 38 | " s = 0\n", 39 | " for i in range(n//2):\n", 40 | " s += alpha*(x[2*i+1] - x[2*i]*x[2*i])**2 + (1-x[2*i])**2\n", 41 | " return s\n", 42 | "\n", 43 | "def gradient_f(x):\n", 44 | " # 计算函数梯度\n", 45 | " g = np.zeros(n)\n", 46 | " for i in range(n//2):\n", 47 | " t = 2*alpha*(x[2*i+1] - x[2*i]*x[2*i])\n", 48 | " g[2*i] = -2*t*x[2*i] + 2*x[2*i] - 2\n", 49 | " g[2*i+1] = t\n", 50 | " return g\n", 51 | "\n", 52 | "def linesearch(f, x, g, d, a=0.4, b=0.8):\n", 53 | " # backtrack linesearch using Armijo rules\n", 54 | " t = 1\n", 55 | " value = f(x)\n", 56 | " while f(x + t*d) > value + a*t*np.dot(g, d):\n", 57 | " t *= b\n", 58 | " return t\n", 59 | "\n", 60 | "def L_BFGS(grad, m):\n", 61 | " # Main algorithm of L-BFGS\n", 62 | " q = grad.copy()\n", 63 | " alphalist = []\n", 64 | " for i, xg in enumerate(m):\n", 65 | " alphalist.append(np.dot(xg[0], q)/np.dot(xg[0], xg[1]))\n", 66 | " q -= alphalist[i]*xg[1]\n", 67 | " l = len(m)\n", 68 | " if l > 0: # H_0\n", 69 | " q *= np.dot(m[0][0], m[0][1])/np.dot(m[0][1], m[0][1]) # p\n", 70 | " for i, xg in enumerate(reversed(m)):\n", 71 | " beta = np.dot(xg[1], q)/np.dot(xg[0], xg[1])\n", 72 | " q += (alphalist[l-i-1] - beta)*xg[0]\n", 73 | " return -q" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 4, 79 | "metadata": { 80 | "tags": [] 81 | }, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "0 2000.0\n", 88 | "1 138.03348376521532\n", 89 | "2 220.01246235861774\n", 90 | "3 194.09749416433664\n", 91 | "4 54.13683781308755\n", 92 | "5 6.921774448766465\n", 93 | "6 1.4546908079557794\n", 94 | "7 0.0979480623982054\n", 95 | "8 2.755394772906357e-05\n", 96 | "9 1.8695267089148618e-07\n", 97 | "10 3.34487447947818e-09\n", 98 | "11 1.1658855969216335e-16\n", 99 | "result: x: [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 100 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 101 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 102 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 103 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 104 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 105 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 106 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 107 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 108 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 109 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 110 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 111 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 112 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 113 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 114 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 115 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 116 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 117 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 118 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n", 119 | " 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.] \n", 120 | "f(x): 1.1658855969216335e-16\n" 121 | ] 122 | } 123 | ], 124 | "source": [ 125 | "a = 0.4\n", 126 | "b = 0.8\n", 127 | "maxIter = 200\n", 128 | "\n", 129 | "# 设置变量初值\n", 130 | "x = -np.ones(n)\n", 131 | "g = gradient_f(x)\n", 132 | "d = -g\n", 133 | "eta = 1e-5\n", 134 | "timestep = 0\n", 135 | "m = []\n", 136 | "memlimit = 7\n", 137 | "\n", 138 | "while True:\n", 139 | " print(timestep, f(x))\n", 140 | " # print(timestep, \"th iteration, x=\", x, \", f(x)=\", f(x))\n", 141 | " # print(\"g=\", g)\n", 142 | " if np.linalg.norm(g) < eta:\n", 143 | " break\n", 144 | " d = L_BFGS(g, m)\n", 145 | " alpha = linesearch(f, x, g, d, a, b)\n", 146 | " dx = alpha*d\n", 147 | " x += dx\n", 148 | " dg = gradient_f(x) - g\n", 149 | " g += dg\n", 150 | " if len(m) >= memlimit:\n", 151 | " m.pop()\n", 152 | " m.insert(0, (dx, dg))\n", 153 | " timestep += 1\n", 154 | " if timestep >= maxIter:\n", 155 | " break\n", 156 | "print(\"result: x:\", x, \"\\nf(x):\",f(x))" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [] 165 | } 166 | ], 167 | "metadata": { 168 | "interpreter": { 169 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 170 | }, 171 | "kernelspec": { 172 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 173 | "name": "python3" 174 | }, 175 | "language_info": { 176 | "name": "python", 177 | "version": "" 178 | }, 179 | "orig_nbformat": 2 180 | }, 181 | "nbformat": 4, 182 | "nbformat_minor": 2 183 | } -------------------------------------------------------------------------------- /13/135.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# HW13.5 Majorant Minimization\n", 8 | "By ZincCat\n", 9 | "\n", 10 | "$\\min_{\\mathbf{x}} \\frac{1}{2} \\|\\mathbf{A}\\mathbf{x}-\\mathbf{b}\\|^{2} + \\lambda\\|\\mathbf{x}\\|_{1}$" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import numpy as np\n", 20 | "from lbfgs import LBFGS\n", 21 | "from bfgs import BFGS\n", 22 | "from matplotlib import pyplot as plt\n", 23 | "\n", 24 | "# 设置随机种子\n", 25 | "np.random.seed(19890817)" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 2, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "# 定义一些辅助函数\n", 35 | "n = 20\n", 36 | "A = np.random.normal(3, 5, (n, n))\n", 37 | "b = np.random.normal(3, 3, n) \n", 38 | "x0 = np.ones(n)" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 3, 44 | "metadata": {}, 45 | "outputs": [ 46 | { 47 | "name": "stdout", 48 | "output_type": "stream", 49 | "text": [ 50 | "4353.79001549459\n" 51 | ] 52 | } 53 | ], 54 | "source": [ 55 | "L = np.linalg.svd(A.T@A)[1][0]\n", 56 | "print(L)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 7, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "lam = 1e-3 # lambda\n", 66 | "def f(x):\n", 67 | " # 原函数\n", 68 | " return 1/2*(np.linalg.norm(A@x-b))**2 + lam*np.linalg.norm(x, ord=1)\n", 69 | "def grad1(xk):\n", 70 | " # Lipschitz majorant function的梯度\n", 71 | " return lambda x: A.T@(A@xk-b) + L*(x-xk) + lam*np.sign(x)\n", 72 | "def grad2(xk):\n", 73 | " # 第二个majorant function的梯度\n", 74 | " return lambda x: A.T@(A@x-b) + lam*x/np.abs(xk)\n", 75 | "def grad(x):\n", 76 | " # 原函数的梯度\n", 77 | " return A.T@(A@x-b) + lam*np.sign(x)\n", 78 | "def G1(xk):\n", 79 | " # Lipschitz gradient majorant\n", 80 | " return lambda x: 1/2*(np.linalg.norm(A@xk-b))**2 + np.dot(A.T@(A@xk-b), x-xk) + L/2*(np.linalg.norm(x-xk))**2 + lam*np.linalg.norm(x, ord=1)\n", 81 | "def G2(xk):\n", 82 | " # variational majorant function\n", 83 | " return lambda x: 1/2*(np.linalg.norm(A@x-b))**2 + lam*np.dot(x, x/np.abs(xk))/2" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 10, 89 | "metadata": { 90 | "tags": [ 91 | "outputPrepend" 92 | ] 93 | }, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | " f(x)= 0.007156736217091667\n", 100 | "15436 th iteration, f(x)= 0.007156736217028679\n", 101 | "15437 th iteration, f(x)= 0.007156736216965811\n", 102 | "15438 th iteration, f(x)= 0.007156736216903058\n", 103 | "15439 th iteration, f(x)= 0.007156736216840425\n", 104 | "15440 th iteration, f(x)= 0.007156736216777909\n", 105 | "15441 th iteration, f(x)= 0.007156736216715508\n", 106 | "15442 th iteration, f(x)= 0.007156736216653224\n", 107 | "15443 th iteration, f(x)= 0.007156736216591059\n", 108 | "15444 th iteration, f(x)= 0.007156736216529008\n", 109 | "15445 th iteration, f(x)= 0.00715673621646707\n", 110 | "15446 th iteration, f(x)= 0.0071567362164052534\n", 111 | "15447 th iteration, f(x)= 0.007156736216343547\n", 112 | "15448 th iteration, f(x)= 0.007156736216281962\n", 113 | "15449 th iteration, f(x)= 0.007156736216220488\n", 114 | "15450 th iteration, f(x)= 0.007156736216159131\n", 115 | "15451 th iteration, f(x)= 0.007156736216097886\n", 116 | "15452 th iteration, f(x)= 0.007156736216036756\n", 117 | "15453 th iteration, f(x)= 0.007156736215975744\n", 118 | "15454 th iteration, f(x)= 0.00715673621591484\n", 119 | "15455 th iteration, f(x)= 0.007156736215854051\n", 120 | "15456 th iteration, f(x)= 0.007156736215793381\n", 121 | "15457 th iteration, f(x)= 0.007156736215732819\n", 122 | "15458 th iteration, f(x)= 0.0071567362156723735\n", 123 | "15459 th iteration, f(x)= 0.007156736215612039\n", 124 | "15460 th iteration, f(x)= 0.007156736215551817\n", 125 | "15461 th iteration, f(x)= 0.00715673621549171\n", 126 | "15462 th iteration, f(x)= 0.007156736215431713\n", 127 | "15463 th iteration, f(x)= 0.0071567362153718275\n", 128 | "15464 th iteration, f(x)= 0.007156736215312058\n", 129 | "15465 th iteration, f(x)= 0.007156736215252395\n", 130 | "15466 th iteration, f(x)= 0.0071567362151928465\n", 131 | "15467 th iteration, f(x)= 0.007156736215133406\n", 132 | "15468 th iteration, f(x)= 0.00715673621507408\n", 133 | "15469 th iteration, f(x)= 0.007156736215014866\n", 134 | "15470 th iteration, f(x)= 0.00715673621495576\n", 135 | "15471 th iteration, f(x)= 0.007156736214896764\n", 136 | "15472 th iteration, f(x)= 0.007156736214837882\n", 137 | "15473 th iteration, f(x)= 0.007156736214779106\n", 138 | "15474 th iteration, f(x)= 0.007156736214720441\n", 139 | "15475 th iteration, f(x)= 0.0071567362146618884\n", 140 | "15476 th iteration, f(x)= 0.00715673621460344\n", 141 | "15477 th iteration, f(x)= 0.007156736214545104\n", 142 | "15478 th iteration, f(x)= 0.0071567362144868765\n", 143 | "15479 th iteration, f(x)= 0.0071567362144287616\n", 144 | "15480 th iteration, f(x)= 0.007156736214370748\n", 145 | "15481 th iteration, f(x)= 0.007156736214312847\n", 146 | "15482 th iteration, f(x)= 0.007156736214255054\n", 147 | "15483 th iteration, f(x)= 0.0071567362141973685\n", 148 | "15484 th iteration, f(x)= 0.007156736214139789\n", 149 | "15485 th iteration, f(x)= 0.00715673621408232\n", 150 | "15486 th iteration, f(x)= 0.0071567362140249596\n", 151 | "15487 th iteration, f(x)= 0.007156736213967703\n", 152 | "15488 th iteration, f(x)= 0.007156736213910553\n", 153 | "15489 th iteration, f(x)= 0.007156736213853513\n", 154 | "15490 th iteration, f(x)= 0.007156736213796575\n", 155 | "15491 th iteration, f(x)= 0.007156736213739751\n", 156 | "15492 th iteration, f(x)= 0.007156736213683026\n", 157 | "15493 th iteration, f(x)= 0.007156736213626411\n", 158 | "15494 th iteration, f(x)= 0.0071567362135699\n", 159 | "15495 th iteration, f(x)= 0.007156736213513497\n", 160 | "15496 th iteration, f(x)= 0.007156736213457195\n", 161 | "15497 th iteration, f(x)= 0.007156736213401005\n", 162 | "15498 th iteration, f(x)= 0.007156736213344913\n", 163 | "15499 th iteration, f(x)= 0.0071567362132889285\n", 164 | "15500 th iteration, f(x)= 0.007156736213233049\n", 165 | "15501 th iteration, f(x)= 0.007156736213177273\n", 166 | "15502 th iteration, f(x)= 0.007156736213121601\n", 167 | "15503 th iteration, f(x)= 0.007156736213066034\n", 168 | "15504 th iteration, f(x)= 0.007156736213010573\n", 169 | "15505 th iteration, f(x)= 0.007156736212955211\n", 170 | "15506 th iteration, f(x)= 0.007156736212899957\n", 171 | "15507 th iteration, f(x)= 0.007156736212844801\n", 172 | "15508 th iteration, f(x)= 0.007156736212789754\n", 173 | "15509 th iteration, f(x)= 0.007156736212734802\n", 174 | "15510 th iteration, f(x)= 0.0071567362126799625\n", 175 | "15511 th iteration, f(x)= 0.007156736212625219\n", 176 | "15512 th iteration, f(x)= 0.007156736212570581\n", 177 | "15513 th iteration, f(x)= 0.007156736212516043\n", 178 | "15514 th iteration, f(x)= 0.007156736212461607\n", 179 | "15515 th iteration, f(x)= 0.0071567362124072735\n", 180 | "15516 th iteration, f(x)= 0.0071567362123530435\n", 181 | "15517 th iteration, f(x)= 0.0071567362122989114\n", 182 | "15518 th iteration, f(x)= 0.007156736212244881\n", 183 | "15519 th iteration, f(x)= 0.007156736212190952\n", 184 | "15520 th iteration, f(x)= 0.007156736212137125\n", 185 | "15521 th iteration, f(x)= 0.007156736212083395\n", 186 | "15522 th iteration, f(x)= 0.007156736212029767\n", 187 | "15523 th iteration, f(x)= 0.007156736211976242\n", 188 | "15524 th iteration, f(x)= 0.007156736211922816\n", 189 | "15525 th iteration, f(x)= 0.007156736211869488\n", 190 | "15526 th iteration, f(x)= 0.007156736211816261\n", 191 | "15527 th iteration, f(x)= 0.007156736211763135\n", 192 | "15528 th iteration, f(x)= 0.0071567362117101055\n", 193 | "15529 th iteration, f(x)= 0.0071567362116571765\n", 194 | "15530 th iteration, f(x)= 0.007156736211604346\n", 195 | "15531 th iteration, f(x)= 0.007156736211551615\n", 196 | "15532 th iteration, f(x)= 0.007156736211498983\n", 197 | "15533 th iteration, f(x)= 0.007156736211446447\n", 198 | "15534 th iteration, f(x)= 0.00715673621139401\n", 199 | "15535 th iteration, f(x)= 0.007156736211341674\n", 200 | "15536 th iteration, f(x)= 0.007156736211289432\n", 201 | "15537 th iteration, f(x)= 0.007156736211237288\n", 202 | "15538 th iteration, f(x)= 0.007156736211185244\n", 203 | "15539 th iteration, f(x)= 0.0071567362111332935\n", 204 | "15540 th iteration, f(x)= 0.007156736211081443\n", 205 | "15541 th iteration, f(x)= 0.007156736211029686\n", 206 | "15542 th iteration, f(x)= 0.007156736210978029\n", 207 | "15543 th iteration, f(x)= 0.007156736210926469\n", 208 | "15544 th iteration, f(x)= 0.007156736210875003\n", 209 | "15545 th iteration, f(x)= 0.007156736210823634\n", 210 | "15546 th iteration, f(x)= 0.0071567362107723635\n", 211 | "15547 th iteration, f(x)= 0.0071567362107211865\n", 212 | "15548 th iteration, f(x)= 0.007156736210670104\n", 213 | "15549 th iteration, f(x)= 0.00715673621061912\n", 214 | "15550 th iteration, f(x)= 0.007156736210568229\n", 215 | "15551 th iteration, f(x)= 0.007156736210517434\n", 216 | "15552 th iteration, f(x)= 0.007156736210466734\n", 217 | "15553 th iteration, f(x)= 0.007156736210416128\n", 218 | "15554 th iteration, f(x)= 0.007156736210365616\n", 219 | "15555 th iteration, f(x)= 0.007156736210315199\n", 220 | "15556 th iteration, f(x)= 0.0071567362102648744\n", 221 | "15557 th iteration, f(x)= 0.007156736210214647\n", 222 | "15558 th iteration, f(x)= 0.007156736210164514\n", 223 | "15559 th iteration, f(x)= 0.007156736210114474\n", 224 | "15560 th iteration, f(x)= 0.007156736210064526\n", 225 | "15561 th iteration, f(x)= 0.007156736210014673\n", 226 | "15562 th iteration, f(x)= 0.007156736209964909\n", 227 | "15563 th iteration, f(x)= 0.0071567362099152435\n", 228 | "15564 th iteration, f(x)= 0.007156736209865668\n", 229 | "15565 th iteration, f(x)= 0.007156736209816185\n", 230 | "15566 th iteration, f(x)= 0.0071567362097667946\n", 231 | "15567 th iteration, f(x)= 0.007156736209717497\n", 232 | "15568 th iteration, f(x)= 0.00715673620966829\n", 233 | "15569 th iteration, f(x)= 0.007156736209619178\n", 234 | "15570 th iteration, f(x)= 0.007156736209570158\n", 235 | "15571 th iteration, f(x)= 0.007156736209521225\n", 236 | "15572 th iteration, f(x)= 0.007156736209472388\n", 237 | "15573 th iteration, f(x)= 0.00715673620942364\n", 238 | "15574 th iteration, f(x)= 0.007156736209374986\n", 239 | "15575 th iteration, f(x)= 0.007156736209326417\n", 240 | "15576 th iteration, f(x)= 0.0071567362092779425\n", 241 | "15577 th iteration, f(x)= 0.00715673620922956\n", 242 | "15578 th iteration, f(x)= 0.007156736209181266\n", 243 | "15579 th iteration, f(x)= 0.007156736209133061\n", 244 | "15580 th iteration, f(x)= 0.007156736209084948\n", 245 | "15581 th iteration, f(x)= 0.007156736209036927\n", 246 | "15582 th iteration, f(x)= 0.007156736208988993\n", 247 | "15583 th iteration, f(x)= 0.007156736208941146\n", 248 | "15584 th iteration, f(x)= 0.007156736208893392\n", 249 | "15585 th iteration, f(x)= 0.007156736208845729\n", 250 | "15586 th iteration, f(x)= 0.007156736208798153\n", 251 | "15587 th iteration, f(x)= 0.0071567362087506655\n", 252 | "15588 th iteration, f(x)= 0.007156736208703266\n", 253 | "15589 th iteration, f(x)= 0.007156736208655955\n", 254 | "15590 th iteration, f(x)= 0.007156736208608732\n", 255 | "15591 th iteration, f(x)= 0.007156736208561599\n", 256 | "15592 th iteration, f(x)= 0.007156736208514555\n", 257 | "15593 th iteration, f(x)= 0.007156736208467597\n", 258 | "15594 th iteration, f(x)= 0.0071567362084207255\n", 259 | "15595 th iteration, f(x)= 0.007156736208373943\n", 260 | "15596 th iteration, f(x)= 0.00715673620832725\n", 261 | "15597 th iteration, f(x)= 0.007156736208280642\n", 262 | "15598 th iteration, f(x)= 0.007156736208234124\n", 263 | "15599 th iteration, f(x)= 0.007156736208187688\n", 264 | "15600 th iteration, f(x)= 0.007156736208141344\n", 265 | "15601 th iteration, f(x)= 0.007156736208095083\n", 266 | "15602 th iteration, f(x)= 0.007156736208048909\n", 267 | "15603 th iteration, f(x)= 0.007156736208002821\n", 268 | "15604 th iteration, f(x)= 0.007156736207956821\n", 269 | "15605 th iteration, f(x)= 0.007156736207910905\n", 270 | "15606 th iteration, f(x)= 0.007156736207865075\n", 271 | "15607 th iteration, f(x)= 0.00715673620781933\n", 272 | "15608 th iteration, f(x)= 0.0071567362077736745\n", 273 | "15609 th iteration, f(x)= 0.007156736207728101\n", 274 | "15610 th iteration, f(x)= 0.007156736207682613\n", 275 | "15611 th iteration, f(x)= 0.007156736207637209\n", 276 | "15612 th iteration, f(x)= 0.007156736207591893\n", 277 | "15613 th iteration, f(x)= 0.007156736207546659\n", 278 | "15614 th iteration, f(x)= 0.007156736207501511\n", 279 | "15615 th iteration, f(x)= 0.007156736207456446\n", 280 | "15616 th iteration, f(x)= 0.007156736207411467\n", 281 | "15617 th iteration, f(x)= 0.007156736207366572\n", 282 | "15618 th iteration, f(x)= 0.00715673620732176\n", 283 | "15619 th iteration, f(x)= 0.007156736207277031\n", 284 | "15620 th iteration, f(x)= 0.0071567362072323835\n", 285 | "15621 th iteration, f(x)= 0.007156736207187824\n", 286 | "15622 th iteration, f(x)= 0.007156736207143345\n", 287 | "15623 th iteration, f(x)= 0.007156736207098953\n", 288 | "15624 th iteration, f(x)= 0.0071567362070546385\n", 289 | "15625 th iteration, f(x)= 0.00715673620701041\n", 290 | "15626 th iteration, f(x)= 0.007156736206966264\n", 291 | "15627 th iteration, f(x)= 0.0071567362069221985\n", 292 | "15628 th iteration, f(x)= 0.007156736206878217\n", 293 | "15629 th iteration, f(x)= 0.007156736206834319\n", 294 | "15630 th iteration, f(x)= 0.0071567362067905\n", 295 | "15631 th iteration, f(x)= 0.007156736206746767\n", 296 | "15632 th iteration, f(x)= 0.007156736206703112\n", 297 | "15633 th iteration, f(x)= 0.0071567362066595405\n", 298 | "15634 th iteration, f(x)= 0.007156736206616047\n", 299 | "15635 th iteration, f(x)= 0.007156736206572639\n", 300 | "15636 th iteration, f(x)= 0.00715673620652931\n", 301 | "15637 th iteration, f(x)= 0.007156736206486063\n", 302 | "15638 th iteration, f(x)= 0.007156736206442896\n", 303 | "15639 th iteration, f(x)= 0.00715673620639981\n", 304 | "15640 th iteration, f(x)= 0.0071567362063568035\n", 305 | "15641 th iteration, f(x)= 0.007156736206313878\n", 306 | "15642 th iteration, f(x)= 0.0071567362062710345\n", 307 | "15643 th iteration, f(x)= 0.00715673620622827\n", 308 | "15644 th iteration, f(x)= 0.0071567362061855854\n", 309 | "15645 th iteration, f(x)= 0.00715673620614298\n", 310 | "15646 th iteration, f(x)= 0.007156736206100455\n", 311 | "15647 th iteration, f(x)= 0.007156736206058009\n", 312 | "15648 th iteration, f(x)= 0.00715673620601564\n", 313 | "15649 th iteration, f(x)= 0.007156736205973354\n", 314 | "15650 th iteration, f(x)= 0.007156736205931146\n", 315 | "15651 th iteration, f(x)= 0.007156736205889016\n", 316 | "15652 th iteration, f(x)= 0.007156736205846965\n", 317 | "15653 th iteration, f(x)= 0.007156736205804993\n", 318 | "15654 th iteration, f(x)= 0.0071567362057631\n", 319 | "15655 th iteration, f(x)= 0.007156736205721287\n", 320 | "15656 th iteration, f(x)= 0.007156736205679545\n", 321 | "15657 th iteration, f(x)= 0.007156736205637888\n", 322 | "15658 th iteration, f(x)= 0.007156736205596306\n", 323 | "15659 th iteration, f(x)= 0.007156736205554801\n", 324 | "15660 th iteration, f(x)= 0.007156736205513374\n", 325 | "15661 th iteration, f(x)= 0.007156736205472028\n", 326 | "15662 th iteration, f(x)= 0.007156736205430757\n", 327 | "15663 th iteration, f(x)= 0.007156736205389558\n", 328 | "15664 th iteration, f(x)= 0.0071567362053484435\n", 329 | "15665 th iteration, f(x)= 0.007156736205307401\n", 330 | "15666 th iteration, f(x)= 0.0071567362052664405\n", 331 | "15667 th iteration, f(x)= 0.007156736205225554\n", 332 | "15668 th iteration, f(x)= 0.00715673620518474\n", 333 | "15669 th iteration, f(x)= 0.007156736205144005\n", 334 | "15670 th iteration, f(x)= 0.007156736205103346\n", 335 | "15671 th iteration, f(x)= 0.007156736205062764\n", 336 | "15672 th iteration, f(x)= 0.007156736205022257\n", 337 | "15673 th iteration, f(x)= 0.007156736204981826\n", 338 | "15674 th iteration, f(x)= 0.00715673620494147\n", 339 | "15675 th iteration, f(x)= 0.007156736204901191\n", 340 | "15676 th iteration, f(x)= 0.007156736204860985\n", 341 | "15677 th iteration, f(x)= 0.007156736204820858\n", 342 | "15678 th iteration, f(x)= 0.007156736204780802\n", 343 | "15679 th iteration, f(x)= 0.007156736204740823\n", 344 | "15680 th iteration, f(x)= 0.007156736204700918\n", 345 | "15681 th iteration, f(x)= 0.0071567362046610844\n", 346 | "15682 th iteration, f(x)= 0.007156736204621331\n", 347 | "15683 th iteration, f(x)= 0.007156736204581647\n", 348 | "15684 th iteration, f(x)= 0.0071567362045420425\n", 349 | "15685 th iteration, f(x)= 0.007156736204502507\n", 350 | "15686 th iteration, f(x)= 0.007156736204463047\n", 351 | "15687 th iteration, f(x)= 0.007156736204423661\n", 352 | "15688 th iteration, f(x)= 0.007156736204384349\n", 353 | "15689 th iteration, f(x)= 0.007156736204345108\n", 354 | "15690 th iteration, f(x)= 0.007156736204305946\n", 355 | "15691 th iteration, f(x)= 0.007156736204266852\n", 356 | "15692 th iteration, f(x)= 0.007156736204227834\n", 357 | "15693 th iteration, f(x)= 0.007156736204188886\n", 358 | "15694 th iteration, f(x)= 0.007156736204150012\n", 359 | "15695 th iteration, f(x)= 0.0071567362041112135\n", 360 | "15696 th iteration, f(x)= 0.007156736204072484\n", 361 | "15697 th iteration, f(x)= 0.007156736204033829\n", 362 | "15698 th iteration, f(x)= 0.007156736203995245\n", 363 | "15699 th iteration, f(x)= 0.00715673620395673\n", 364 | "15700 th iteration, f(x)= 0.00715673620391829\n", 365 | "15701 th iteration, f(x)= 0.007156736203879925\n", 366 | "15702 th iteration, f(x)= 0.007156736203841627\n", 367 | "15703 th iteration, f(x)= 0.0071567362038034025\n", 368 | "15704 th iteration, f(x)= 0.007156736203765249\n", 369 | "15705 th iteration, f(x)= 0.007156736203727167\n", 370 | "15706 th iteration, f(x)= 0.007156736203689156\n", 371 | "15707 th iteration, f(x)= 0.007156736203651215\n", 372 | "15708 th iteration, f(x)= 0.00715673620361335\n", 373 | "15709 th iteration, f(x)= 0.007156736203575553\n", 374 | "15710 th iteration, f(x)= 0.007156736203537823\n", 375 | "15711 th iteration, f(x)= 0.007156736203500163\n", 376 | "15712 th iteration, f(x)= 0.00715673620346258\n", 377 | "15713 th iteration, f(x)= 0.007156736203425064\n", 378 | "15714 th iteration, f(x)= 0.007156736203387617\n", 379 | "15715 th iteration, f(x)= 0.007156736203350241\n", 380 | "15716 th iteration, f(x)= 0.007156736203312933\n", 381 | "15717 th iteration, f(x)= 0.007156736203275698\n", 382 | "15718 th iteration, f(x)= 0.00715673620323853\n", 383 | "15719 th iteration, f(x)= 0.007156736203201432\n", 384 | "15720 th iteration, f(x)= 0.007156736203164405\n", 385 | "15721 th iteration, f(x)= 0.007156736203127447\n", 386 | "15722 th iteration, f(x)= 0.007156736203090556\n", 387 | "15723 th iteration, f(x)= 0.007156736203053736\n", 388 | "15724 th iteration, f(x)= 0.007156736203016983\n", 389 | "15725 th iteration, f(x)= 0.007156736202980299\n", 390 | "15726 th iteration, f(x)= 0.007156736202943682\n", 391 | "15727 th iteration, f(x)= 0.007156736202907138\n", 392 | "15728 th iteration, f(x)= 0.00715673620287066\n", 393 | "15729 th iteration, f(x)= 0.007156736202834248\n", 394 | "15730 th iteration, f(x)= 0.007156736202797906\n", 395 | "15731 th iteration, f(x)= 0.007156736202761632\n", 396 | "15732 th iteration, f(x)= 0.007156736202725425\n", 397 | "15733 th iteration, f(x)= 0.007156736202689289\n", 398 | "15734 th iteration, f(x)= 0.007156736202653217\n", 399 | "15735 th iteration, f(x)= 0.007156736202617214\n", 400 | "15736 th iteration, f(x)= 0.007156736202581274\n", 401 | "15737 th iteration, f(x)= 0.007156736202545406\n", 402 | "15738 th iteration, f(x)= 0.007156736202509603\n", 403 | "15739 th iteration, f(x)= 0.007156736202473869\n", 404 | "15740 th iteration, f(x)= 0.007156736202438199\n", 405 | "15741 th iteration, f(x)= 0.007156736202402599\n", 406 | "15742 th iteration, f(x)= 0.007156736202367063\n", 407 | "15743 th iteration, f(x)= 0.007156736202331592\n", 408 | "15744 th iteration, f(x)= 0.0071567362022961906\n", 409 | "15745 th iteration, f(x)= 0.007156736202260856\n", 410 | "15746 th iteration, f(x)= 0.007156736202225585\n", 411 | "15747 th iteration, f(x)= 0.007156736202190379\n", 412 | "15748 th iteration, f(x)= 0.007156736202155239\n", 413 | "15749 th iteration, f(x)= 0.007156736202120166\n", 414 | "15750 th iteration, f(x)= 0.00715673620208516\n", 415 | "15751 th iteration, f(x)= 0.0071567362020502154\n", 416 | "15752 th iteration, f(x)= 0.007156736202015341\n", 417 | "15753 th iteration, f(x)= 0.007156736201980527\n", 418 | "15754 th iteration, f(x)= 0.007156736201945782\n", 419 | "15755 th iteration, f(x)= 0.007156736201911102\n", 420 | "15756 th iteration, f(x)= 0.007156736201876482\n", 421 | "15757 th iteration, f(x)= 0.007156736201841932\n", 422 | "15758 th iteration, f(x)= 0.007156736201807443\n", 423 | "15759 th iteration, f(x)= 0.007156736201773021\n", 424 | "15760 th iteration, f(x)= 0.007156736201738661\n", 425 | "15761 th iteration, f(x)= 0.007156736201704368\n", 426 | "15762 th iteration, f(x)= 0.007156736201670137\n", 427 | "15763 th iteration, f(x)= 0.007156736201635968\n", 428 | "15764 th iteration, f(x)= 0.007156736201601868\n", 429 | "15765 th iteration, f(x)= 0.007156736201567826\n", 430 | "15766 th iteration, f(x)= 0.007156736201533853\n", 431 | "15767 th iteration, f(x)= 0.007156736201499942\n", 432 | "15768 th iteration, f(x)= 0.0071567362014660916\n", 433 | "15769 th iteration, f(x)= 0.007156736201432308\n", 434 | "15770 th iteration, f(x)= 0.007156736201398587\n", 435 | "15771 th iteration, f(x)= 0.007156736201364926\n", 436 | "15772 th iteration, f(x)= 0.0071567362013313296\n", 437 | "15773 th iteration, f(x)= 0.007156736201297797\n", 438 | "15774 th iteration, f(x)= 0.007156736201264324\n", 439 | "15775 th iteration, f(x)= 0.007156736201230917\n", 440 | "15776 th iteration, f(x)= 0.007156736201197571\n", 441 | "15777 th iteration, f(x)= 0.007156736201164286\n", 442 | "15778 th iteration, f(x)= 0.007156736201131068\n", 443 | "15779 th iteration, f(x)= 0.007156736201097907\n", 444 | "15780 th iteration, f(x)= 0.007156736201064808\n", 445 | "15781 th iteration, f(x)= 0.007156736201031774\n", 446 | "15782 th iteration, f(x)= 0.007156736200998798\n", 447 | "15783 th iteration, f(x)= 0.007156736200965888\n", 448 | "15784 th iteration, f(x)= 0.0071567362009330405\n", 449 | "15785 th iteration, f(x)= 0.00715673620090025\n", 450 | "15786 th iteration, f(x)= 0.00715673620086752\n", 451 | "15787 th iteration, f(x)= 0.007156736200834856\n", 452 | "15788 th iteration, f(x)= 0.00715673620080225\n", 453 | "15789 th iteration, f(x)= 0.007156736200769703\n", 454 | "15790 th iteration, f(x)= 0.00715673620073722\n", 455 | "15791 th iteration, f(x)= 0.007156736200704796\n", 456 | "15792 th iteration, f(x)= 0.007156736200672436\n", 457 | "15793 th iteration, f(x)= 0.007156736200640132\n", 458 | "15794 th iteration, f(x)= 0.00715673620060789\n", 459 | "15795 th iteration, f(x)= 0.00715673620057571\n", 460 | "15796 th iteration, f(x)= 0.00715673620054359\n", 461 | "15797 th iteration, f(x)= 0.007156736200511528\n", 462 | "15798 th iteration, f(x)= 0.007156736200479527\n", 463 | "15799 th iteration, f(x)= 0.007156736200447582\n", 464 | "15800 th iteration, f(x)= 0.007156736200415702\n", 465 | "15801 th iteration, f(x)= 0.007156736200383878\n", 466 | "15802 th iteration, f(x)= 0.007156736200352118\n", 467 | "15803 th iteration, f(x)= 0.007156736200320413\n", 468 | "15804 th iteration, f(x)= 0.007156736200288768\n", 469 | "15805 th iteration, f(x)= 0.007156736200257182\n", 470 | "15806 th iteration, f(x)= 0.007156736200225658\n", 471 | "15807 th iteration, f(x)= 0.0071567362001941906\n", 472 | "15808 th iteration, f(x)= 0.007156736200162782\n", 473 | "15809 th iteration, f(x)= 0.007156736200131434\n", 474 | "15810 th iteration, f(x)= 0.007156736200100141\n", 475 | "15811 th iteration, f(x)= 0.00715673620006891\n", 476 | "15812 th iteration, f(x)= 0.007156736200037736\n", 477 | "15813 th iteration, f(x)= 0.007156736200006617\n", 478 | "15814 th iteration, f(x)= 0.007156736199975563\n", 479 | "15815 th iteration, f(x)= 0.007156736199944561\n", 480 | "15816 th iteration, f(x)= 0.0071567361999136225\n", 481 | "15817 th iteration, f(x)= 0.007156736199882734\n", 482 | "15818 th iteration, f(x)= 0.007156736199851907\n", 483 | "15819 th iteration, f(x)= 0.00715673619982114\n", 484 | "15820 th iteration, f(x)= 0.00715673619979043\n", 485 | "15821 th iteration, f(x)= 0.007156736199759774\n", 486 | "15822 th iteration, f(x)= 0.00715673619972918\n", 487 | "15823 th iteration, f(x)= 0.007156736199698642\n", 488 | "15824 th iteration, f(x)= 0.0071567361996681565\n", 489 | "15825 th iteration, f(x)= 0.007156736199637733\n", 490 | "15826 th iteration, f(x)= 0.007156736199607364\n", 491 | "15827 th iteration, f(x)= 0.007156736199577051\n", 492 | "15828 th iteration, f(x)= 0.007156736199546795\n", 493 | "15829 th iteration, f(x)= 0.0071567361995166\n", 494 | "15830 th iteration, f(x)= 0.007156736199486455\n", 495 | "15831 th iteration, f(x)= 0.007156736199456369\n", 496 | "15832 th iteration, f(x)= 0.007156736199426341\n", 497 | "15833 th iteration, f(x)= 0.0071567361993963696\n", 498 | "15834 th iteration, f(x)= 0.00715673619936645\n", 499 | "15835 th iteration, f(x)= 0.007156736199336588\n", 500 | "15836 th iteration, f(x)= 0.007156736199306783\n", 501 | "15837 th iteration, f(x)= 0.007156736199277032\n", 502 | "15838 th iteration, f(x)= 0.0071567361992473395\n", 503 | "15839 th iteration, f(x)= 0.0071567361992177\n", 504 | "15840 th iteration, f(x)= 0.007156736199188118\n", 505 | "15841 th iteration, f(x)= 0.00715673619915859\n", 506 | "15842 th iteration, f(x)= 0.007156736199129116\n", 507 | "15843 th iteration, f(x)= 0.007156736199099697\n", 508 | "15844 th iteration, f(x)= 0.0071567361990703335\n", 509 | "15845 th iteration, f(x)= 0.007156736199041028\n", 510 | "15846 th iteration, f(x)= 0.007156736199011771\n", 511 | "15847 th iteration, f(x)= 0.007156736198982575\n", 512 | "15848 th iteration, f(x)= 0.007156736198953432\n", 513 | "15849 th iteration, f(x)= 0.007156736198924341\n", 514 | "15850 th iteration, f(x)= 0.007156736198895307\n", 515 | "15851 th iteration, f(x)= 0.007156736198866323\n", 516 | "15852 th iteration, f(x)= 0.0071567361988374\n", 517 | "15853 th iteration, f(x)= 0.0071567361988085255\n", 518 | "15854 th iteration, f(x)= 0.007156736198779706\n", 519 | "15855 th iteration, f(x)= 0.007156736198750938\n", 520 | "15856 th iteration, f(x)= 0.007156736198722227\n", 521 | "15857 th iteration, f(x)= 0.007156736198693576\n", 522 | "15858 th iteration, f(x)= 0.0071567361986649685\n", 523 | "15859 th iteration, f(x)= 0.007156736198636417\n", 524 | "15860 th iteration, f(x)= 0.007156736198607921\n" 525 | ] 526 | } 527 | ], 528 | "source": [ 529 | "# 1. Lipschitz gradient majorant\n", 530 | "x = x0.copy()\n", 531 | "eta = 1e-5\n", 532 | "maxIter = 30000\n", 533 | "g = G1(x)\n", 534 | "for i in range(maxIter):\n", 535 | " print(i, \"th iteration, f(x)=\", f(x))\n", 536 | " x = BFGS(x, g, grad1(x), n=n, eps = 1e-30, eta=1e-5, maxIter=10000, display=False)\n", 537 | " g = G1(x)\n", 538 | " if np.linalg.norm(grad(x)) < eta:\n", 539 | " break" 540 | ] 541 | }, 542 | { 543 | "cell_type": "code", 544 | "execution_count": 13, 545 | "metadata": { 546 | "tags": [] 547 | }, 548 | "outputs": [ 549 | { 550 | "name": "stdout", 551 | "output_type": "stream", 552 | "text": [ 553 | "0 th iteration, f(x)= 34014.40633230235\n", 554 | "1 th iteration, f(x)= 0.007157022779348722\n", 555 | "2 th iteration, f(x)= 0.00715673620460766\n" 556 | ] 557 | } 558 | ], 559 | "source": [ 560 | "# 2. variational majorant function\n", 561 | "x = x0.copy()\n", 562 | "eta = 1e-5\n", 563 | "maxIter = 500\n", 564 | "g = G2(x)\n", 565 | "for i in range(maxIter):\n", 566 | " print(i, \"th iteration, f(x)=\", f(x))\n", 567 | " x = BFGS(x, g, grad2(x), n=n, eps = 1e-30, eta=1e-5, maxIter=50, display=False)\n", 568 | " g = G2(x)\n", 569 | " if np.linalg.norm(grad(x)) < eta:\n", 570 | " break" 571 | ] 572 | }, 573 | { 574 | "cell_type": "code", 575 | "execution_count": 9, 576 | "metadata": {}, 577 | "outputs": [ 578 | { 579 | "name": "stdout", 580 | "output_type": "stream", 581 | "text": [ 582 | "0 th iteration, f(x)= 34014.40633230235\n", 583 | "1 th iteration, f(x)= 2291.679326044753\n", 584 | "2 th iteration, f(x)= 1957.1101064777113\n", 585 | "3 th iteration, f(x)= 1882.4287405418183\n", 586 | "4 th iteration, f(x)= 1857.0881833929786\n", 587 | "5 th iteration, f(x)= 1838.6493876902111\n", 588 | "6 th iteration, f(x)= 1800.7849290664635\n", 589 | "7 th iteration, f(x)= 1760.3617652784192\n", 590 | "8 th iteration, f(x)= 1681.4262954544977\n", 591 | "9 th iteration, f(x)= 1576.937191454927\n", 592 | "10 th iteration, f(x)= 1402.1502767324323\n", 593 | "11 th iteration, f(x)= 1271.0111857486038\n", 594 | "12 th iteration, f(x)= 910.8846708762891\n", 595 | "13 th iteration, f(x)= 448.1025462048118\n", 596 | "14 th iteration, f(x)= 330.877280085147\n", 597 | "15 th iteration, f(x)= 280.123771258122\n", 598 | "16 th iteration, f(x)= 232.5162053287782\n", 599 | "17 th iteration, f(x)= 88.92704246172858\n", 600 | "18 th iteration, f(x)= 67.38263969220485\n", 601 | "19 th iteration, f(x)= 51.09222688445992\n", 602 | "20 th iteration, f(x)= 31.48812502815125\n", 603 | "21 th iteration, f(x)= 2.9201599464235812\n", 604 | "22 th iteration, f(x)= 0.8353168374585138\n", 605 | "23 th iteration, f(x)= 0.0851741147946558\n", 606 | "24 th iteration, f(x)= 0.009328007341712848\n", 607 | "25 th iteration, f(x)= 0.007215648593719789\n", 608 | "26 th iteration, f(x)= 0.007158311945326502\n", 609 | "27 th iteration, f(x)= 0.007156778644639016\n", 610 | "28 th iteration, f(x)= 0.007156737327560264\n", 611 | "29 th iteration, f(x)= 0.00715673621421872\n", 612 | "30 th iteration, f(x)= 0.00715673618421831\n", 613 | "31 th iteration, f(x)= 0.0071567361834099065\n" 614 | ] 615 | } 616 | ], 617 | "source": [ 618 | "# 3. Optimize original function\n", 619 | "x = x0.copy()\n", 620 | "x = BFGS(x, f, grad, n=n, eta=1e-5, maxIter=200)" 621 | ] 622 | }, 623 | { 624 | "cell_type": "code", 625 | "execution_count": 11, 626 | "metadata": {}, 627 | "outputs": [ 628 | { 629 | "name": "stdout", 630 | "output_type": "stream", 631 | "text": [ 632 | "Optimization terminated successfully.\n", 633 | " Current function value: 0.007157\n", 634 | " Iterations: 28\n", 635 | " Function evaluations: 693\n", 636 | " Gradient evaluations: 33\n", 637 | " fun: 0.007156736184194109\n", 638 | " hess_inv: array([[ 4.23427080e-02, -1.06309180e-02, -1.48198659e-03,\n", 639 | " 1.24075698e-03, -3.34344891e-04, -1.31286902e-02,\n", 640 | " -2.24944442e-02, -1.36267548e-02, -4.84116502e-03,\n", 641 | " 1.65344109e-02, 9.23167961e-03, -6.58852090e-03,\n", 642 | " -8.21886807e-03, 1.08767300e-02, 1.28608211e-02,\n", 643 | " -8.42232407e-03, -2.61280369e-02, 8.08160159e-03,\n", 644 | " 4.43320852e-03, -4.52721976e-03],\n", 645 | " [-1.06309180e-02, 8.27871281e-02, 3.67155213e-02,\n", 646 | " -7.30128839e-02, -2.69442771e-03, -1.49508423e-03,\n", 647 | " 4.38572900e-03, 2.79822412e-02, 1.74246664e-03,\n", 648 | " -3.89679723e-02, -1.00243816e-02, 7.33531889e-03,\n", 649 | " -3.09195421e-02, 8.62890484e-03, 1.63281611e-02,\n", 650 | " -2.57996607e-02, 6.04191897e-02, -2.14600806e-02,\n", 651 | " -4.17641906e-02, -1.64650718e-02],\n", 652 | " [-1.48198659e-03, 3.67155213e-02, 2.54545096e-02,\n", 653 | " -2.68633779e-02, -3.49510415e-03, -7.78507132e-03,\n", 654 | " 6.24334871e-03, 1.38226548e-02, -3.66874605e-04,\n", 655 | " -2.51576819e-02, -1.08490094e-02, 1.83091992e-03,\n", 656 | " -1.40875180e-02, 7.93827685e-03, 2.31982562e-03,\n", 657 | " -1.22790561e-02, 2.38444359e-02, -1.27578421e-02,\n", 658 | " -1.75629233e-02, -1.29305702e-02],\n", 659 | " [ 1.24075698e-03, -7.30128839e-02, -2.68633779e-02,\n", 660 | " 1.13712413e-01, -1.11434489e-02, -1.98909955e-02,\n", 661 | " 1.61379843e-02, -1.53719205e-02, -3.84382988e-03,\n", 662 | " 1.47375979e-02, -4.97015217e-03, -1.67637589e-02,\n", 663 | " 4.12283666e-02, -1.52692714e-02, -2.92336979e-02,\n", 664 | " 2.49530055e-02, -5.39036269e-02, -2.41218173e-03,\n", 665 | " 3.16257496e-02, -4.49622801e-03],\n", 666 | " [-3.34344891e-04, -2.69442771e-03, -3.49510415e-03,\n", 667 | " -1.11434489e-02, 9.79571308e-03, 1.08410801e-02,\n", 668 | " -3.45886277e-03, -3.69899476e-03, -2.97610439e-03,\n", 669 | " 5.82610737e-03, 2.58640911e-03, 5.07449692e-03,\n", 670 | " -9.83026815e-04, -2.98810669e-03, -1.21452352e-03,\n", 671 | " 6.67839630e-03, -4.03568603e-03, 4.46080975e-03,\n", 672 | " 1.25035424e-03, 5.59744904e-03],\n", 673 | " [-1.31286902e-02, -1.49508423e-03, -7.78507132e-03,\n", 674 | " -1.98909955e-02, 1.08410801e-02, 2.60734225e-02,\n", 675 | " -1.69692205e-03, 4.51219172e-05, -2.16712842e-03,\n", 676 | " 6.71028602e-03, 3.81559973e-03, 8.96818983e-03,\n", 677 | " 3.81482534e-04, -8.73619707e-03, -1.16799203e-03,\n", 678 | " 1.21663661e-02, 3.64223194e-03, 1.08752095e-02,\n", 679 | " 6.44174504e-04, 1.12280744e-02],\n", 680 | " [-2.24944442e-02, 4.38572900e-03, 6.24334871e-03,\n", 681 | " 1.61379843e-02, -3.45886277e-03, -1.69692205e-03,\n", 682 | " 2.28120968e-02, 8.73576814e-03, 2.14738528e-03,\n", 683 | " -1.72613139e-02, -1.32726829e-02, 1.94544969e-05,\n", 684 | " 7.90363827e-03, -6.28985070e-03, -1.57128634e-02,\n", 685 | " 4.98403632e-03, 1.25793175e-02, -1.30355824e-02,\n", 686 | " -1.78287478e-03, -4.71080826e-03],\n", 687 | " [-1.36267548e-02, 2.79822412e-02, 1.38226548e-02,\n", 688 | " -1.53719205e-02, -3.69899476e-03, 4.51219172e-05,\n", 689 | " 8.73576814e-03, 1.93441614e-02, -1.71651104e-03,\n", 690 | " -2.06091844e-02, -7.56600644e-03, 3.41676248e-03,\n", 691 | " -7.34460563e-03, -6.69896475e-04, -1.17528394e-03,\n", 692 | " -2.47194164e-03, 1.88063673e-02, -8.53345460e-03,\n", 693 | " -1.56038090e-02, -8.70867456e-03],\n", 694 | " [-4.84116502e-03, 1.74246664e-03, -3.66874605e-04,\n", 695 | " -3.84382988e-03, -2.97610439e-03, -2.16712842e-03,\n", 696 | " 2.14738528e-03, -1.71651104e-03, 1.15762984e-02,\n", 697 | " -1.17342321e-04, -6.38234703e-04, -5.34493547e-05,\n", 698 | " -1.81071228e-03, 3.28095192e-03, 2.90032555e-03,\n", 699 | " -8.28588883e-03, 1.22309918e-02, -1.63883422e-03,\n", 700 | " 1.62076313e-03, 4.46981952e-03],\n", 701 | " [ 1.65344109e-02, -3.89679723e-02, -2.51576819e-02,\n", 702 | " 1.47375979e-02, 5.82610737e-03, 6.71028602e-03,\n", 703 | " -1.72613139e-02, -2.06091844e-02, -1.17342321e-04,\n", 704 | " 3.56103386e-02, 1.70404992e-02, -2.29570435e-03,\n", 705 | " 8.03404433e-03, -9.24950892e-04, 5.12738630e-03,\n", 706 | " 6.79379323e-03, -3.07512436e-02, 1.98624993e-02,\n", 707 | " 2.11918115e-02, 1.55566893e-02],\n", 708 | " [ 9.23167961e-03, -1.00243816e-02, -1.08490094e-02,\n", 709 | " -4.97015217e-03, 2.58640911e-03, 3.81559973e-03,\n", 710 | " -1.32726829e-02, -7.56600644e-03, -6.38234703e-04,\n", 711 | " 1.70404992e-02, 1.38503875e-02, -3.37206703e-04,\n", 712 | " -8.37798432e-04, 2.42495781e-03, 9.61669586e-03,\n", 713 | " -1.64101400e-03, -1.06171712e-02, 1.06036225e-02,\n", 714 | " 5.16083227e-03, 7.57660107e-03],\n", 715 | " [-6.58852090e-03, 7.33531889e-03, 1.83091992e-03,\n", 716 | " -1.67637589e-02, 5.07449692e-03, 8.96818983e-03,\n", 717 | " 1.94544969e-05, 3.41676248e-03, -5.34493547e-05,\n", 718 | " -2.29570435e-03, -3.37206703e-04, 7.19185866e-03,\n", 719 | " -4.49772086e-03, -1.63971733e-03, -5.87610579e-05,\n", 720 | " 2.98304937e-03, 5.46569428e-03, 2.44583672e-03,\n", 721 | " -2.91810585e-03, 3.37864214e-03],\n", 722 | " [-8.21886807e-03, -3.09195421e-02, -1.40875180e-02,\n", 723 | " 4.12283666e-02, -9.83026815e-04, 3.81482534e-04,\n", 724 | " 7.90363827e-03, -7.34460563e-03, -1.81071228e-03,\n", 725 | " 8.03404433e-03, -8.37798432e-04, -4.49772086e-03,\n", 726 | " 2.18447817e-02, -9.55016271e-03, -1.38147812e-02,\n", 727 | " 1.40890509e-02, -1.76323200e-02, 1.04555671e-03,\n", 728 | " 1.38529555e-02, 2.55908147e-03],\n", 729 | " [ 1.08767300e-02, 8.62890484e-03, 7.93827685e-03,\n", 730 | " -1.52692714e-02, -2.98810669e-03, -8.73619707e-03,\n", 731 | " -6.28985070e-03, -6.69896475e-04, 3.28095192e-03,\n", 732 | " -9.24950892e-04, 2.42495781e-03, -1.63971733e-03,\n", 733 | " -9.55016271e-03, 1.53969867e-02, 8.16972339e-03,\n", 734 | " -1.45226826e-02, 3.51494326e-03, 1.04412678e-03,\n", 735 | " -1.33937490e-03, -1.12045983e-04],\n", 736 | " [ 1.28608211e-02, 1.63281611e-02, 2.31982562e-03,\n", 737 | " -2.92336979e-02, -1.21452352e-03, -1.16799203e-03,\n", 738 | " -1.57128634e-02, -1.17528394e-03, 2.90032555e-03,\n", 739 | " 5.12738630e-03, 9.61669586e-03, -5.87610579e-05,\n", 740 | " -1.38147812e-02, 8.16972339e-03, 2.11688075e-02,\n", 741 | " -1.55056183e-02, 1.01686998e-02, 4.85537851e-03,\n", 742 | " -8.11606079e-03, 1.69825365e-03],\n", 743 | " [-8.42232407e-03, -2.57996607e-02, -1.22790561e-02,\n", 744 | " 2.49530055e-02, 6.67839630e-03, 1.21663661e-02,\n", 745 | " 4.98403632e-03, -2.47194164e-03, -8.28588883e-03,\n", 746 | " 6.79379323e-03, -1.64101400e-03, 2.98304937e-03,\n", 747 | " 1.40890509e-02, -1.45226826e-02, -1.55056183e-02,\n", 748 | " 2.60112910e-02, -2.29130446e-02, 6.59059980e-03,\n", 749 | " 1.00607822e-02, 3.52150421e-03],\n", 750 | " [-2.61280369e-02, 6.04191897e-02, 2.38444359e-02,\n", 751 | " -5.39036269e-02, -4.03568603e-03, 3.64223194e-03,\n", 752 | " 1.25793175e-02, 1.88063673e-02, 1.22309918e-02,\n", 753 | " -3.07512436e-02, -1.06171712e-02, 5.46569428e-03,\n", 754 | " -1.76323200e-02, 3.51494326e-03, 1.01686998e-02,\n", 755 | " -2.29130446e-02, 6.66619308e-02, -1.96817050e-02,\n", 756 | " -2.78117788e-02, -4.39692186e-03],\n", 757 | " [ 8.08160159e-03, -2.14600806e-02, -1.27578421e-02,\n", 758 | " -2.41218173e-03, 4.46080975e-03, 1.08752095e-02,\n", 759 | " -1.30355824e-02, -8.53345460e-03, -1.63883422e-03,\n", 760 | " 1.98624993e-02, 1.06036225e-02, 2.44583672e-03,\n", 761 | " 1.04555671e-03, 1.04412678e-03, 4.85537851e-03,\n", 762 | " 6.59059980e-03, -1.96817050e-02, 2.26970517e-02,\n", 763 | " 1.30733655e-02, 1.07013171e-02],\n", 764 | " [ 4.43320852e-03, -4.17641906e-02, -1.75629233e-02,\n", 765 | " 3.16257496e-02, 1.25035424e-03, 6.44174504e-04,\n", 766 | " -1.78287478e-03, -1.56038090e-02, 1.62076313e-03,\n", 767 | " 2.11918115e-02, 5.16083227e-03, -2.91810585e-03,\n", 768 | " 1.38529555e-02, -1.33937490e-03, -8.11606079e-03,\n", 769 | " 1.00607822e-02, -2.78117788e-02, 1.30733655e-02,\n", 770 | " 2.64808639e-02, 1.06207110e-02],\n", 771 | " [-4.52721976e-03, -1.64650718e-02, -1.29305702e-02,\n", 772 | " -4.49622801e-03, 5.59744904e-03, 1.12280744e-02,\n", 773 | " -4.71080826e-03, -8.70867456e-03, 4.46981952e-03,\n", 774 | " 1.55566893e-02, 7.57660107e-03, 3.37864214e-03,\n", 775 | " 2.55908147e-03, -1.12045983e-04, 1.69825365e-03,\n", 776 | " 3.52150421e-03, -4.39692186e-03, 1.07013171e-02,\n", 777 | " 1.06207110e-02, 1.65831856e-02]])\n", 778 | " jac: array([-4.35276888e-07, -9.11240932e-07, -2.20240327e-06, -1.98109774e-06,\n", 779 | " -2.34023901e-06, -2.17754859e-07, -4.26953193e-07, -1.42026693e-06,\n", 780 | " -1.01199839e-06, -1.19372271e-06, 1.36205927e-08, -2.99885869e-07,\n", 781 | " -8.95815901e-07, -1.06927473e-06, -2.11352017e-06, -1.46136153e-06,\n", 782 | " -1.45408558e-06, -5.64497896e-07, -9.83825885e-07, -1.19529432e-06])\n", 783 | " message: 'Optimization terminated successfully.'\n", 784 | " nfev: 693\n", 785 | " nit: 28\n", 786 | " njev: 33\n", 787 | " status: 0\n", 788 | " success: True\n", 789 | " x: array([ 3.60222164e-01, 2.62752516e-01, -4.47953200e-01, -5.59635718e-01,\n", 790 | " 1.79442091e-04, 1.83403764e-01, -7.64242034e-01, 6.73805650e-02,\n", 791 | " 1.21327600e-01, 4.23963437e-01, 6.23063269e-01, -5.05295103e-02,\n", 792 | " -4.20346515e-01, 3.03286538e-01, 8.43360383e-01, -3.22351004e-01,\n", 793 | " 2.98278572e-01, 4.45273258e-01, -2.73509343e-01, 3.84753045e-01])\n", 794 | ":3: OptimizeWarning: Unknown solver options: xtol\n", 795 | " res = opt.minimize(fun=f, x0=x0, options={'xtol': 1e-9, 'disp': True})\n" 796 | ] 797 | } 798 | ], 799 | "source": [ 800 | "# True Result\n", 801 | "import scipy.optimize as opt\n", 802 | "res = opt.minimize(fun=f, x0=x0, options={'xtol': 1e-9, 'disp': True})\n", 803 | "print(res)" 804 | ] 805 | } 806 | ], 807 | "metadata": { 808 | "interpreter": { 809 | "hash": "a50a0192a5b2fb734ecb67499b76119af5e5f18ac3596dcca270a0de52a88374" 810 | }, 811 | "kernelspec": { 812 | "display_name": "Python 3.8.5 64-bit ('base': conda)", 813 | "name": "python3" 814 | }, 815 | "language_info": { 816 | "name": "python", 817 | "version": "" 818 | }, 819 | "orig_nbformat": 2 820 | }, 821 | "nbformat": 4, 822 | "nbformat_minor": 2 823 | } -------------------------------------------------------------------------------- /13/bfgs.py: -------------------------------------------------------------------------------- 1 | # BFGS Algorithm 2 | # By ZincCat 3 | import numpy as np 4 | 5 | 6 | def linesearch_Armijo(f, x, grad, d, alpha=0.4, beta=0.8): 7 | # backtrack linesearch using Armijo rules 8 | t = 1.0 9 | value = f(x) 10 | g = grad(x) 11 | while f(x + t*d) > value + alpha*t*np.dot(g, d): 12 | t *= beta 13 | return t 14 | 15 | 16 | def linesearch_Wolfe(f, x, grad, d, start=0, end=1e10, rho=0.3, sigma=0.4): 17 | # linesearch using strong Wolfe rules 18 | value = f(x) 19 | g = grad(x) 20 | reg1 = np.dot(g, d) 21 | reg2 = sigma*g 22 | t = 0 23 | while t < 50: 24 | alpha = (start + end)/2 25 | x_new = x + alpha*d 26 | cond1 = (f(x_new) < value + rho*alpha*reg1) 27 | cond2 = (np.abs(np.dot(grad(x_new), d)) 28 | < np.abs(np.dot(reg2, d))) 29 | if (cond1 and cond2): 30 | break 31 | if not cond1: 32 | end = alpha 33 | else: 34 | start = alpha 35 | t += 1 36 | return alpha 37 | 38 | 39 | def updateH(H, dx, dg, eps=1e-30): 40 | finished = False 41 | t1 = H@dg 42 | t2 = np.dot(dg, dx) 43 | if np.abs(t2) < eps: 44 | finished = True 45 | return H + (1+np.dot(dg, t1)/t2)*np.outer(dx, dx)/t2 - (np.outer(t1, dx)+np.outer(dx, t1))/t2, finished 46 | 47 | 48 | def BFGS(x0, f, grad, n, maxIter=100, eta=1e-10, eps=1e-20, display=True, a=0.4, b=0.8): 49 | timestep = 0 50 | x = x0.copy() 51 | H = np.eye(n) # H_0 52 | g = grad(x) 53 | d = -g 54 | while True: 55 | if display: 56 | print(timestep, "th iteration, f(x)=", f(x)) 57 | if np.linalg.norm(g) < eta: 58 | break 59 | alpha = linesearch_Armijo(f, x, g, d, a, b) 60 | # alpha = linesearch_Wolfe(f, x, grad, d) 61 | dx = alpha*d 62 | x += dx 63 | dg = grad(x) - g 64 | g += dg 65 | H, finished = updateH(H, dx, dg, eps) 66 | if finished: 67 | break 68 | d = -H@g 69 | timestep += 1 70 | if timestep >= maxIter: 71 | break 72 | return x 73 | -------------------------------------------------------------------------------- /13/lbfgs.py: -------------------------------------------------------------------------------- 1 | # Low Memory BFGS Algorithm 2 | # By ZincCat 3 | import numpy as np 4 | 5 | 6 | def linesearch(f, x, g, d, a=0.4, b=0.8): 7 | # backtrack linesearch 8 | t = 0.8 9 | value = f(x) 10 | while f(x + t*d) > value + a*t*np.dot(g, d): 11 | t *= b 12 | return t 13 | 14 | 15 | def getd(grad, m): 16 | q = grad.copy() 17 | alphalist = [] 18 | for i, xg in enumerate(m): 19 | alphalist.append(np.dot(xg[0], q)/np.dot(xg[0], xg[1])) 20 | q -= alphalist[i]*xg[1] 21 | l = len(m) 22 | if l > 0: # H_0 23 | q *= np.dot(m[0][0], m[0][1])/np.dot(m[0][1], m[0][1]) # p 24 | for i, xg in enumerate(reversed(m)): 25 | beta = np.dot(xg[1], q)/np.dot(xg[0], xg[1]) 26 | q += (alphalist[l-i-1] - beta)*xg[0] 27 | return -q 28 | 29 | 30 | def LBFGS(x0, f, grad, memlimit=3, maxIter=100, eta=1e-10, display=True, a=0.4, b=0.8): 31 | timestep = 0 32 | x = x0 33 | g = grad(x) 34 | d = -g 35 | m = [] 36 | while True: 37 | if display: 38 | print(timestep, "th iteration, f(x)=", f(x)) 39 | if np.linalg.norm(g) < eta: 40 | break 41 | d = getd(g, m) 42 | alpha = linesearch(f, x, g, d, a, b) 43 | dx = alpha*d 44 | x += dx 45 | dg = grad(x) - g 46 | g += dg 47 | if len(m) >= memlimit: 48 | m.pop() 49 | m.insert(0, (dx, dg)) 50 | timestep += 1 51 | if timestep >= maxIter: 52 | break 53 | return x 54 | -------------------------------------------------------------------------------- /13/rs1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 30, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "from matplotlib import pyplot as plt\n", 11 | "\n", 12 | "# 设置随机种子\n", 13 | "np.random.seed(19890817)" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "R1 update" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 29, 26 | "metadata": { 27 | "tags": [] 28 | }, 29 | "outputs": [ 30 | { 31 | "name": "stdout", 32 | "output_type": "stream", 33 | "text": [ 34 | "H [[1. 0.]\n", 35 | " [0. 1.]]\n", 36 | "H [[0.5 0. ]\n", 37 | " [0. 1. ]]\n", 38 | "result: [0. 0.]\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "# 定义一些辅助函数\n", 44 | "Q = np.array([[2, 0], [0, 1]])\n", 45 | "\n", 46 | "def f(x):\n", 47 | " # 计算函数值\n", 48 | " return x[0]*x[0] + 1/2*x[1]*x[1] + 3\n", 49 | "\n", 50 | "def gradient_f(x):\n", 51 | " # 计算函数梯度\n", 52 | " return Q@x\n", 53 | "\n", 54 | "# 设置变量初值\n", 55 | "x = np.array([1, 2], dtype=np.float64)\n", 56 | "H = np.eye(2)\n", 57 | "g = gradient_f(x)\n", 58 | "d = -g\n", 59 | "eta = 1e-5\n", 60 | "while True:\n", 61 | " print(\"H\", H)\n", 62 | " if np.linalg.norm(g) < eta:\n", 63 | " break\n", 64 | " alpha = - np.dot(g, d)/np.dot(d, Q@d)\n", 65 | " dx = alpha*d\n", 66 | " x += dx\n", 67 | " dg = Q@dx\n", 68 | " g += dg\n", 69 | " t = dx - H@dg\n", 70 | " if np.abs(np.dot(dg, t)) < eta:\n", 71 | " break\n", 72 | " H = H + np.outer(t, t)/np.dot(dg, t)\n", 73 | " d = -H@g\n", 74 | "print(\"result:\", x)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "DFP" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 38, 87 | "metadata": {}, 88 | "outputs": [ 89 | { 90 | "name": "stdout", 91 | "output_type": "stream", 92 | "text": [ 93 | "H [[1. 0.]\n", 94 | " [0. 1.]]\n", 95 | "x [0. 0.]\n", 96 | "[ 1. -1.]\n", 97 | "H [[ 0.5 -0.5]\n", 98 | " [-0.5 1.5]]\n", 99 | "x [ 1. -1.]\n", 100 | "[ 0. -0.5]\n", 101 | "H [[ 0.5 -0.5]\n", 102 | " [-0.5 1. ]]\n", 103 | "x [ 1. -1.5]\n", 104 | "result: [ 1. -1.5]\n" 105 | ] 106 | } 107 | ], 108 | "source": [ 109 | "# 定义一些辅助函数\n", 110 | "Q = np.array([[4, 2], [2, 2]])\n", 111 | "b = np.array([-1, 1])\n", 112 | "\n", 113 | "def f(x):\n", 114 | " # 计算函数值\n", 115 | " return np.dot(x, Q@x) + np.dot(x, b)\n", 116 | "\n", 117 | "def gradient_f(x):\n", 118 | " # 计算函数梯度\n", 119 | " return Q@x + b\n", 120 | "\n", 121 | "# 设置变量初值\n", 122 | "x = np.array([0, 0], dtype=np.float64)\n", 123 | "H = np.eye(2)\n", 124 | "g = gradient_f(x)\n", 125 | "d = -g\n", 126 | "eta = 1e-5\n", 127 | "while True:\n", 128 | " print(\"H\", H)\n", 129 | " print(\"x\", x)\n", 130 | " if np.linalg.norm(g) < eta:\n", 131 | " break\n", 132 | " alpha = -np.dot(g, d)/np.dot(d, Q@d)\n", 133 | " dx = alpha*d\n", 134 | " print(dx)\n", 135 | " x += dx\n", 136 | " dg = Q@dx\n", 137 | " g += dg\n", 138 | " t = H@dg\n", 139 | " if np.abs(np.dot(dg, t)) < eta:\n", 140 | " break\n", 141 | " H = H + np.outer(dx, dx)/np.dot(dx, dg) - np.outer(t, t)/np.dot(dg, t)\n", 142 | " d = -H@g\n", 143 | "print(\"result:\", x)" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "BFGS" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 39, 156 | "metadata": {}, 157 | "outputs": [ 158 | { 159 | "name": "stdout", 160 | "output_type": "stream", 161 | "text": [ 162 | "H [[1. 0.]\n", 163 | " [0. 1.]]\n", 164 | "x [0. 0.]\n", 165 | "[-0. 0.5]\n", 166 | "H [[1. 1.5 ]\n", 167 | " [1.5 2.75]]\n", 168 | "x [0. 0.5]\n", 169 | "[3. 4.5]\n", 170 | "H [[2. 3.]\n", 171 | " [3. 5.]]\n", 172 | "x [3. 5.]\n", 173 | "result: [3. 5.]\n" 174 | ] 175 | } 176 | ], 177 | "source": [ 178 | "# 定义一些辅助函数\n", 179 | "Q = np.array([[5, -3], [-3, 2]])\n", 180 | "b = np.array([0, 1])\n", 181 | "\n", 182 | "def f(x):\n", 183 | " # 计算函数值\n", 184 | " return np.dot(x, Q@x) - np.dot(x, b) + np.log(np.pi)\n", 185 | "\n", 186 | "def gradient_f(x):\n", 187 | " # 计算函数梯度\n", 188 | " return Q@x - b\n", 189 | "\n", 190 | "# 设置变量初值\n", 191 | "x = np.array([0, 0], dtype=np.float64)\n", 192 | "H = np.eye(2)\n", 193 | "g = gradient_f(x)\n", 194 | "d = -g\n", 195 | "eta = 1e-5\n", 196 | "while True:\n", 197 | " print(\"H\", H)\n", 198 | " print(\"x\", x)\n", 199 | " if np.linalg.norm(g) < eta:\n", 200 | " break\n", 201 | " alpha = -np.dot(g, d)/np.dot(d, Q@d)\n", 202 | " dx = alpha*d\n", 203 | " print(dx)\n", 204 | " x += dx\n", 205 | " dg = Q@dx\n", 206 | " g += dg\n", 207 | " t = H@dg\n", 208 | " if np.abs(np.dot(dg, t)) < eta:\n", 209 | " break\n", 210 | " H = H + (1+np.dot(dg, t)/np.dot(dg, dx))*np.outer(dx, dx)/np.dot(dg, dx) - (np.outer(t, dx) + np.outer(dx, t))/np.dot(dg, dx)\n", 211 | " d = -H@g\n", 212 | "print(\"result:\", x)" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": null, 218 | "metadata": {}, 219 | "outputs": [], 220 | "source": [] 221 | } 222 | ], 223 | "metadata": { 224 | "interpreter": { 225 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 226 | }, 227 | "kernelspec": { 228 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 229 | "name": "python3" 230 | }, 231 | "language_info": { 232 | "name": "python", 233 | "version": "" 234 | }, 235 | "orig_nbformat": 2 236 | }, 237 | "nbformat": 4, 238 | "nbformat_minor": 2 239 | } -------------------------------------------------------------------------------- /18/1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Penalty Function\n", 8 | "By ZincCat\n", 9 | "\n", 10 | "$\\min _{\\mathbf{x}} \\frac{1}{2}\\|\\mathbf{x}\\|_{2}$\n", 11 | "\n", 12 | "s.t. $\\mathbf{A x}=\\mathbf{b}$" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "import numpy as np\n", 22 | "from matplotlib import pyplot as plt" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "p = 200\n", 32 | "n = 300" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "np.random.seed(19890817)\n", 42 | "A = np.random.normal(10, 5, (p, n))\n", 43 | "b = np.random.normal(10, 5, p)\n", 44 | "\n", 45 | "def linesearch_Armijo(f, x, g, d, alpha=0.4, beta=0.8):\n", 46 | " # backtrack linesearch using Armijo rules\n", 47 | " t = 1.0\n", 48 | " value = f(x)\n", 49 | " while f(x + t*d) > value + alpha*t*np.dot(g, d):\n", 50 | " t *= beta\n", 51 | " return t" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "def f(x):\n", 61 | " return np.linalg.norm(x, 2)/2\n", 62 | "def f_ab(gamma):\n", 63 | " return lambda x: np.linalg.norm(x, 2)**2/2 + gamma * np.linalg.norm(A@x-b)**2\n", 64 | "def g_ab(gamma):\n", 65 | " return lambda x: x + 2 * gamma * A.T@(A@x-b)\n", 66 | "def f_bc(gamma):\n", 67 | " return lambda x: np.linalg.norm(x, 2)**2/2 + gamma * np.sum((A@x-b)**4)\n", 68 | "def g_bc(gamma):\n", 69 | " return lambda x: x + 4 * gamma * A.T@((A@x-b)**3)\n", 70 | "def f_ab2(gamma):\n", 71 | " return lambda x: np.linalg.norm(x, 2)**2/2 + gamma * np.linalg.norm(A@x-b, 1)\n", 72 | "def g_ab2(gamma):\n", 73 | " return lambda x: x + gamma * A.T@np.sign(A@x-b)\n", 74 | "cons = ({'type': 'eq', 'fun': lambda x: A.dot(x)-b})" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "x0 = np.random.rand(n)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "minValue = f(A.T@np.linalg.inv(A@A.T)@b)\n", 93 | "print(minValue)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "Direct projected gradient, with inexact line search." 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "def descent(f, x, grad):\n", 110 | " # 梯度下降函数\n", 111 | " # 输入函数f, 目前x取值, 梯度函数\n", 112 | " # 输出下降后x取值, 步长t\n", 113 | " xn = x.copy()\n", 114 | " g = grad(xn)\n", 115 | " grad_norm = np.linalg.norm(g, 2)\n", 116 | " d = -g/grad_norm\n", 117 | " t = linesearch_Armijo(f, xn, g, d)\n", 118 | " xn += t*d\n", 119 | " return xn, t" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": {}, 125 | "source": [ 126 | "absolute value penalty function" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": { 133 | "tags": [ 134 | "outputPrepend" 135 | ] 136 | }, 137 | "outputs": [], 138 | "source": [ 139 | "# 绘图\n", 140 | "time1 = [] # 记录时间步, 用于绘图\n", 141 | "values1 = [] # 记录某一时间步下函数值, 用于绘图\n", 142 | "pvalues1 = [] # 记录某一时间步下含惩罚函数值, 用于绘图\n", 143 | "Plot = True # 是否绘图, 请保证此时alpha, beta均为单一取值\n", 144 | "timestep = 0\n", 145 | "\n", 146 | "x = x0.copy() #满足约束的初值\n", 147 | "gamma = 0\n", 148 | "\n", 149 | "# 用于判定终止\n", 150 | "count = 0 \n", 151 | "eps = 1e-10\n", 152 | "oldvalue = f(x)\n", 153 | "maxIter = 20000 # 最大迭代次数\n", 154 | "\n", 155 | "while True:\n", 156 | " value = f(x)\n", 157 | " print(\"Iteration:\", timestep, \"Value\", value)\n", 158 | " # 用函数值改变量作为终止条件\n", 159 | " if abs(value - oldvalue) < eps:\n", 160 | " count += 1\n", 161 | " else:\n", 162 | " count = 0\n", 163 | " oldvalue = value\n", 164 | " if timestep > maxIter or count >= 5:\n", 165 | " break\n", 166 | " for i in range(20):\n", 167 | " x, t = descent(f_ab(gamma), x, g_ab(gamma)) # 此时使用无穷范数\n", 168 | " print(f_ab(gamma)(x))\n", 169 | " if Plot:\n", 170 | " time1.append(timestep)\n", 171 | " values1.append(value)\n", 172 | " pvalues1.append(f_ab(gamma)(x))\n", 173 | " gamma += 100\n", 174 | " timestep += 1" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "Courant-Beltrami penalty function" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": { 188 | "tags": [ 189 | "outputPrepend" 190 | ] 191 | }, 192 | "outputs": [], 193 | "source": [ 194 | "# 绘图\n", 195 | "time2 = [] # 记录时间步, 用于绘图\n", 196 | "values2 = [] # 记录某一时间步下函数值, 用于绘图\n", 197 | "pvalues2 = [] # 记录某一时间步下含惩罚函数值, 用于绘图\n", 198 | "Plot = True # 是否绘图, 请保证此时alpha, beta均为单一取值\n", 199 | "timestep = 0\n", 200 | "\n", 201 | "x = x0.copy() #满足约束的初值\n", 202 | "gamma = 0\n", 203 | "\n", 204 | "# 用于判定终止\n", 205 | "count = 0 \n", 206 | "eps = 1e-10\n", 207 | "oldvalue = f(x)\n", 208 | "maxIter = 20000 # 最大迭代次数\n", 209 | "\n", 210 | "while True:\n", 211 | " value = f(x)\n", 212 | " print(\"Iteration:\", timestep, \"Value\", value)\n", 213 | " # 用函数值改变量作为终止条件\n", 214 | " if abs(value - oldvalue) < eps:\n", 215 | " count += 1\n", 216 | " else:\n", 217 | " count = 0\n", 218 | " oldvalue = value\n", 219 | " if timestep > maxIter or count >= 5:\n", 220 | " break\n", 221 | " for i in range(10):\n", 222 | " x, t = descent(f_bc(gamma), x, g_bc(gamma)) # 此时使用无穷范数\n", 223 | " \n", 224 | " if Plot:\n", 225 | " time2.append(timestep)\n", 226 | " values2.append(value)\n", 227 | " pvalues2.append(f_bc(gamma)(x))\n", 228 | " gamma += 100\n", 229 | " timestep += 1" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "metadata": {}, 236 | "outputs": [], 237 | "source": [ 238 | "plt.plot(time1, np.log(values1)-minValue)\n", 239 | "plt.plot(time2, np.log(values2)-minValue)\n", 240 | "plt.legend(['Absolute', 'Courant-Beltrami'])\n", 241 | "plt.xlabel(\"Iteration number $(k/100)$\")\n", 242 | "plt.ylabel(\"$\\log (f(\\mathbf{x}_{k})-f^{*})$\")\n", 243 | "plt.savefig('result1gamma100.png', dpi=200)\n", 244 | "plt.show()" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": {}, 250 | "source": [ 251 | "并不exact的一点佐证" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": null, 257 | "metadata": { 258 | "tags": [ 259 | "outputPrepend" 260 | ] 261 | }, 262 | "outputs": [], 263 | "source": [ 264 | "# 绘图\n", 265 | "time3 = [] # 记录时间步, 用于绘图\n", 266 | "values3 = [] # 记录某一时间步下函数值, 用于绘图\n", 267 | "Plot = True # 是否绘图, 请保证此时alpha, beta均为单一取值\n", 268 | "timestep = 0\n", 269 | "\n", 270 | "x = x0.copy() #满足约束的初值\n", 271 | "gamma = 30000000\n", 272 | "\n", 273 | "# 用于判定终止\n", 274 | "count = 0 \n", 275 | "eps = 1e-13\n", 276 | "oldvalue = f(x)\n", 277 | "maxIter = 20000 # 最大迭代次数\n", 278 | "\n", 279 | "while True:\n", 280 | " value = f(x)\n", 281 | " print(\"Iteration:\", timestep, \"Value\", value)\n", 282 | " # 用函数值改变量作为终止条件\n", 283 | " if abs(value - oldvalue) < eps:\n", 284 | " count += 1\n", 285 | " else:\n", 286 | " count = 0\n", 287 | " oldvalue = value\n", 288 | " if Plot:\n", 289 | " time3.append(timestep)\n", 290 | " values3.append(value)\n", 291 | " if timestep > maxIter or count >= 5:\n", 292 | " break\n", 293 | " for i in range(10):\n", 294 | " x, t = descent(f_ab(gamma), x, g_ab(gamma)) # 此时使用无穷范数\n", 295 | " print(f_ab(gamma)(x))\n", 296 | " # gamma += 1\n", 297 | " timestep += 1" 298 | ] 299 | } 300 | ], 301 | "metadata": { 302 | "interpreter": { 303 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 304 | }, 305 | "kernelspec": { 306 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 307 | "name": "python3" 308 | }, 309 | "language_info": { 310 | "name": "python", 311 | "version": "" 312 | }, 313 | "orig_nbformat": 2 314 | }, 315 | "nbformat": 4, 316 | "nbformat_minor": 2 317 | } -------------------------------------------------------------------------------- /18/2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Frank-Wolfe Algorithm\n", 8 | "\n", 9 | "By ZincCat\n", 10 | "\n", 11 | "$\\min _{\\mathbf{x} \\in \\mathbb{R}^{N}}\\|\\mathbf{y}-\\mathbf{D} \\mathbf{x}\\|_{2}^{2}$\n", 12 | "\n", 13 | "s.t. $\\|\\mathbf{x}\\|_{?} \\leq 1$" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import numpy as np\n", 23 | "from matplotlib import pyplot as plt" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "p = 200\n", 33 | "n = 300\n", 34 | "np.random.seed(19890817)\n", 35 | "D = np.random.normal(10, 5, (p, n))\n", 36 | "y = np.random.normal(10, 5, p)\n", 37 | "x0 = np.random.rand(n)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "def f(x):\n", 47 | " return np.linalg.norm(y-D@x)**2\n", 48 | "def grad(x):\n", 49 | " return 2*D.T@(D@x-y)" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "# 调用scipy求值\n", 59 | "from scipy.optimize import minimize\n", 60 | "cons1 = ({'type': 'ineq', 'fun': lambda x: 1 - np.linalg.norm(x, 1)})\n", 61 | "res = minimize(f, x0, constraints=cons1, tol=1e-4, options={'maxiter': 1e4, 'disp': True})\n", 62 | "minValue = res.fun\n", 63 | "print(\"Scipy result:\", res.fun)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "# 调用scipy求值\n", 73 | "from scipy.optimize import minimize\n", 74 | "cons2 = ({'type': 'ineq', 'fun': lambda x: 1 - np.linalg.norm(x, np.inf)})\n", 75 | "res = minimize(f, x0, constraints=cons2, tol=1e-10, options={'maxiter': 1e4, 'disp': True})\n", 76 | "minValue2 = res.fun\n", 77 | "print(\"Scipy result:\", res.fun)" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "def fw(f, x, grad, maxIter, mode, log):\n", 87 | " xn = x.copy()\n", 88 | " for i in range(maxIter):\n", 89 | " value = f(xn)\n", 90 | " print(i, \"th iteration, f(x)=\", value)\n", 91 | " log.append(value)\n", 92 | " gamma = 2/(i+2)\n", 93 | " g = grad(xn)\n", 94 | " if mode == 1:\n", 95 | " d = np.argmax(np.abs(g))\n", 96 | " xn = (1-gamma)*xn\n", 97 | " xn[d] -= gamma * np.sign(g[d])\n", 98 | " elif mode == 'inf':\n", 99 | " d = -np.sign(g)\n", 100 | " xn += gamma * (d-xn)\n", 101 | " return xn" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "$l_\\infty$ constraint" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": { 115 | "tags": [ 116 | "outputPrepend" 117 | ] 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "maxIter = 3000000\n", 122 | "linf = []\n", 123 | "x = fw(f, x0/2/np.linalg.norm(x0, np.inf), grad, maxIter, 'inf', linf)" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "plt.plot(np.log(linf))\n", 133 | "plt.xlabel('Iterations')\n", 134 | "plt.ylabel('$\\ln (f(x_k)-f^*)$')\n", 135 | "plt.savefig('wolfeInf')\n", 136 | "plt.show()" 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "metadata": {}, 142 | "source": [ 143 | "$l_1$ constraint" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "metadata": { 150 | "tags": [ 151 | "outputPrepend" 152 | ] 153 | }, 154 | "outputs": [], 155 | "source": [ 156 | "maxIter = 300000\n", 157 | "l1 = []\n", 158 | "x2 = fw(f, x0/2/np.linalg.norm(x0, 1), grad, maxIter, 1, l1)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [ 167 | "plt.plot(np.log(l1-minValue))\n", 168 | "plt.xlabel('Iterations')\n", 169 | "plt.ylabel('$\\ln (f(x_k)-f^*)$')\n", 170 | "plt.savefig('wolfe1')\n", 171 | "plt.show()" 172 | ] 173 | }, 174 | { 175 | "cell_type": "markdown", 176 | "metadata": {}, 177 | "source": [ 178 | "projected gradient descent" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "def P1(x):\n", 188 | " norm = np.linalg.norm(x, 1)\n", 189 | " if norm > 1:\n", 190 | " return x/norm\n", 191 | " return x\n", 192 | "def Pinf(x):\n", 193 | " t = np.minimum(x, np.ones(n))\n", 194 | " return np.maximum(t, -np.ones(n))" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": null, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "def linesearch_Armijo(f, x, g, d, alpha=0.4, beta=0.8):\n", 204 | " # backtrack linesearch using Armijo rules\n", 205 | " t = 10.0\n", 206 | " value = f(x)\n", 207 | " while f(x + t*d) > value + alpha*t*np.dot(g, d):\n", 208 | " t *= beta\n", 209 | " return t\n", 210 | "def projectedDescent(f, x, grad, proj):\n", 211 | " # 投影梯度下降函数\n", 212 | " # 输入函数f, 目前x取值, 梯度函数, 要投影到的矩阵A\n", 213 | " # 输出下降后x取值, 步长t\n", 214 | " xn = x.copy()\n", 215 | " g = grad(xn)\n", 216 | " grad_norm = np.linalg.norm(g, 2)\n", 217 | " d = -g/grad_norm\n", 218 | " t = linesearch_Armijo(f, xn, g, d)\n", 219 | " xn += t*d\n", 220 | " return proj(xn), t" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": { 227 | "tags": [ 228 | "outputPrepend" 229 | ] 230 | }, 231 | "outputs": [], 232 | "source": [ 233 | "# 绘图\n", 234 | "time1 = [] # 记录时间步, 用于绘图\n", 235 | "values1 = [] # 记录某一时间步下函数值, 用于绘图\n", 236 | "Plot = True # 是否绘图, 请保证此时alpha, beta均为单一取值\n", 237 | "timestep = 0\n", 238 | "\n", 239 | "x = x0.copy() #满足约束的初值\n", 240 | "\n", 241 | "# 用于判定终止\n", 242 | "count = 0 \n", 243 | "eps = 1e-13\n", 244 | "oldvalue = f(x)\n", 245 | "maxIter = 200000 # 最大迭代次数\n", 246 | "\n", 247 | "while True:\n", 248 | " value = f(x)\n", 249 | " print(\"Iteration:\", timestep, \"Value\", value)\n", 250 | " # 用函数值改变量作为终止条件\n", 251 | " if abs(value - oldvalue) < eps:\n", 252 | " count += 1\n", 253 | " else:\n", 254 | " count = 0\n", 255 | " oldvalue = value\n", 256 | " if Plot:\n", 257 | " time1.append(timestep)\n", 258 | " values1.append(value)\n", 259 | " if timestep > maxIter or count >= 5:\n", 260 | " break\n", 261 | " x, t = projectedDescent(f, x, grad, Pinf) # 此时使用无穷范数\n", 262 | " timestep += 1" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "metadata": {}, 269 | "outputs": [], 270 | "source": [] 271 | } 272 | ], 273 | "metadata": { 274 | "interpreter": { 275 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 276 | }, 277 | "kernelspec": { 278 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 279 | "name": "python3" 280 | }, 281 | "language_info": { 282 | "name": "python", 283 | "version": "" 284 | }, 285 | "orig_nbformat": 2 286 | }, 287 | "nbformat": 4, 288 | "nbformat_minor": 2 289 | } -------------------------------------------------------------------------------- /19/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.suggest.snippetsPreventQuickSuggestions": false, 3 | "python.pythonPath": "/Users/zinccat/opt/anaconda3/envs/ml/bin/python", 4 | "jupyter.jupyterServerType": "local" 5 | } -------------------------------------------------------------------------------- /19/1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "By ZincCat" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np\n", 17 | "from matplotlib import pyplot as plt" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "p = 20\n", 27 | "m = 25\n", 28 | "n = 20\n", 29 | "# np.random.seed(100)\n", 30 | "# A = np.random.uniform(0, 1, (n, m, p))\n", 31 | "A = np.ones((n, m, p))\n", 32 | "for i in range(n):\n", 33 | " A[i] *= i+1\n", 34 | "b = np.random.normal(0, 1, m)\n", 35 | "# b = np.ones(m)\n", 36 | "x0 = np.random.normal(0, 1, (n, p))\n", 37 | "# x0 = np.ones((n, p))" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "def f(x):\n", 47 | " return np.sum(np.linalg.norm(x, 1, axis=1))\n", 48 | "def cons(x, A, b):\n", 49 | " return np.einsum('ijk, ik -> j', A, x) - b\n", 50 | "def shrink(x, eps):\n", 51 | " x[abs(x) <= eps] = 0\n", 52 | " x[x > eps] -= eps\n", 53 | " x[x < -eps] += eps\n", 54 | " return x\n", 55 | "def update_x(x, A, b, beta, eta, lam):\n", 56 | " for i in range(n):\n", 57 | " x[i] = shrink(x[i] - A[i].T@(lam + beta*cons(x, A, b))/eta[i]/beta, 1/beta/eta[i])\n", 58 | " return x" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": { 65 | "tags": [] 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "eps1 = 1e-5\n", 70 | "eps2 = 1e-6\n", 71 | "maxBeta = 1000\n", 72 | "beta = 1\n", 73 | "eta = [(np.linalg.norm(i, 'fro')**2) for i in A]\n", 74 | "rho0 = 1.5\n", 75 | "maxIter = 1000000\n", 76 | "\n", 77 | "x = x0.copy()\n", 78 | "lam = np.zeros(m)\n", 79 | "\n", 80 | "l = []\n", 81 | "v = []\n", 82 | "value = f(x)\n", 83 | "for i in range(maxIter):\n", 84 | " if i % 1000 == 0:\n", 85 | " print(i, \"th iteration, f(x) =\", value)\n", 86 | " l.append(beta)\n", 87 | " v.append(value)\n", 88 | " xold = x.copy()\n", 89 | " x = update_x(x, A, b, beta, eta, lam)\n", 90 | " lam += beta * cons(x, A, b)\n", 91 | " if beta*np.max(np.sqrt(eta)*np.linalg.norm(x-xold, 2, axis=1))/np.linalg.norm(b) < eps2:\n", 92 | " rho = rho0\n", 93 | " else:\n", 94 | " rho = 1\n", 95 | " value = f(x)\n", 96 | " if abs(value-f(xold)) < eps1:\n", 97 | " count += 1\n", 98 | " if count == 5:\n", 99 | " print(i, \"th iteration, f(x) =\", value)\n", 100 | " l.append(beta)\n", 101 | " v.append(value)\n", 102 | " break\n", 103 | " else:\n", 104 | " count = 0\n", 105 | " beta = min(beta*rho, maxBeta)\n", 106 | "plt.plot(l)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "plt.plot(v)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "# 调用scipy求值\n", 125 | "from scipy.optimize import minimize\n", 126 | "def constraint(A, b, m, n):\n", 127 | " def f(x):\n", 128 | " ans = np.zeros(m)\n", 129 | " for i in range(n):\n", 130 | " ans += A[i]@x[i*p:(i+1)*p]\n", 131 | " return ans - b\n", 132 | " return f\n", 133 | "con = {'type': 'eq', 'fun': constraint(A, b, m, n)}\n", 134 | "def f1(x):\n", 135 | " return np.linalg.norm(x, 1)\n", 136 | "res = minimize(f1, x0, constraints=con, tol=1e-10, options={'maxiter': 1e3, 'disp': True})\n", 137 | "minValue = res.fun\n", 138 | "print(\"Scipy result:\", res.fun)" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "eta" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "A[0].shape" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [] 165 | } 166 | ], 167 | "metadata": { 168 | "interpreter": { 169 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 170 | }, 171 | "kernelspec": { 172 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 173 | "name": "python3" 174 | }, 175 | "language_info": { 176 | "name": "python", 177 | "version": "" 178 | }, 179 | "orig_nbformat": 2 180 | }, 181 | "nbformat": 4, 182 | "nbformat_minor": 2 183 | } -------------------------------------------------------------------------------- /19/2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# LADMAP\n", 8 | "\n", 9 | "### By ZincCat\n", 10 | "$$\\min _{\\mathbf{Z}, \\mathbf{E}}\\|\\mathbf{Z}\\|_{*}+\\lambda\\|\\mathbf{E}\\|_{2,1}$$\n", 11 | "\n", 12 | "$$\\text{s.t. }\\mathbf{D}=\\mathbf{D Z}+\\mathbf{E}, \\mathbf{Z}^{T} \\mathbf{1}=\\mathbf{1}$$" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 28, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "import numpy as np\n", 22 | "from matplotlib import pyplot as plt\n", 23 | "from time import time\n", 24 | "from sklearn.decomposition import TruncatedSVD\n", 25 | "from sklearn.utils.extmath import randomized_svd\n", 26 | "%matplotlib inline" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 29, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "p = 200\n", 36 | "n = 300\n", 37 | "np.random.seed(19890817)\n", 38 | "D = np.random.normal(0, 1, (p, n))\n", 39 | "D = np.vstack((D, np.ones(n)))" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 30, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "def norm21(X):\n", 49 | " return np.linalg.norm(np.linalg.norm(X, 2, axis=0), 1)\n", 50 | "\n", 51 | "def f(Z, E, lam):\n", 52 | " return np.linalg.norm(Z, 'nuc') + lam * norm21(E)\n", 53 | "\n", 54 | "# l2,1-norm shrinkage operator\n", 55 | "def H(M, eps):\n", 56 | " norm = np.linalg.norm(M, 2, axis=0)\n", 57 | " M[:, norm>eps] *= (norm[norm>eps] - eps)/norm[norm>eps]\n", 58 | " M[:, norm<=eps] = 0\n", 59 | " return M\n", 60 | "\n", 61 | "\n", 62 | "# soft shrinkage operator\n", 63 | "def S(x, eps):\n", 64 | " count = np.sum([x >= eps])\n", 65 | " x[np.abs(x) < eps] = 0\n", 66 | " x[x >= eps] -= eps\n", 67 | " x[x <= -eps] += eps\n", 68 | " return x, count" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 36, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "def update_E(Z, L, lam, beta):\n", 78 | " M = D - D@Z + L/beta\n", 79 | " Enew = H(M, lam/beta)\n", 80 | " Enew[-1] = 0\n", 81 | " return Enew\n", 82 | "\n", 83 | "def update_Z(Z, E, L, beta, eta, r):\n", 84 | " N = D + L/beta - E\n", 85 | " W = Z - (D.T@(D@Z-N))/eta\n", 86 | " U, sigma, V = np.linalg.svd(W)\n", 87 | " # U, sigma, V = randomized_svd(W, n_components=r)\n", 88 | " sigma, count = S(sigma, 1/beta/eta)\n", 89 | " # if count < r:\n", 90 | " # r = min(n, count+1)\n", 91 | " # else:\n", 92 | " # r = min(n, r + int(0.05*n))\n", 93 | " Znew = U@np.diag(sigma)@V\n", 94 | " return Znew, r\n", 95 | "\n", 96 | "def update_L(L, dL, beta):\n", 97 | " return L + beta * dL\n", 98 | "\n", 99 | "def update_beta(beta, maxBeta, rho0, eta, eps2, dE, dZ, normD):\n", 100 | " satisfied = False\n", 101 | " if beta*max(np.sqrt(2.1)*dE, np.sqrt(eta)*dZ)/normD < eps2:\n", 102 | " rho = rho0\n", 103 | " satisfied = True\n", 104 | " else:\n", 105 | " rho = 1\n", 106 | " return min(beta*rho, maxBeta), satisfied" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 37, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "beta = 1e-4\n", 116 | "maxBeta = 1e4\n", 117 | "maxIter = 100000\n", 118 | "eps1 = 1e-3\n", 119 | "eps2 = 1e-3\n", 120 | "rho0 = 1.9\n", 121 | "lam = 0.1\n", 122 | "r = 5" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 38, 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [ 131 | "def LADMAP(f, Din, lam, beta, maxBeta, maxIter, rho0, eps1, eps2, r, log, logbeta):\n", 132 | " step = 0\n", 133 | " D = Din.copy()\n", 134 | " p, n= D.shape\n", 135 | " E = np.zeros((p, n))\n", 136 | " Z = np.zeros((n, n))\n", 137 | " L = np.zeros((p, n))\n", 138 | " dL = D - D@Z - E\n", 139 | " dLnorm = np.linalg.norm(dL)\n", 140 | " normD = np.linalg.norm(D)\n", 141 | " eta = normD**2.1\n", 142 | " while step < maxIter:\n", 143 | " if step % 100 == 0:\n", 144 | " value = f(Z, E, lam)\n", 145 | " print(step, value, 'gap', dLnorm)\n", 146 | " log.append(value)\n", 147 | " logbeta.append(beta)\n", 148 | " Enew = update_E(Z, L, lam, beta)\n", 149 | " Znew, r = update_Z(Z, Enew, L, beta, eta, r)\n", 150 | " dE = np.linalg.norm(Enew-E)\n", 151 | " dZ = np.linalg.norm(Znew-Z)\n", 152 | " E = Enew\n", 153 | " Z = Znew\n", 154 | " dl = np.sum(Z, axis=0) - np.ones((1, n))\n", 155 | " dL = D - D@Z - E\n", 156 | " dLnorm = np.linalg.norm(dL)\n", 157 | " L = update_L(L, dL, beta)\n", 158 | " crit1 = dLnorm/normD < eps1\n", 159 | " beta, crit2 = update_beta(\n", 160 | " beta, maxBeta, rho0, eta, eps2, dE, dZ, normD)\n", 161 | " if crit1 and crit2:\n", 162 | " print(\"Converged at step\", step)\n", 163 | " value = f(Z, E, lam)\n", 164 | " print(step, value, 'gap', dLnorm)\n", 165 | " break\n", 166 | " step += 1\n", 167 | " return Z, E, L" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 39, 173 | "metadata": { 174 | "tags": [ 175 | "outputPrepend" 176 | ] 177 | }, 178 | "outputs": [ 179 | { 180 | "name": "stdout", 181 | "output_type": "stream", 182 | "text": [ 183 | "0 0.0 gap 246.53949594651502\n", 184 | "100 414.30001794788654 gap 0.8370096818523203\n", 185 | "200 414.4655383136877 gap 10.328989144803254\n", 186 | "300 414.093033645796 gap 9.958459548548609\n", 187 | "400 413.29111424503895 gap 2.321450503206335\n", 188 | "500 412.7611120409464 gap 5.126823701270025\n", 189 | "600 412.5715526012972 gap 6.943716677872358\n", 190 | "700 412.5906754712457 gap 3.185739649193045\n", 191 | "800 412.8205902517316 gap 1.989145534763514\n", 192 | "900 412.93757336995037 gap 4.413762129637172\n", 193 | "1000 412.754224799284 gap 2.9797742446376567\n", 194 | "1100 412.4614552301714 gap 0.29748367728679415\n", 195 | "1200 412.2895912066168 gap 2.5397089713616463\n", 196 | "1300 412.2614607055167 gap 2.356049445405693\n", 197 | "1400 412.32839232497486 gap 0.4792333192455525\n", 198 | "1500 412.40374223437533 gap 1.2857905226451145\n", 199 | "1600 412.40230197178386 gap 1.6646296714050144\n", 200 | "1700 412.31898280373514 gap 0.7237482801479137\n", 201 | "1800 412.22157509911125 gap 0.5179269521075035\n", 202 | "1900 412.16951109286526 gap 1.0694728150149309\n", 203 | "2000 412.1754881881704 gap 0.696819525297019\n", 204 | "2100 412.213200194542 gap 0.09718690620299807\n", 205 | "2200 412.23619577027165 gap 0.6225839111325161\n", 206 | "2300 412.2215627118176 gap 0.5582428939712575\n", 207 | "2400 412.18184981139893 gap 0.09788179621297131\n", 208 | "2500 412.14930252610094 gap 0.3206736967268821\n", 209 | "2600 412.13908284535364 gap 0.3986587552014835\n", 210 | "2700 412.1478814609209 gap 0.16310201776910327\n", 211 | "2800 412.16080684530124 gap 0.1344130775261857\n", 212 | "2900 412.1647483800312 gap 0.25890154025221135\n", 213 | "3000 412.15572705413535 gap 0.16173750426874553\n", 214 | "3100 412.14083216954435 gap 0.030694243660896594\n", 215 | "3200 412.130411023154 gap 0.15275930617813438\n", 216 | "Converged at step 3249\n", 217 | "3249 412.14833022378735 gap 0.15942138826341534\n" 218 | ] 219 | } 220 | ], 221 | "source": [ 222 | "log = []\n", 223 | "logbeta = []\n", 224 | "Z, E, L = LADMAP(f, D, lam, beta, maxBeta, maxIter, rho0, eps1, eps2, r, log, logbeta)" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 43, 230 | "metadata": {}, 231 | "outputs": [ 232 | { 233 | "data": { 234 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAYNUlEQVR4nO3dfZBk1Xnf8e8zszvA7iLztkaYF60i4biQKkLyBoPlchAqKYAdg1JIEeVEWKGyUQpXoVKqIuykIjtlV0lyLPmlbKVQIEaxzEskJRCV4oQgiIIjgRYJkASWtFaJgjWCBYHEdJvu7e4nf9zTvXd6e2aH3e3pGd3vp2pqbp++PfNMw95fn3vuPScyE0mSAOZmXYAkaf0wFCRJI4aCJGnEUJAkjRgKkqSRTbMu4EiccsopuWPHjlmXIUkbygMPPPBMZm6f9NyGDoUdO3awe/fuWZchSRtKRDy23HOePpIkjRgKkqQRQ0GSNGIoSJJGDAVJ0oihIEkaMRQkSSMb+j6Fabv328/wradeYJBZvmCQSSYMBkm/tAFsmgs2zUf1fW6OzfPBpvk5Ns0Fm+fnmJ8Ljt08z3Gb5zlu4cD3LQvzo/bN80FEzPaPltRohsIK/sWfPsALnd6a/b75ueC4zfMsbJpjYX6OzZuChfk5FjbNszAfVXt5bn5ujrmoXjMXwdxcVI9r23O1gKkvm5HkkvZh2PWHQTeotuvtozCsfwdy7HFQ/d6qpvr2gZrmogrP+fJV3z7weI758vqI2muX/ByICOo5Ghx4MJ6vmdXfPnwvhmuJVO1Ln8/yxHj7SxVR1VR9rxqitM/Vtod/R33fudrfFnHgLxs+P/wAMfrZqzD8Ew56D8bbx193hOuujNcP1B7HxPZ6YxzcdMRW8ycd7dVmJpU+6e+JCXuO7/d3d5zIq3/8+KNTWI2hsIz+IHmh0+M9f+9VXPOmV40OZsN/zMMDckSQ5cDZG371B+zvJ73BgF4/2d8f0Bsknf0D/mZ/v/rq9sr3Ae1ujxdH7QO6/T7dXvUzur0Bnd6A/f0B3d6AF/cP+OHf9EYH6gMH7QMH8ExGz6/mgDk8gA8DZT6iFjb1oInRwYqgHJTnmJs7cDADltTUHwxG25lV6PQHVU+rNxgwSOgNBvT71Xs3yOF7mEt7aIMD22upfgAeHsBX60DYHAgX6Wj5rctfayispXa36iGcvHWB44/dvOK+EeXU0fxaVKbMYeBUQTF5n9o2OQrEA5+8lwbZ+IF/mqfxxntXS3ooY72VYc+M0Sf4Wk+GA4Ez/JtXU/ZBn9BHj4fPT/rIfvif0MfrH7Wx9O+oP166z5L/mIf+fay+57SaHSd9aj8cOan41TVN/EBx/LHTOXwbCstod/sAbDnGI/16ExHMl57aRjQ8VTR3lA420tE09auPImI+Ir4aEZ8tj18ZEfdFxJ6IuDUiFkr7MeXxnvL8jmnXtpLFMpaw7RhzU1JzrMUlqdcCj9Yefwj4aGa+GngOuLq0Xw08V9o/WvabmXan9BQWDAVJzTHVUIiIM4BfAP5jeRzARcCnyi43AZeX7cvKY8rzb44ZXp/ZKmMKWxc8fSSpOabdU/g94F8Bg/L4ZOD5zBxe5/kEcHrZPh14HKA8/4Oy/xIRsSsidkfE7n379k2t8FY5fbTV00eSGmRqoRARvwg8nZkPHM2fm5nXZ+bOzNy5ffvEhYOOilYZaN7qQLOkBpnmx+A3Ar8UEZcCxwIvA34fOCEiNpXewBnA3rL/XuBM4ImI2AT8GPDsFOtbUbv0FBxTkNQkU+spZOavZeYZmbkDeCfw+cz8ZeBu4Iqy21XA7WX7jvKY8vzn80hvozwCi54+ktRAs5gQ7/3A+yJiD9WYwQ2l/Qbg5NL+PuC6GdQ2MrpPwYFmSQ2yJh+DM/Me4J6y/R3gvAn7vAi8fS3qWY1Wt8fCpjk2zzuRrKTm8Ii3jHan741rkhrHUFhGq9Pz1JGkxjEUltHq9tjqlUeSGsZQWEa723cyPEmNYygsY7HTc0xBUuMYCstod/qOKUhqHENhGY4pSGoiQ2EZrU7Pu5klNY6hsIyWA82SGshQmGB/f0C3N/D0kaTGMRQmGK665ukjSU1jKEzgqmuSmspQmKBdQmGLPQVJDWMoTLBYTh9tc6BZUsMYChO46pqkpjIUJhitz2woSGoYQ2GC1mgpTk8fSWoWQ2GC0dVHDjRLahhDYYLhfQpOiCepaQyFCRYdaJbUUIbCBO1uj+M2zzM/F7MuRZLWlKEwQavbd5BZUiMZChM4bbakpjIUJmh1+o4nSGokQ2GCdrfnZHiSGslQmMDTR5KaylCYwIFmSU1lKEzQ7vQcU5DUSIbCBIudHts8fSSpgQyFMZlJu9t3igtJjWQojOn2B/QG6UCzpEYyFMY4GZ6kJjMUxix2nDZbUnMZCmParromqcEMhTHDBXa2eJ+CpAYyFMYMl+L0klRJTWQojGk50CypwQyFMe3h+syOKUhqIENhTMurjyQ12NRCISKOjYj7I+KhiPhGRPxmaX9lRNwXEXsi4taIWCjtx5THe8rzO6ZV20paw6uPHGiW1EDT7Cl0gIsy83XAucDFEXE+8CHgo5n5auA54Oqy/9XAc6X9o2W/Ndfu9IiA4zYbCpKaZ2qhkJXF8nBz+UrgIuBTpf0m4PKyfVl5THn+zRER06pvOYudPlsXNjGDXy1JMzfVMYWImI+IB4GngTuBvwKez8xe2eUJ4PSyfTrwOEB5/gfAyRN+5q6I2B0Ru/ft23fUa253e155JKmxphoKmdnPzHOBM4DzgJ86Cj/z+szcmZk7t2/ffsQ1jqsW2HGQWVIzrcnVR5n5PHA3cAFwQkQMj7pnAHvL9l7gTIDy/I8Bz65FfXXVUpz2FCQ10zSvPtoeESeU7eOAtwCPUoXDFWW3q4Dby/Yd5THl+c9nZk6rvuW0XHVNUoNN8+h3GnBTRMxThc9tmfnZiHgEuCUifgv4KnBD2f8G4D9HxB7g+8A7p1jbstrdPqdsW5jFr5akmZtaKGTmw8DrJ7R/h2p8Ybz9ReDt06pntVqdHq84ecusy5CkmfCO5jGtbs8pLiQ1lqEwpt3pO222pMYyFGoyk1a357TZkhrLUKh5cf+AQeLVR5Iay1CoGa665n0KkprKUKgZTZttT0FSQxkKNcNV1+wpSGoqQ6FmuOqaYwqSmspQqFnsOKYgqdkMhZr2aNU1ewqSmslQqHGgWVLTGQo1w56Ci+xIaipDoebAmII9BUnNZCjUtLs95ueCYzb5tkhqJo9+Na1Ony0L80TErEuRpJkwFGpaHSfDk9RshkJNu9t3kFlSoxkKNa1uz0FmSY1mKNS0Oq66JqnZDIWaVqfvFBeSGs1QqGl3e06GJ6nRDIWaxU7fMQVJjWYo1LS7PbZ69ZGkBjMUisEgq0tS7SlIajBDoWjvrybD2+ZAs6QGMxSKdsdV1yTJUChaXddnlqRVh0JEbJlmIbPmAjuStIpQiIifjYhHgL8sj18XEX889crWWMu1FCRpVT2FjwJ/H3gWIDMfAn5+mkXNgquuSdIqTx9l5uNjTf0p1DJTw1XXnDpbUpOt5gj4eET8LJARsRm4Fnh0umWtvXa3XH1kKEhqsNX0FN4DXAOcDuwFzi2Pf6S0OuXqI08fSWqwQ34szsxngF9eg1pmquV9CpJ06FCIiP8E5Hh7Zv7TqVQ0I61un4X5ORY2eeuGpOZazcfiz9a2jwXeBvz1dMqZnXa3xxZvXJPUcKs5ffTp+uOIuBm4d2oVzciiq65J0mFNc3E28ONHu5BZa7vqmiSt6o7mFyLih8PvwH8H3r+K150ZEXdHxCMR8Y2IuLa0nxQRd0bEt8v3E0t7RMQfRMSeiHg4It5wpH/cS9Fy1TVJOnQoZObxmfmy2vefHD+ltIwe8C8z8xzgfOCaiDgHuA64KzPPBu4qjwEuoeqFnA3sAj52GH/PYWt1evYUJDXesh+ND/VJPTO/cojnnwSeLNsvRMSjVPc6XAZcWHa7CbiHqudxGfCJzEzgSxFxQkScVn7O1LW7fU7Zdsxa/CpJWrdWOl/yuys8l8BFq/0lEbEDeD1wH3Bq7UD/PeDUsn06UJ9O44nStiah0Or2nAxPUuMtexTMzDcdjV8QEduATwPvzcwfRkT9d2REHHQPxCF+3i6q00ucddZZR6NEoBpodjI8SU23qo/GEfFa4Byq+xQAyMxPrOJ1m6kC4ZOZ+ZnS/NTwtFBEnAY8Xdr3AmfWXn5GaVsiM68HrgfYuXPnSwqUlSx2ek6GJ6nxVnP10QeAPyxfbwI+DPzSKl4XwA3Ao5n5kdpTdwBXle2rgNtr7e8qVyGdD/xgrcYTev0Bnd7Aq48kNd5qjoJXAK8DvpqZ746IU4E/XcXr3gj8E+BrEfFgaft14IPAbRFxNfAY8I7y3OeAS4E9QBt496r/iiPU3u9SnJIEqwuFFzNzEBG9iHgZ1emeMw/1osy8F4hlnn7zhP2TGc2+6qprklRZ6ZLUPwJuBu6PiBOAjwMPAIvAF9emvLUxnDbbgWZJTbfSR+NvAb8D/ATQogqItwAvy8yH16C2NTNcYMe5jyQ13bIDzZn5+5l5AdV6zM8CNwJ/DrwtIs5eo/rWxKKnjyQJWN00F49l5ocy8/XAlcDlwF9OvbI11O440CxJsLpLUjdFxD+IiE8C/wP4JvAPp17ZGmp1XXVNkmDlgea3UPUMLgXuB24BdmVma41qWzPDgWZvXpPUdCsdBX8N+DOqmU6fW6N6ZmI40OzKa5KabqW5j1Y94d1GN7okdbOhIKnZXKWeakzh2M1zbJr37ZDUbB4FKQvsOMgsSYYCVAvsOJ4gSYYCUN28Zk9BkgwFoLr6yLuZJclQAKqrj5wMT5IMBcCBZkkaMhSoBpo9fSRJhgJQ3afgZHiSZCgA1ekjJ8OTJEOBbm/A/n6yzZ6CJBkKbafNlqSRxofCgVXX7ClIUuNDod0drrpmT0GSGh8KrWFPwdNHkmQoDHsK3tEsSYZCbUzBnoIkNT4UhlcfGQqSZCiMluLc6ukjSTIUWp4+kqQRQ6EMNB+32Z6CJDU+FNqdHlsW5pmbi1mXIkkz1/hQaLnqmiSNGAqdvoPMklQ0PhTaXafNlqShxofCYqfHNk8fSRJgKNDu9tniDKmSBBgKtDo9J8OTpMJQ6PRdS0GSCkPBgWZJGml0KGQm7a49BUkamlooRMSNEfF0RHy91nZSRNwZEd8u308s7RERfxAReyLi4Yh4w7Tqquv0BvQHaU9Bkopp9hT+BLh4rO064K7MPBu4qzwGuAQ4u3ztAj42xbpGhpPheUmqJFWmFgqZ+QXg+2PNlwE3le2bgMtr7Z/IypeAEyLitGnVNuSqa5K01FqPKZyamU+W7e8Bp5bt04HHa/s9UdoOEhG7ImJ3ROzet2/fERXjqmuStNTMBpozM4E8jNddn5k7M3Pn9u3bj6gGV12TpKXWOhSeGp4WKt+fLu17gTNr+51R2qbKVdckaam1DoU7gKvK9lXA7bX2d5WrkM4HflA7zTQ1w4Fmrz6SpMrUjoYRcTNwIXBKRDwBfAD4IHBbRFwNPAa8o+z+OeBSYA/QBt49rbrqhquuefWRJFWmdjTMzCuXeerNE/ZN4Jpp1bKc4ZiCE+JJUqXRdzSPrj7y9JEkAQ0PhXanz1zAsZsb/TZI0kijj4atbjVtdkTMuhRJWheaHQqdnuMJklTT7FDo9r1xTZJqGh0KbVddk6QlGh0KrW7fyfAkqabZodDpeeOaJNU0OhTa3T5bDAVJGml0KLQ6PSfDk6QaQ8GegiSNNDYUBoOkvb9vT0GSahobCi/2+mTimIIk1TQ2FFyKU5IO1thQaLvqmiQdpLGh0Oq66pokjWtuKHRcdU2SxjU3FFx1TZIO0thQODCmYE9BkoYaGwqtznBMwZ6CJA01NxTK6SPHFCTpgMaGQrtbnT5yTEGSDmhsKCx2emyaCxbmG/sWSNJBGntEbJfJ8CJi1qVI0rrR2FBodZ0MT5LGNTcUOj0nw5OkMc0NhW7fyfAkaUxjQ6HtqmuSdJDGhsJip+dkeJI0prGh0O722eY9CpK0RINDwYFmSRrX2FBYdExBkg7SyFDoD5IX9w+8+kiSxjQyFNplMjynzZakpRoaCk6GJ0mTNDIUFjtOmy1JkzQyFIarrnmfgiQt1chQaI3GFDx9JEl1zQyFcvrIq48kaal1FQoRcXFEfDMi9kTEddP6Pa0y0LzVgWZJWmLdhEJEzAN/BFwCnANcGRHnTON3tUtPwTEFSVpq3YQCcB6wJzO/k5ld4Bbgsmn8okVPH0nSROspFE4HHq89fqK0LRERuyJid0Ts3rdv32H9orNO2sLFr3k5WxxolqQlNtxH5cy8HrgeYOfOnXk4P+Otr3k5b33Ny49qXZL0o2A99RT2AmfWHp9R2iRJa2Q9hcKXgbMj4pURsQC8E7hjxjVJUqOsm9NHmdmLiF8F/icwD9yYmd+YcVmS1CjrJhQAMvNzwOdmXYckNdV6On0kSZoxQ0GSNGIoSJJGDAVJ0khkHtb9X+tCROwDHjvMl58CPHMUy1lrG7n+jVw7WP8sbeTaYf3U/4rM3D7piQ0dCkciInZn5s5Z13G4NnL9G7l2sP5Z2si1w8ao39NHkqQRQ0GSNNLkULh+1gUcoY1c/0auHax/ljZy7bAB6m/smIIk6WBN7ilIksYYCpKkkUaGQkRcHBHfjIg9EXHdrOt5KSLiuxHxtYh4MCJ2z7qeQ4mIGyPi6Yj4eq3tpIi4MyK+Xb6fOMsaV7JM/b8REXvLf4MHI+LSWda4nIg4MyLujohHIuIbEXFtaV/37/8KtW+U9/7YiLg/Ih4q9f9maX9lRNxXjj23lmUC1pXGjSlExDzwLeAtVEt+fhm4MjMfmWlhqxQR3wV2ZuZ6uAHmkCLi54FF4BOZ+drS9mHg+5n5wRLKJ2bm+2dZ53KWqf83gMXM/PezrO1QIuI04LTM/EpEHA88AFwO/Arr/P1fofZ3sDHe+wC2ZuZiRGwG7gWuBd4HfCYzb4mI/wA8lJkfm2Wt45rYUzgP2JOZ38nMLnALcNmMa/qRlZlfAL4/1nwZcFPZvonqH/u6tEz9G0JmPpmZXynbLwCPUq17vu7f/xVq3xCyslgebi5fCVwEfKq0r8v3vomhcDrweO3xE2yg/9mo/sf6XxHxQETsmnUxh+nUzHyybH8POHWWxRymX42Ih8vppXV3+mVcROwAXg/cxwZ7/8dqhw3y3kfEfEQ8CDwN3An8FfB8ZvbKLuvy2NPEUNjofi4z3wBcAlxTTm9sWFmdv9xo5zA/BrwKOBd4Evjd2ZazsojYBnwaeG9m/rD+3Hp//yfUvmHe+8zsZ+a5VOvNnwf81IxLWpUmhsJe4Mza4zNK24aQmXvL96eB/0r1P9tG81Q5Zzw8d/z0jOt5STLzqfIPfgB8nHX836Ccz/408MnM/Exp3hDv/6TaN9J7P5SZzwN3AxcAJ0TEcMXLdXnsaWIofBk4u1wFsAC8E7hjxjWtSkRsLYNuRMRW4K3A11d+1bp0B3BV2b4KuH2GtbxkwwNq8TbW6X+DMth5A/BoZn6k9tS6f/+Xq30DvffbI+KEsn0c1YUtj1KFwxVlt/X53jft6iOAchnb7wHzwI2Z+dszLmlVIuJvUfUOoFpf+8/We+0RcTNwIdWUwU8BHwD+G3AbcBbV1OfvyMx1OZi7TP0XUp2+SOC7wD+vnaNfNyLi54D/C3wNGJTmX6c6N7+u3/8Var+SjfHe/x2qgeR5qg/ft2Xmvyv/hm8BTgK+CvzjzOzMrtKDNTIUJEmTNfH0kSRpGYaCJGnEUJAkjRgKkqQRQ0GSNGIoSKsQEf+6zHb5cJmd82ci4r0RsWXWtUlHk5ekSocQERcAHwEuzMxORJwCLAD/jw00Y620GvYUpEM7DXhmeJNRCYErgJ8A7o6IuwEi4q0R8cWI+EpE/Jcyb89wDYwPl3Uw7o+IV5f2t0fE18uc+1+YzZ8mLWVPQTqEcnC/F9gC/G/g1sz8P/W1LUrv4TPAJZnZioj3A8eUu1i/C3w8M387It5FdQfxL0bE14CLM3NvRJxQ5siRZsqegnQIZV78nwZ2AfuAWyPiV8Z2Ox84B/iLMl3yVcAras/fXPt+Qdn+C+BPIuKfUU2HIM3cpkPvIikz+8A9wD3lE/5VY7sEcGdmXrncjxjfzsz3RMTPAL8APBARP52Zzx7dyqWXxp6CdAgR8bcj4uxa07lUE8m9ABxf2r4EvLE2XrA1In6y9pp/VPv+xbLPqzLzvsz8t1Q9kPqU7tJM2FOQDm0b8IdlKuQesIfqVNKVwJ9HxF9n5pvKKaWbI+KY8rp/Q7UeOMCJEfEw0CmvA/idEjYB3AU8tCZ/jbQCB5qlKasPSM+6FulQPH0kSRqxpyBJGrGnIEkaMRQkSSOGgiRpxFCQJI0YCpKkkf8PDUrGAVclcaMAAAAASUVORK5CYII=", 235 | "text/plain": [ 236 | "
" 237 | ] 238 | }, 239 | "metadata": { 240 | "needs_background": "light" 241 | }, 242 | "output_type": "display_data" 243 | } 244 | ], 245 | "source": [ 246 | "plt.plot(log)\n", 247 | "plt.xlabel('Steps')\n", 248 | "plt.ylabel('Value')\n", 249 | "plt.savefig('21.png', dpi=200)" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 44, 255 | "metadata": {}, 256 | "outputs": [ 257 | { 258 | "data": { 259 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAXPElEQVR4nO3df7DddX3n8eebJAQIIIm5jSEEAzSs4tYNmOGHui6tLSB1JrirFDqj0WU2bRd3ZLe7s6idVrvDjmsrndVxcbCwhq7LjxZZ0g5WKabrSgUMCAGCQFQiiYEELAFBAvfe9/5xPvfcQ3Jyc3Nvzvl+z/0+HzN3zvd8zveE9/nm3Lz4fD7f7+cbmYkkSQCHVF2AJKk+DAVJUpuhIElqMxQkSW2GgiSpbXbVBUzHwoULc9myZVWXIUkD5d57730mM4e6vTbQobBs2TI2bNhQdRmSNFAiYsu+XnP4SJLUZihIktoMBUlSm6EgSWozFCRJbYaCJKnNUJAktQ30dQqS1CvPvfQK/+uuLbwyPFp1KV2tXLaAd53c9fqzaTEUJKmLOx7ZwZ9+8zEAIioupovf/RcnGQqS1C8j5QZkd17+ayw55vCKq+kf5xQkqZuG3pSyZ6EQEUsjYn1EbIqIhyPiY6X9UxGxLSLuLz/nd7zn4xGxOSIejYhze1WbJE1WDUeOeqqXw0fDwO9n5n0RcRRwb0TcXl77s8z8086dI+IU4CLgLcCxwN9FxMmZOdLDGiWpq2xoV6FnPYXM3J6Z95XtF4BHgCUTvGUVcENm7s7MHwObgdN7VZ8kTUYdJ5l7qS9zChGxDDgVuLs0fTQiNkbEtRExv7QtAZ7seNtWuoRIRKyJiA0RsWHnzp09rFpSk2UzOwq9D4WIOBK4GbgsM58HrgJOAlYA24HPHcifl5lXZ+bKzFw5NHTwT8eSJBifZ46GzSr0NBQiYg6tQPhqZn4NIDOfzsyRzBwFvsz4ENE2YGnH248rbZKkPunl2UcBXAM8kplXdrQv7tjtfcBDZXsdcFFEzI2IE4DlwD29qk+SJjI2fNS0OYVenn30DuCDwIMRcX9p+wRwcUSsoNU7ewL4HYDMfDgibgI20Tpz6VLPPJKk/upZKGTmd+h+iu9tE7znCuCKXtUkSZM1dkpqwzoKXtEsSRpnKEhSFzl++lGjGAqSpDZDQZK68DoFSVLjGQqS1E2ZVGjadQqGgiSpzVCQpC4aevKRoSBJGmcoSFIX42sfNauvYChIktoMBUnqItO1jyRJDWcoSFIX7bOPGtZVMBQkSW2GgiR10T77qGGzCoaCJKnNUJCkLsbmFBrWUTAUJEnjDAVJ6iJdJVWS1HSGgiRNoGEdBUNBkjTOUJCkLlwlVZLUeIaCJHWRuEqqJKnI3P8+M5GhIEkTaNiUgqEgSd00tKPQu1CIiKURsT4iNkXEwxHxsdK+ICJuj4jHy+P80h4R8fmI2BwRGyPitF7VJkmT5SqpB88w8PuZeQpwJnBpRJwCXA7ckZnLgTvKc4D3AMvLzxrgqh7WJkkTck7hIMvM7Zl5X9l+AXgEWAKsAtaW3dYCF5TtVcB12XIXcExELO5VfZI0Gc4p9EBELANOBe4GFmXm9vLSU8Cisr0EeLLjbVtLmyT1XTZ0VqHnoRARRwI3A5dl5vOdr2VrGcIDOvIRsSYiNkTEhp07dx7ESiVJPQ2FiJhDKxC+mplfK81Pjw0LlccdpX0bsLTj7ceVttfIzKszc2VmrhwaGupd8ZIazTmFgyxaC4ZcAzySmVd2vLQOWF22VwO3drR/qJyFdCawq2OYSZIq0bQ5hdk9/LPfAXwQeDAi7i9tnwA+A9wUEZcAW4ALy2u3AecDm4GXgI/0sDZJUhc9C4XM/A77Xjbk3V32T+DSXtUjSVPhdQqSpPbtOJvGUJCkCTRtTsFQkKQuGtpRMBQkaSIN6ygYCpLUTUM7CoaCJE3EezRLkpxTkCTtrVn9BENBkrpylVRJ0l4aNqVgKEhSN84pSJL24tlHkqSGzigYCpKkDoaCJHXT0EkFQ0GS9qFh0wmAoSBJXTWzn2AoSFJXmc27mhkMBUlSB0NBkrpIsnHXKIChIEnqYChIUhfOKUiSGs9QkKQuEq9TkCQ1nKEgSV205hSa11UwFCRJbYaCJHWRNPP0I0NBktRmKEhSN83sKPQuFCLi2ojYEREPdbR9KiK2RcT95ef8jtc+HhGbI+LRiDi3V3VJkvatlz2FrwDndWn/s8xcUX5uA4iIU4CLgLeU9/yPiJjVw9okaUJep3CQZea3gZ9NcvdVwA2ZuTszfwxsBk7vVW2SpO6qmFP4aERsLMNL80vbEuDJjn22lra9RMSaiNgQERt27tzZ61olNVRmep1CH1wFnASsALYDnzvQPyAzr87MlZm5cmho6GDXJ0ltDh/1WGY+nZkjmTkKfJnxIaJtwNKOXY8rbZJUiWzo/Tj7GgoRsbjj6fuAsTOT1gEXRcTciDgBWA7c08/aJGlPDewoMLtXf3BEXA+cDSyMiK3AHwFnR8QKWhP7TwC/A5CZD0fETcAmYBi4NDNHelWbJO1PQzsKvQuFzLy4S/M1E+x/BXBFr+qRpAPl7TglSYBzCpKkPTSvn3AAw0cR8Zu0rjg+bKwtM/+4F0VJUtWyobMKk+opRMSXgN8C/h2t8PwA8MYe1iVJ1WtgV2GyPYW3Z+ZbI2JjZn46Ij4HfL2XhUlqhjXXbWDT9uerLmMvz730aiMvXptsKPyiPL4UEccCzwKLJ9hfkiblWz/YwUlDR/KWJUdXXcpefmXJ66ouoe8mGwp/ExHHAH8C3EfrFN4/71lVkhojgV8/5Zf4T+e+qepSxORD4bOZuRu4OSL+htZk88u9K0tSUzR14bm6muwpqd8d2yjLW+/qbJOk6Wji2H1dTdhTiIg30FrC+vCIOJXxufijgSN6XJukBkgaeZJPbe1v+Ohc4MO0Vi29sqP9eeATPapJUoNkU29xVlMThkJmrgXWRsS/ysyb+1STpIYxEupjsnMKd0bENRHxdWjdUzkiLulhXZIaIMsCQ3YU6mOyofA/gW8Ax5bnjwGX9aQiSY0xtuicZx/Vx2RDYWFm3gSMAmTmMOD9DiRNy9jqQvYU6mOyofBiRLye8ncYEWcCu3pWlaRGaA8fVVyHxk324rX/QOuWmSdGxJ3AEPD+nlUlqRHsKdTPZENhE3AL8BLwAvB/aM0rSNKUtecUTIXamOzw0XXAm4D/CnwBOBn4i14VJakZmnrPgjqbbE/hn2bmKR3P10fEpl4UJKk5mnrLyzqbbE/hvjK5DEBEnAFs6E1JkprG0aP62N/aRw/SmguaA/xDRPykPH8j8IPelydpJvM6hfrZ3/DRe/tShaRGGptTsKdQH/tb+2hLvwqR1DzjPQXVxWTnFCTpoPM6hfoxFCRVZvyKZlOhLgwFSZWxp1A/hoKkynidQv0YCpKq4zIXtdOzUIiIayNiR0Q81NG2ICJuj4jHy+P80h4R8fmI2BwRGyPitF7VJak+2qekVlyHxvWyp/AV4Lw92i4H7sjM5cAd5TnAe4Dl5WcNcFUP65JUE+ML4lVbh8b1LBQy89vAz/ZoXgWsLdtrgQs62q/LlruAYyJica9qk1QP7YnmSqtQp37PKSzKzO1l+ylgUdleAjzZsd/W0raXiFgTERsiYsPOnTt7V6mknhu/R7OxUBeVTTRn69twwOceZObVmbkyM1cODQ31oDJJ/eIpqfXT71B4emxYqDzuKO3bgKUd+x1X2iTNYC5zUT/9DoV1wOqyvRq4taP9Q+UspDOBXR3DTJJmqMSZ5rqZ7E12DlhEXA+cDSyMiK3AHwGfAW6KiEuALcCFZffbgPOBzbRu+fmRXtUlqUbsKdROz0IhMy/ex0vv7rJvApf2qhZJ9eScQv14RbOkyniTnfoxFCRVxpvs1I+hIKlyZkJ9GAqSKuMyF/VjKEiqzPgyF6ZCXRgKkiqTXr1WO4aCpMqYCfVjKEiqnAvi1YehIKky9hTqx1CQVBmvU6gfQ0FSZTwltX4MBUmVOeAbqqjnDAVJlWnfec1ZhdowFCRVxlVS68dQkFSZdPyodnp2PwVJ/ZeZfOFbm3n6+ZerLmVSdv3iVcDrFOrEUJBmkJ0/382Vtz/GvENncfihs6ouZ1KWHHM4vzx0ZNVlqDAUpBlkdLT1+AfvPYWLTz++2mI0kJxTkGaQ0TJIf4ijMZoiQ0GaQcZCwTF6TZWhIM0gY2fzHGIoaIoMBWkGcfhI02UoSDOIPQVNl6EgzSDjcwoVF6KBZShIM8hoe9VRU0FTYyhIM0g6p6BpMhSkGWTUOQVNk6EgzSCefaTpMhSkGcSL1zRdlax9FBFPAC8AI8BwZq6MiAXAjcAy4Angwsz8xyrqkwZV+/aW1ZahAVZlT+FXM3NFZq4szy8H7sjM5cAd5bmkA+B1CpquOg0frQLWlu21wAUV1iINpPacQp1+szVQqvrqJPDNiLg3ItaUtkWZub1sPwUsqqY0aXA5p6Dpqup+Cu/MzG0R8UvA7RHxg84XMzMjouuN+kqIrAE4/njXi5c6eUqqpquSnkJmbiuPO4BbgNOBpyNiMUB53LGP916dmSszc+XQ0FC/SpYGghevabr6HgoRMS8ijhrbBs4BHgLWAavLbquBW/tdmzTo2stceP6RpqiK4aNFwC1lzHM28L8z828j4nvATRFxCbAFuLCC2qSB5sVrmq6+h0Jm/gj4Z13anwXe3e96pJkkXRBP0+SJa9IM4pyCpstQkGaQ9tlHpoKmyFCQZpD2dQoV16HBVdV1CtJA+9YPnuaW7/+06jL2suP5lwHnFDR1hoI0BV+96yf8v8ef4bj5h1ddyl7eetzrWPb6I6ouQwPKUJCmYHg0efOxR3Prpe+ouhTpoHJOQZqC0UxmOUKjGchQkKZgeCSZ5Rk+moEMBWkKRtJQ0MxkKEhTMDJqKGhmMhSkKWiFgr8+mnn8VktTMDLqRLNmJkNBmgJ7Cpqp/FZLU9AKhaqrkA4+v9bSFIxkMtuegmYgv9XSFIyMpiuRakYyFKQpGBlNZhsKmoFc+0i19ehTL7Dqi9/h5VdHqy6lqzNPXFB1CdJBZyiotrY8+yIvvzrKb59xPENHzq26nL28962Lqy5BOugMBdXWKyOtHsKH376MkxcdVXE1UjM4p6DaerWEwhzP/ZT6xt821dYrw61QOHS2X1OpX/xtU221Q8GegtQ3/raptl4Zad2E3lCQ+seJZvHpv36Y7/7w2arL2MuzL74COHwk9ZOhIG69/6fMmzuLN7/h6KpLeY3jFxzBCQvncdgcQ0HqF0NBvLh7mA+87Tg+fv6bqy5FUsUMhT7aPTxCZtVVvNbwaLJ7eJQjDvWrIMlQ6Jtbvr+Vf3/jA1WXsU9HH+5XQZKh0BPDI6PsHn7tej3f/8lzzJ19CJf9+skVVbVvc2YFF6xYUnUZkmqgdqEQEecB/x2YBfx5Zn6m4pIOyPDIKP/8s+vZvuvlvV47ceE8fu/skyqoSpImp1ahEBGzgC8CvwFsBb4XEesyc1Mv/nvZZYB/1y9eZd0DP21fODWROx7ZwV0/frbrPMG/PG0Jb3rDa9frWbF0/pRrlaR+qFUoAKcDmzPzRwARcQOwCjioobD+0R38l7/exI4XdvPz3cPT+rPmHzGHD5617DVth805hNVnLWPe3LodXkmaWN3+1VoCPNnxfCtwRucOEbEGWANw/PHHT+k/cszhc3jzsUfz9iPm8Pp5c4k97pVy+gkL+JUlr5vUnzXv0NnegUvSjFG3UNivzLwauBpg5cqVUzrB89Tj5/PF33YoR5L2VLdLRbcBSzueH1faJEl9ULdQ+B6wPCJOiIhDgYuAdRXXJEmNUavho8wcjoiPAt+gdUrqtZn5cMVlSVJj1CoUADLzNuC2quuQpCaq2/CRJKlChoIkqc1QkCS1GQqSpLbotv7PoIiIncCWKb59IfDMQSynn6y9GtZejUGtvc51vzEzh7q9MNChMB0RsSEzV1Zdx1RYezWsvRqDWvug1u3wkSSpzVCQJLU1ORSurrqAabD2alh7NQa19oGsu7FzCpKkvTW5pyBJ2oOhIElqa2QoRMR5EfFoRGyOiMurrqebiHgiIh6MiPsjYkNpWxARt0fE4+VxfmmPiPh8+TwbI+K0Ptd6bUTsiIiHOtoOuNaIWF32fzwiVldU96ciYls57vdHxPkdr3281P1oRJzb0d7371NELI2I9RGxKSIejoiPlfZBOO77qr32xz4iDouIeyLigVL7p0v7CRFxd6njxrL0PxExtzzfXF5ftr/PVLnMbNQPrSW5fwicCBwKPACcUnVdXep8Ali4R9tngcvL9uXAfyvb5wNfBwI4E7i7z7W+CzgNeGiqtQILgB+Vx/lle34FdX8K+I9d9j2lfFfmAieU79Csqr5PwGLgtLJ9FPBYqXEQjvu+aq/9sS/H78iyPQe4uxzPm4CLSvuXgN8r2/8W+FLZvgi4caLP1OvvzWR+mthTOB3YnJk/ysxXgBuAVRXXNFmrgLVley1wQUf7ddlyF3BMRCzuV1GZ+W3gZ3s0H2it5wK3Z+bPMvMfgduB8yqoe19WATdk5u7M/DGwmdZ3qZLvU2Zuz8z7yvYLwCO07nE+CMd9X7XvS22OfTl+Py9P55SfBH4N+KvSvudxH/v7+Cvg3RERE3ymyjUxFJYAT3Y838rEX8iqJPDNiLg3ItaUtkWZub1sPwUsKtt1/EwHWmudPsNHyxDLtWPDL9S47jIkcSqt/2sdqOO+R+0wAMc+ImZFxP3ADloh+kPgucwc7lJHu8by+i7g9VXVPhlNDIVB8c7MPA14D3BpRLyr88Vs9UEH4nziQaoVuAo4CVgBbAc+V205E4uII4Gbgcsy8/nO1+p+3LvUPhDHPjNHMnMFrXvInw68qeKSDqomhsI2YGnH8+NKW61k5rbyuAO4hdaX7+mxYaHyuKPsXsfPdKC11uIzZObT5Zd+FPgy41362tUdEXNo/aP61cz8WmkeiOPerfZBOvYAmfkcsB44i9Zw3NidLDvraNdYXn8d8Cw1+b5308RQ+B6wvJwtcCityZ91Fdf0GhExLyKOGtsGzgEeolXn2Nkhq4Fby/Y64EPlDJMzgV0dQwhVOdBavwGcExHzy7DBOaWtr/aYi3kfreMOrbovKmeTnAAsB+6hou9TGZe+BngkM6/seKn2x31ftQ/CsY+IoYg4pmwfDvwGrTmR9cD7y257Hvexv4/3A98qPbh9fabqVT3TXcUPrTMxHqM1FvjJquvpUt+JtM5MeAB4eKxGWmORdwCPA38HLCjtAXyxfJ4HgZV9rvd6Wt39V2mNjV4ylVqBf01rwm0z8JGK6v6LUtdGWr+4izv2/2Sp+1HgPVV+n4B30hoa2gjcX37OH5Djvq/aa3/sgbcC3y81PgT8YWk/kdY/6puBvwTmlvbDyvPN5fUT9/eZqv5xmQtJUlsTh48kSftgKEiS2gwFSVKboSBJajMUJElthoI0CRHxybIq5saygucZEXFZRBxRdW3SweQpqdJ+RMRZwJXA2Zm5OyIW0lqV8x9one//TKUFSgeRPQVp/xYDz2TmboASAu8HjgXWR8R6gIg4JyK+GxH3RcRflrV9xu6N8dlo3R/jnoj45dL+gYh4qKzN/+1qPpr0WvYUpP0o/7h/BziC1lXCN2bm/42IJyg9hdJ7+BqtK1NfjIj/TOuq1j8u+305M6+IiA8BF2bmeyPiQeC8zNwWEcdkay0dqVL2FKT9yNb6+W8D1gA7gRsj4sN77HYmrRun3FmWVV4NvLHj9es7Hs8q23cCX4mIf0PrhjFS5WbvfxdJmTkC/D3w9+X/8Pe8bWXQulnNxfv6I/bczszfjYgzgN8E7o2It2Xmswe3cunA2FOQ9iMi/klELO9oWgFsAV6gdTtJgLuAd3TMF8yLiJM73vNbHY/fLfuclJl3Z+Yf0uqBdC6lLFXCnoK0f0cCXyhLJg/TWvFyDXAx8LcR8dPM/NUypHR9RMwt7/sDWit4AsyPiI3A7vI+gD8pYRO0VjZ9oC+fRpqAE81Sj3VOSFddi7Q/Dh9JktrsKUiS2uwpSJLaDAVJUpuhIElqMxQkSW2GgiSp7f8DkwfgfGJAPqIAAAAASUVORK5CYII=", 260 | "text/plain": [ 261 | "
" 262 | ] 263 | }, 264 | "metadata": { 265 | "needs_background": "light" 266 | }, 267 | "output_type": "display_data" 268 | } 269 | ], 270 | "source": [ 271 | "plt.plot(logbeta)\n", 272 | "plt.xlabel('Steps')\n", 273 | "plt.ylabel('beta')\n", 274 | "plt.savefig('22.png', dpi=200)" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": null, 280 | "metadata": {}, 281 | "outputs": [], 282 | "source": [] 283 | } 284 | ], 285 | "metadata": { 286 | "interpreter": { 287 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 288 | }, 289 | "kernelspec": { 290 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 291 | "name": "python3" 292 | }, 293 | "language_info": { 294 | "name": "python", 295 | "version": "" 296 | }, 297 | "orig_nbformat": 2 298 | }, 299 | "nbformat": 4, 300 | "nbformat_minor": 2 301 | } -------------------------------------------------------------------------------- /19/3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# LADMPSAP\n", 8 | "\n", 9 | "### By ZincCat\n", 10 | "\n", 11 | "$$\\min _{\\mathbf{Z}, \\mathbf{E}}\\|\\mathbf{Z}\\|_{*}+\\lambda\\|\\mathbf{E}\\|_{2,1}$$\n", 12 | "\n", 13 | "$$\\text{s.t. }\\mathbf{D}=\\mathbf{D Z}+\\mathbf{E}, \\mathbf{Z}^{T} \\mathbf{1}=\\mathbf{1}, \\mathbf{Z}\\geqslant \\mathbf{0} $$" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 176, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import numpy as np\n", 23 | "from matplotlib import pyplot as plt\n", 24 | "from time import time\n", 25 | "from sklearn.decomposition import TruncatedSVD\n", 26 | "from sklearn.utils.extmath import randomized_svd\n", 27 | "%matplotlib inline" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 177, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "p = 200\n", 37 | "n = 300\n", 38 | "np.random.seed(19890817)\n", 39 | "D = np.random.normal(0, 1, (p, n))" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 178, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "def norm21(X):\n", 49 | " return np.linalg.norm(np.linalg.norm(X, 2, axis=0), 1)\n", 50 | "\n", 51 | "def f(Z, E, lam):\n", 52 | " return np.linalg.norm(Z, 'nuc') + lam * norm21(E)\n", 53 | "\n", 54 | "# l2,1-norm shrinkage operator\n", 55 | "def H(M, eps):\n", 56 | " norm = np.linalg.norm(M, 2, axis=0)\n", 57 | " M[:, norm>eps] *= (norm[norm>eps] - eps)/norm[norm>eps]\n", 58 | " M[:, norm<=eps] = 0\n", 59 | " return M\n", 60 | "\n", 61 | "\n", 62 | "# soft shrinkage operator\n", 63 | "def S(x, eps):\n", 64 | " x[np.abs(x) < eps] = 0\n", 65 | " x[x >= eps] -= eps\n", 66 | " x[x <= -eps] += eps\n", 67 | " return x" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 179, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "def update_E(Z, L, lam, beta):\n", 77 | " M = D - D@Z + L/beta\n", 78 | " # M = M[0:-1]\n", 79 | " Enew = H(M, lam/beta)\n", 80 | " # Enew = np.vstack((Enew, np.zeros(n)))\n", 81 | " return Enew\n", 82 | "\n", 83 | "def update_Z(Z, E, L, Y, M, l, beta, etaZ, r, n):\n", 84 | " N = D + L/beta - E\n", 85 | " W = Z - (D.T@(D@Z-N) + np.sum(l)/beta + np.sum(Z, axis=0)/beta-np.ones((1, n)))/etaZ + Y/etaZ - M/beta/etaZ\n", 86 | " U, sigma, V = np.linalg.svd(W)\n", 87 | " # U, sigma, V = randomized_svd(W, n_components=r)\n", 88 | " sigma = S(sigma, 1/beta/etaZ)\n", 89 | " # if count < r:\n", 90 | " # r = min(n, count+1)\n", 91 | " # else:\n", 92 | " # r = min(n, r + int(0.05*n))\n", 93 | " Znew = U@np.diag(sigma)@V\n", 94 | " return Znew\n", 95 | "\n", 96 | "def update_beta(beta, maxBeta, rho0, eta, eps2, dE, dZ, dY, normD):\n", 97 | " satisfied = False\n", 98 | " if beta*np.max((np.sqrt(3.1)*dE, np.sqrt(eta)*dZ, np.sqrt(3.1)*dY))/normD < eps2:\n", 99 | " rho = rho0\n", 100 | " satisfied = True\n", 101 | " else:\n", 102 | " rho = 1\n", 103 | " return min(beta*rho, maxBeta), satisfied" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 193, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "beta = 1e-4\n", 113 | "maxBeta = 1e4\n", 114 | "maxIter = 100000\n", 115 | "eps1 = 1e-2\n", 116 | "eps2 = 1e-2\n", 117 | "rho0 = 1.9\n", 118 | "lam = 0.01" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 194, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "def LADMPSAP(f, Din, lam, beta, maxBeta, maxIter, rho0, eps1, eps2, r, log, logbeta):\n", 128 | " step = 0\n", 129 | " D = Din.copy()\n", 130 | " p, n= D.shape\n", 131 | " E = np.zeros((p, n))\n", 132 | " Z = np.zeros((n, n))\n", 133 | " L = np.zeros((p, n))\n", 134 | " Y = np.zeros((n, n))\n", 135 | " M = np.zeros((n, n))\n", 136 | " l = np.zeros(n)\n", 137 | " dL = D - D@Z - E\n", 138 | " dLnorm = np.linalg.norm(dL)\n", 139 | " normD = np.linalg.norm(D)\n", 140 | " dM = Z - Y\n", 141 | " dMnorm = np.linalg.norm(dM)\n", 142 | " etaZ = normD**2*3.1\n", 143 | " crit1 = False\n", 144 | " crit2 = False\n", 145 | " while step < maxIter:\n", 146 | " if step % 1 == 0:\n", 147 | " value = f(Z, E, lam)\n", 148 | " print(step, value, 'gap', dLnorm, dMnorm, beta, crit1, crit2)\n", 149 | " log.append(value)\n", 150 | " logbeta.append(beta)\n", 151 | " Enew = update_E(Z, L, lam, beta)\n", 152 | " Znew = update_Z(Z, E, L, Y, M, l, beta, etaZ, r, n)\n", 153 | " Ynew = np.maximum(Z, 0)\n", 154 | " dE = np.linalg.norm(Enew-E)\n", 155 | " dZ = np.linalg.norm(Znew-Z)\n", 156 | " dY = np.linalg.norm(Ynew-Y)\n", 157 | " E = Enew\n", 158 | " Z = Znew\n", 159 | " Y = Ynew\n", 160 | " dl = np.sum(Z, axis=0) - np.ones((1, n))\n", 161 | " dL = -D@Z - E + D\n", 162 | " dLnorm = np.linalg.norm(dL)\n", 163 | " dM = Z - Y\n", 164 | " dMnorm = np.linalg.norm(dM)\n", 165 | " L = L + beta * dL\n", 166 | " l = l + beta * dl\n", 167 | " M = M + beta * dM\n", 168 | " crit1 = dLnorm/normD < eps1\n", 169 | " beta, crit2 = update_beta(\n", 170 | " beta, maxBeta, rho0, etaZ, eps2, dE, dZ, dY, normD)\n", 171 | " if crit1 and crit2:\n", 172 | " print(\"Converged at step\", step)\n", 173 | " value = f(Z, E, lam)\n", 174 | " print(step, value, 'gap', dLnorm, dMnorm)\n", 175 | " break\n", 176 | " step += 1\n", 177 | " return Z, E, L" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 195, 183 | "metadata": { 184 | "tags": [] 185 | }, 186 | "outputs": [ 187 | { 188 | "name": "stdout", 189 | "output_type": "stream", 190 | "text": [ 191 | "0 0.0 gap 245.93032155787887 0.0 0.0001 False False\n", 192 | "1 0.0 gap 245.93032155787887 0.0 0.00019 False True\n", 193 | "2 0.22769675455313595 gap 245.77474815371852 0.22769675455313335 0.000361 False True\n", 194 | "3 0.5381739645331823 gap 245.68596189636415 0.6632821764254848 0.0006858999999999999 False True\n", 195 | "4 40.3412898818391 gap 28.799737493849662 1.1122867082923429 0.0013032099999999997 False True\n", 196 | "5 42.40585673296325 gap 15.73621573588862 1.1123105110910667 0.0024760989999999994 False True\n", 197 | "6 42.98582366505853 gap 8.134669000274894 0.38626172187602087 0.004704588099999999 False True\n", 198 | "Converged at step 6\n", 199 | "6 43.09421131078367 gap 1.6853910063652453 0.22618233327199694\n" 200 | ] 201 | } 202 | ], 203 | "source": [ 204 | "log = []\n", 205 | "logbeta = []\n", 206 | "Z, E, L = LADMPSAP(f, D, lam, beta, maxBeta, maxIter, rho0, eps1, eps2, r, log, logbeta)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 196, 212 | "metadata": {}, 213 | "outputs": [ 214 | { 215 | "data": { 216 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAbT0lEQVR4nO3de3Bc93ne8e+7u7iQBECAIESBFwkUCcpileiGiKZEUbYcJbKt2E7ruFFdV7E9VT3jduxJOrHddJym08zYzdSOp5Mmo1hO5NSRnPoy9nhct64icUWblkzSupLULkmR5p1nwQsuJAHs7ts/9hACSZAEgd09u3uezwwGu2fP7r4rgQ8O3j37e83dERGR+EhEXYCIiFSXgl9EJGYU/CIiMaPgFxGJGQW/iEjMpKIuYCYWL17sfX19UZchIlJXtm3blnP3nou310Xw9/X1sXXr1qjLEBGpK2a2f7rtavWIiMSMgl9EJGYU/CIiMaPgFxGJGQW/iEjMKPhFRGJGwS8iEjN1cR6/iEgjcXfOThQYGcszOlZg5FyekbF8eD3PcPh9dCzPx+5dSdeC5rI+v4JfRGQGJsP6XH4ysIfHJkrBPTbBSBjgo2MXhvj5y+dvOx/qxRmMQkkYvO+2pQp+EZGZcnfOjBcuCNzJEB7PhyFeCu7RsQLDYTiPjucnL08N8ZmGdVtLqvTVmmJBePn6jtbJy1Nva29JTbM9SVtLinlNScys7P9dFPwiUtdOjo6z48gQOw4PsePIELuODnPqzPhkuM8krJMJY0Fz8sJAbk3Ru7CVtpa3rk8N6OnCur2lidamREXCupwU/CJSF4pF58DJM5MBf/77kdPnJve5vqOVt/W2c+vSDtpapwnoy4R4PYR1OSn4RaTmnJsokDk2PBnuO48MsfPIMCNjeaB0hL6qZwHrVi5i7dIO1vYu5JbedrrbWiKuvD4o+EUkUoMjY+w8MsyOI6cng35PMEoh7NEsaE6ydmkH/+zOZdzS28HapR2sWdJOa1My4srrl4JfRKqiWHT2nyi1anYeeatdc3TorVZN78JW1vZ28Jv/5HrWhiG/oms+iUR82jDVoOAXkbI7N1HgjaPDF77pemSI0fECUGrV9F/XxvpV3ZMBf0tvB4vKfNqiTE/BLyJzkhsZKx3BT3nTdU8wMnk2TVtLirW9HfzOwIrJkF99XZtaNRFS8IvIjBSLzr7B0QuO4nccHuL48NjkPss653FLbzvvvvX6yTddl3fNU6umxij4ReQSZ8cLvDF5Vk3pTdddR4c5E7ZqUglj9XVtbOhfPHkUv7a3g875atXUg4oHv5klga3AIXd/2MxWAk8D3cA24CPuPl7pOkTk8rLHhnlm1/HJI/m9U1o17a2lVs2HBlZMBnz/kjZaUmrV1KtqHPF/CtgJdITXvwh82d2fNrO/Aj4O/GUV6hCRy/g3f7eNvblRlnXOY+3SDt77K72TIb+8a16sPtwUBxUNfjNbDrwX+FPg96300/MA8C/CXZ4E/hMKfpHI/HLwDHtzo3z+4bV8bMPKqMuRKqj0evx/DvwhUAyvdwOn3D0fXj8ILJvujmb2mJltNbOtQRBUuEyR+NqULf37esfNPRFXItVSseA3s4eB4+6+bTb3d/fH3X3A3Qd6evQDKVIp6UzA8q55rFy8IOpSpEoq2eq5F3ifmb0HaKXU4/8K0GlmqfCofzlwqII1iMgVTBSKbNkzyG/dtlR9/Bip2BG/u3/O3Ze7ex/wu8A/uvuHgWeBD4a7PQp8r1I1iMiVbd9/kpGxPPevWRx1KVJFUczc/QylN3p3U+r5PxFBDSICpLMByYRxz2oFf5xU5QNc7v4c8Fx4eS9wdzWeV0SuLJ3JcceKTjpam6IuRaooiiN+EakBgyNjvHb4NBvX6OSJuFHwi8TU5t053FHwx5CCXySmNmUCOuc38SvLFkZdilSZgl8khtyd57M5NqxeTFIrZ8aOgl8khnYdHSYYHlObJ6YU/CIxlM6UlmnY2K/gjyMFv0gMpbMBNy9p5/qFrVGXIhFQ8IvEzJnxPD9/8yQb9Wnd2FLwi8TMC3tPMF4oqr8fYwp+kZjZlAlobUrwa32Loi5FIqLgF4mZdDZg3cpuWps0OjGuFPwiMXLw5Bn2BqNq88Scgl8kRtKZHICWYY45Bb9IjKQzAUsXtrKqpy3qUiRCCn6RmMgXivxkT46Na3o0bSvmFPwiMfHSgVMMn8tznz6tG3sKfpGYSGcCEgYbNG0r9hT8IjGxKZvjthWdLJyvaVtxp+AXiYGTo+O8cvCUFmUTQMEvEguatiVTKfhFYiCdCehoTXHbck3bEgW/SMNzd9LZgA39i0kl9U9eFPwiDS9zbIRjQ2Pq78skBb9Ig5uctqX+voQU/CINLp0NWH1dG0s750VditQIBb9IAzs3UeCFN0+ozSMXUPCLNLAX3jzBeL6oMYtyAQW/SANLZwKaUwnWreyOuhSpIQp+kQaWzgSsW7mIec2atiVvUfCLNKjDp86SPT6i/r5cQsEv0qCez+o0Tpmegl+kQaUzOa7vaGXNEk3bkgsp+EUaUKHobN6d477+xZq2JZdQ8Is0oJcPnuL02Qm1eWRaCn6RBpTOBJimbcllKPhFGlA6E/CryxbStaA56lKkBin4RRrM6TMTvHTglNo8clkVC34zazWzF83sZTN73cz+JNy+0sxeMLPdZvZNM9MhiUgZ/WRPjqKmbckVVPKIfwx4wN1vA24HHjKztwNfBL7s7quBk8DHK1iDSOykMwHtLSluX9EZdSlSoyoW/F4yEl5tCr8ceAD4Vrj9SeADlapBJG7cnXQm4J7V3TRp2pZcRkV/MswsaWYvAceBHwN7gFPung93OQgsu8x9HzOzrWa2NQiCSpYp0jD2BCMcPn1ObR65oooGv7sX3P12YDlwN/C2a7jv4+4+4O4DPT36IRaZiU2ZHIDW55Erqsrfgu5+CngWWA90mlkqvGk5cKgaNYjEQToTcNPiBaxYND/qUqSGVfKsnh4z6wwvzwMeBHZS+gXwwXC3R4HvVaoGkTgpTdsaVJtHrip19V1mrRd40sySlH7B/IO7/8DMdgBPm9l/AX4BPFHBGkRi4+f7TnBuQtO25OoqFvzu/gpwxzTb91Lq94tIGT2fzdGcTPD2mzRtS65M53uJNIh0JmCgr4v5zZX8Q14agYJfpAEcGzrHrqPD6u/LjCj4RRpAOhNO29JpnDIDCn6RBpDO5uhpb+GW3vaoS5E6oOAXqXOForM5G2jalsyYgl+kzr126DQnz0xwv/r7MkMKfpE6d76/r2lbMlMKfpE6l84G3Lqsg+62lqhLkTqh4BepY0PnJtj+y1M6m0euiYJfpI79dPcghaLr/H25Jgp+kTqWzgYsaE5y5w1dUZcidUTBL1Knzk/bWr9qMc0p/VOWmdNPi0idejM3ysGTZ7lfq3HKNVLwi9SpyWUa1N+Xa6TgF6lT6WyOG7vnc2P3gqhLkTqj4BepQ2P5Alv2DOo0TpkVBb9IHdq27yRnJwpq88isKPhF6tCmbEAqYaxfpWlbcu0U/CJ1KJ3JcdeNXbS1aNqWXDsFv0idOT58jp1HhtTmkVlT8IvUmc3ZHICWYZZZU/CL1Jl0JqB7QTNrezuiLkXq1IyD38zmV7IQEbm6YtF5Ppvjvv7FJBKatiWzc9XgN7N7zGwHsCu8fpuZ/Y+KVyYil9hxZIjB0XH192VOZnLE/2XgN4FBAHd/GdhYyaJEZHqbwmUa7tMHt2QOZtTqcfcDF20qVKAWEbmKdCZgbW8HPe2atiWzN5PgP2Bm9wBuZk1m9u+BnRWuS0QuMjKWZ9v+k9yn1ThljmYS/J8APgksAw4Bt4fXRaSKtuwZJF907lebR+boqh/7c/cc8OEq1CIiV5DOBMxrSnJXn6ZtydxcNfjN7G8Av3i7u3+sIhWJyLTS2YD1q7ppSSWjLkXq3EwW+vjBlMutwG8DhytTjohMZ//gKPsHz/DRe/qiLkUawExaPd+eet3MngI2V6wiEbmEpm1JOc1myYZ+4LpyFyIil7cpk2N51zxWLta0LZm7mfT4hyn1+C38fhT4TIXrEpHQeL7Ilj053n/HMsy0TIPM3UxaPe3VKEREprf9lycZHS9ozKKUzWWD38zuvNId3X17+csRkYulMwHJhHHPak3bkvK40hH/f7vCbQ48UOZaRGQa6WzAnTd00tHaFHUp0iAuG/zu/s65PLCZrQC+Diyh9IvicXf/ipktAr4J9AH7gA+5+8m5PJdIo8qNjPHaoSH+4ME1UZciDWRGAzvN7FZgLaXz+AFw969f5W554A/cfbuZtQPbzOzHwO8Bz7j7F8zss8Bn0ZvFItM6P21Lp3FKOc3krJ4/Bt5BKfh/CLyb0nn8Vwx+dz8CHAkvD5vZTkrr/bw/fDyAJ4HnUPCLTCudCeia38StyxZGXYo0kJmcx/9B4F3AUXf/KHAbcE0/hWbWB9wBvAAsCX8pQOnU0CWXuc9jZrbVzLYGQXAtTyfSEIpFJ53NsaG/h6SmbUkZzST4z7l7EcibWQdwHFgx0ycwszbg28Cn3X1o6m3u7kyzDlB42+PuPuDuAz09+jNX4mfX0WFyI2Ns7NcyzFJelw1+M/sLM9sAvGhmncBfA9uA7cCWmTy4mTVRCv1vuPt3ws3HzKw3vL2X0i8SEblIOqtlGqQyrtTjzwB/BiwFRoGngAeBDnd/5WoPbKWPGD4B7HT3L0256fvAo8AXwu/fm13pIo0tnQl42/XtLOlovfrOItfgskf87v4Vd19Pab7uIPA14EfAb5tZ/wwe+17gI8ADZvZS+PUeSoH/oJllgV8Pr4vIFGfG82zdd1JH+1IRM1myYT/wReCLZnYHpV8AnweuuCi4u2+mtL7PdN51jXWKxMrP9g4yXihyn/r7UgFXfXPXzFJm9ltm9g3gfwNvAP+04pWJxFg6k6O1KcGv9S2KuhRpQFdaq+dB4BHgPcCLwNPAY+4+WqXaRGIrnQlYt7Kb1iZN25Lyu1Kr53PA31P69K2WVBCpkgMnzrA3N8qH335j1KVIg7rSWj1ahE0kAudP47x/jfr7UhmzmcAlIhWUzgQsXdjKqp62qEuRBqXgF6khE4UiP909yMY1PZq2JRWj4BepIS8dOMXwWF7n70tFKfhFakg6E5AwuHeV+vtSOQp+kRqSzgTcvqKThfM1bUsqR8EvUiNOjI7zyqHTavNIxSn4RWrE5t053LUap1Segl+kRqQzAQvnNXHb8s6oS5EGp+AXqQHuzvPZgA2rF2vallScgl+kBrxxbJhjQ2Ns1Kd1pQoU/CI1IJ3RtC2pHgW/SA1IZ3L0X9dG78J5UZciMaDgF4nY2fECL+47oaN9qRoFv0jEXnhzkPF8UcEvVaPgF4lYOpOjOZXgbk3bkipR8ItELJ0NWLdyEfOaNW1LqkPBLxKhw6fOsvv4CBv71eaR6lHwi0RIp3FKFBT8IhFKZwOu72hlzRJN25LqUfCLRCRfKLI5m+O+/sWatiVVpeAXicjLB08zdE7TtqT6FPwiEUlnAsxgw2qtzyPVpeAXiUg6G/CryzvpWtAcdSkSMwp+kQicPjPBywdOcX+/jval+hT8IhHYvDtHUdO2JCIKfpEIpDMB7a0pbl+haVtSfQp+kSpzd9LZgHtXLSaV1D9BqT791IlU2e7jIxw5fU5tHomMgl+kyjZNLtOgN3YlGgp+kSpLZ3Pc1LOA5V3zoy5FYkrBL1JF5yYKvLB3UKtxSqQU/CJV9OKbJxjLF7lf/X2JkIJfpIqezwY0JxOsu0nTtiQ6FQt+M/uamR03s9embFtkZj82s2z4vatSzy9Si9KZHAN9XcxvTkVdisRYJY/4/xZ46KJtnwWecfd+4JnwukgsHD19jjeODes0TolcxYLf3dPAiYs2vx94Mrz8JPCBSj2/SK1JZ8PTOPXGrkSs2j3+Je5+JLx8FFhyuR3N7DEz22pmW4MgqE51IhWUzgT0tLdwS2971KVIzEX25q67O+BXuP1xdx9w94GeHh0hSX0rFJ3NuzVtS2pDtYP/mJn1AoTfj1f5+UUi8eqh05w6M6HTOKUmVDv4vw88Gl5+FPhelZ9fJBKatiW1pJKncz4FbAFuNrODZvZx4AvAg2aWBX49vC7S8NKZgFuXLqS7rSXqUkSo2MnE7v7IZW56V6WeU6QWDZ2b4BcHTvGJ+2+KuhQRQJ/cFam4n+7OUSi6TuOUmqHgF6mwTZkcbS0p7rxRH1SX2qDgF6kgdyedCVi/qpsmTduSGqGfRJEK2psb5dCps1qmQWqKgl+kgtLhtK371d+XGqLgF6mgdCagr3s+N3Rr2pbUDgW/SIWM5Qv8bO8JtXmk5ij4RSpk676TnJ0o6DROqTkKfpEKSWcCmpLG+lXdUZcicgEFv0iFbMoE3HVjFwtaNG1LaouCX6QCjg+dY9fRYe5Tm0dqkIJfpAKez+YAtAyz1CQFv0gFpLMB3QuaWdvbEXUpIpdQ8IuUWbHoPJ8tTdtKJDRtS2qPgl+kzF4/PMSJ0XGdvy81S8EvUmbpbGmZBr2xK7VKwS9SZpsyAWt7O+hp17QtqU0KfpEyGj43wfb9J9XmkZqm4Bcpoy17BskXnY1rNFRdapeCX6SM0tmA+c1JBm5cFHUpIpel4Bcpo3Qmx/qbumlO6Z+W1C79dIqUyb7cKL88cUb9fal5Cn6RMjl/GqeCX2qdgl+kTNKZgBWL5tGnaVtS4xT8ImUwni+yZc8gG/t7MNMyDVLbFPwiZbBt/0lGxwtq80hdUPCLlEE6G5BKGPdo2pbUAQW/SBmkMwF33tBFe2tT1KWIXJWCX2SOguExXj88pE/rSt1Q8IvM0ebdWo1T6ouCX2SO0pkcXfObuHXZwqhLEZkRBb/IHJSmbQVs6O8hqWlbUicU/CJzsPPoELmRcTb2q78v9UPBLzIH6UwO0DINUl8U/CJzkM4EvO36dpZ0tEZdisiMKfhFZml0LM/W/Sd0tC91JxV1ASK1plB0xvNFxvNFxgoFxiaKjBeKk9vGC0XGJoq8cugUEwVno07jlDoTSfCb2UPAV4Ak8FV3/0IUdUhtcHcmCh4GauGCkB0Lv84Hbmlb4YIQvmS/fJHxaQJ7bDLMz28rXPLY4/ki+aLPuPaO1hQDfV0V/K8jUn5VD34zSwJ/ATwIHAR+bmbfd/cd1a6lkbg7haKTL5YCNF9w8oUiE0VnIl8kXywyUXAmCqXv+ULxkn0nL0/ZN19wJopFJvJvbc8XiqXHKZ6//Na++WKR8fOPH4Z5vhg+zpTnHi9cGNbl0pxM0JJK0Dz1K5mgpan0vTmVYGFz0+S2luR0+yUvuH9L+NWcnLqttM/Sha20NiXLVr9INURxxH83sNvd9wKY2dPA+4GyB/8fffdVXnzzRLkfNhJOqQUxGcZh+OaLPhmwlWYGTckETQmjKZUglUjQlDRSSQu3J966nDRSiQQdzU00JUr7pJKl8Ewl7JIAnRqsLReF9iX7TRPm57dpSWSRq4si+JcBB6ZcPwisu3gnM3sMeAzghhtumNUTLe2cR/+StlndtxYlE2HoJt8K2FQYwqVwTUwJ3fPbLw3jVNJKAZycerl02+TlpNGUSIQBX7q/PqAk0hhq9s1dd38ceBxgYGBg5k3XKT75ztVlrUlEpBFEcTrnIWDFlOvLw20iIlIFUQT/z4F+M1tpZs3A7wLfj6AOEZFYqnqrx93zZvZvgf9D6XTOr7n769WuQ0QkriLp8bv7D4EfRvHcIiJxpyUbRERiRsEvIhIzCn4RkZhR8IuIxIy5z+qzUVVlZgGwf5Z3XwzkylhOlBrltTTK6wC9llrVKK9lrq/jRne/ZPnYugj+uTCzre4+EHUd5dAor6VRXgfotdSqRnktlXodavWIiMSMgl9EJGbiEPyPR11AGTXKa2mU1wF6LbWqUV5LRV5Hw/f4RUTkQnE44hcRkSkU/CIiMdPQwW9mD5nZG2a228w+G3U9s2VmXzOz42b2WtS1zIWZrTCzZ81sh5m9bmafirqm2TKzVjN70cxeDl/Ln0Rd01yYWdLMfmFmP4i6lrkws31m9qqZvWRmW6OuZy7MrNPMvmVmu8xsp5mtL9tjN2qPPxzqnmHKUHfgkXoc6m5mG4ER4OvufmvU9cyWmfUCve6+3czagW3AB+r0/4kBC9x9xMyagM3Ap9z9ZxGXNitm9vvAANDh7g9HXc9smdk+YMDd6/7DW2b2JPC8u381nF0y391PleOxG/mIf3Kou7uPA+eHutcdd08DdT813t2PuPv28PIwsJPSDOa64yUj4dWm8Ksuj6LMbDnwXuCrUdciJWa2ENgIPAHg7uPlCn1o7OCfbqh7XYZMIzKzPuAO4IVoK5m9sD3yEnAc+LG71+tr+XPgD4Fi1IWUgQP/18y2mdljURczByuBAPibsAX3VTNbUK4Hb+TglxplZm3At4FPu/tQ1PXMlrsX3P12SnOj7zazumvDmdnDwHF33xZ1LWWywd3vBN4NfDJsk9ajFHAn8JfufgcwCpTtfcpGDn4Nda9BYT/828A33P07UddTDuGf4M8CD0VdyyzcC7wv7I0/DTxgZv8z2pJmz90Phd+PA9+l1PKtRweBg1P+ivwWpV8EZdHIwa+h7jUmfEP0CWCnu38p6nrmwsx6zKwzvDyP0kkEu6Kt6tq5++fcfbm791H6N/KP7v4vIy5rVsxsQXjSAGFb5DeAujwTzt2PAgfM7OZw07uAsp0EEcnM3WpopKHuZvYU8A5gsZkdBP7Y3Z+ItqpZuRf4CPBq2BsH+A/hDOZ60ws8GZ49lgD+wd3r+lTIBrAE+G7p+IIU8Pfu/qNoS5qTfwd8Izxw3Qt8tFwP3LCnc4qIyPQaudUjIiLTUPCLiMSMgl9EJGYU/CIiMaPgFxGJGQW/yBRm9kfhapuvhCs8rjOzT5vZ/KhrEykXnc4pEgqXvf0S8A53HzOzxUAz8FMaZMVHEdARv8hUvUDO3ccAwqD/ILAUeNbMngUws98wsy1mtt3M/le49tD5teD/a7ge/Itmtjrc/jtm9lq4dn86mpcm8hYd8YuEwgDfDMwH/h/wTXffNHWN9/CvgO8A73b3UTP7DNDi7v853O+v3f1PzexfAR9y94fN7FXgIXc/ZGad5VxeV2Q2dMQvEgrX178LeIzSkrjfNLPfu2i3twNrgZ+Ey048Ctw45fanpnw/PzHpJ8Dfmtm/prR8iEikGnatHpHZcPcC8BzwXHik/uhFuxiltfcfudxDXHzZ3T9hZusoDTvZZmZ3uftgeSsXmTkd8YuEzOxmM+ufsul2YD8wDLSH234G3Dulf7/AzNZMuc8/n/J9S7jPKnd/wd0/T+kvianLhYtUnY74Rd7SBvz3cLnlPLCbUtvnEeBHZnbY3d8Ztn+eMrOW8H7/kdJ8Z4AuM3sFGAvvB/Bn4S8UA54BXq7KqxG5DL25K1ImjTToWxqbWj0iIjGjI34RkZjREb+ISMwo+EVEYkbBLyISMwp+EZGYUfCLiMTM/weppJ8xNXwgigAAAABJRU5ErkJggg==", 217 | "text/plain": [ 218 | "
" 219 | ] 220 | }, 221 | "metadata": { 222 | "needs_background": "light" 223 | }, 224 | "output_type": "display_data" 225 | } 226 | ], 227 | "source": [ 228 | "plt.plot(log)\n", 229 | "plt.xlabel('Steps')\n", 230 | "plt.ylabel('Value')\n", 231 | "plt.savefig('31.png', dpi=200)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 197, 237 | "metadata": {}, 238 | "outputs": [ 239 | { 240 | "data": { 241 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXwV9b3/8deHbJCwJ2EPJEDAghbUCG7gVgV/2lJvbcXbWm21tre21dbWW3uX3tpfb+tea7WtFatSW7Rab7m2FVBQcEODG7JkYQdREnYCZP3cP85IIwY4gTOZcw7v5+ORR86Z+c7MZyLmnZnvnO/X3B0REZF4dYq6ABERSS0KDhERaRcFh4iItIuCQ0RE2kXBISIi7ZIZdQEdoaCgwIuLi6MuQ0QkZSxatKjW3QvbWndUBEdxcTHl5eVRlyEikjLMbM2B1ulWlYiItIuCQ0RE2kXBISIi7aLgEBGRdlFwiIhIuyg4RESkXRQcIiLSLgoOEZE0NL+yhgdfXEVjc0vC963gEBFJM80tzk/+uowHX1odyv4VHCIiaeYvb26g4v2dXH/eSLIyEv9rXsEhIpJGGppauGNOJaMHdOeC4/qHcgwFh4hIGvnjq2tZv3UP35s0kk6dLJRjKDhERNJEXX0Td8+tYnxJb84Y0ebAtgmh4BARSRMPvLCK2l0N3DD5GMzCudoABYeISFrYWtfAffNXcu6ovpw4pFeox1JwiIikgV89v4JdDU1897yRoR9LwSEikuI2bt/Dgy+t5qLjBzKyX7fQj6fgEBFJcXc9U4W78+1PjOiQ4yk4RERS2IqaXfxp0Xo+P34IRb1zO+SYCg4RkRR2x+xKcjI78Y2zh3fYMRUcIiIp6u312/jr4o1cdXoJBV1zOuy4Cg4RkRR166wKeuVmcdXEoR16XAWHiEgKeqm6lgVVtVxz1nC6d87q0GMrOEREUoy7c/OsCvr36MwXTh7S4cdXcIiIpJhZS97nrXXbuO4TpXTOyujw4ys4RERSSHOLc9vsCoYW5vGZEwZFUoOCQ0Qkhfz59fVUb9rF984bSWYIkzTFQ8EhIpIi9jY28/Nnqvj4oB5MPrZfZHUoOEREUsQjC9eyYdsebpgU7rDph6LgEBFJAbvqm7hnXjWnDc/n9NKCSGtRcIiIpID7F6xkS10DN0w6JupSFBwiIslu8656fjt/JZNH92NMUc+oy1FwiIgku3vmrWBPYzPfndQxw6YfioJDRCSJbdi2h9+/soaLTxzE8D7hT9IUDwWHiEgS+/mcSjC4toMmaYpHqMFhZpPNrMLMqs3s+22szzGzR4P1C82suNW6G4PlFWY2ab/tMszsDTN7Ksz6RUSiVPX+Tp54fT2XnTyEgT27RF3OPqEFh5llAPcA5wOjgEvNbNR+za4Etrr7cOBO4OZg21HAVGA0MBm4N9jfB64FloVVu4hIMrhtdgW52Zl8/cxhUZfyIWFecYwDqt19pbs3ADOAKfu1mQI8FLx+HDjHYp9qmQLMcPd6d18FVAf7w8wGARcA94dYu4hIpN5ct41ZS97nKxOGkt+BkzTFI8zgGAisa/V+fbCszTbu3gRsB/IPse3PgRuAloMd3MyuNrNyMyuvqak53HMQEelw7s7Nf19Ofl42V04oibqcj0ipznEzuxDY5O6LDtXW3e9z9zJ3LyssLOyA6kREEuOF6lpeXrmZa84aTteczKjL+Ygwg2MDUNTq/aBgWZttzCwT6AFsPsi2pwGfMrPVxG59nW1mvw+jeBGRKLg7tzxdwcCeXfj8yYOjLqdNYQbHa0CpmZWYWTaxzu6Z+7WZCVwevL4YmOvuHiyfGjx1VQKUAq+6+43uPsjdi4P9zXX3L4R4DiIiHerv77zH4g3b+fa5I8jJ7PhJmuIR2jWQuzeZ2TeAWUAG8IC7LzGzm4Byd58JTAOmm1k1sIVYGBC0ewxYCjQB17h7c1i1iogkg6bmFm6bVUFpn65cdPz+XcLJw2J/4Ke3srIyLy8vj7oMEZGDmvHqWr7/58X85rITmTQ6uvk2AMxskbuXtbUupTrHRUTS1d7GZu56toqxRT05b1TfqMs5KAWHiEgSmP7yGjZu38u/To52kqZ4KDhERCK2Y28j9zxXzcQRhZwyLD/qcg5JwSEiErHfzl/Jtt2N3DBpZNSlxEXBISISoZqd9Ux7YRUXfLw/xw7sEXU5cVFwiIhE6J551dQ3tXD9uckzbPqhKDhERCKybstuHlm4hs+VFTG0sGvU5cRNwSEiEpE751TSyYxrzymNupR2UXCIiESg4r2dPPnmBq44tZh+PTpHXU67KDhERCJw66wKuuZk8i9JNklTPBQcIiIdbNGaLTyz7H2+OnEoPXOzoy6n3RQcIiIdyN25+ekKCrrm8KXTkm+SpngoOEREOtDzlTW8umoL3zpnOHlJOElTPBQcIiIdpKUlNklTUe8uTD0pOSdpioeCQ0Skgzy1eCNLN+7gO+eOIDszdX/9pm7lIiIppLG5hdtnV3BMv258akzyTtIUDwWHiEgHeKx8HWs27+Z7k0aS0Sm5h00/FAWHiEjI9jQ0c9czVZQN6cXZx/SJupwjpuAQEQnZgy+tZtPOem5IgUma4qHgEBEJ0fbdjfzquWrOGlnIuJLeUZeTEAoOEZEQ/Wb+CnbsbeJ7k46JupSEUXCIiIRk0469PPDiKqaMHcCoAd2jLidhFBwiIiH5xdwqmpqd76TQJE3xUHCIiIRgzeY6Zry6jqnjihiSnxd1OQml4BARCcEdcyrJzDC+dXZqTdIUDwWHiEiCLXl3O395812+fFoJfbqn1iRN8VBwiIgk2G2zKujeOZOvTky9SZrioeAQEUmgV1dtYV5FDf9y5nB65GZFXU4oFBwiIgni7tzy9HL6dMvhilOLoy4nNAoOEZEEmbt8E+VrtnLtJ0rpkp0RdTmhUXCIiCRAS4tz66wKivNz+VxZUdTlhErBISKSADPfepfl7+3kO+eNJCsjvX+1pvfZiYh0gIamFm6fU8Go/t258Lj+UZcTOgWHiMgRmvHaWtZt2cMNk0fSKcUnaYpHqMFhZpPNrMLMqs3s+22szzGzR4P1C82suNW6G4PlFWY2KVjW2cxeNbO3zGyJmf0ozPpFRA5ld0MTv3i2mnElvTljRGHU5XSI0ILDzDKAe4DzgVHApWY2ar9mVwJb3X04cCdwc7DtKGAqMBqYDNwb7K8eONvdxwBjgclmdnJY5yAicii/e3E1tbvq+dfJI9NikqZ4hHnFMQ6odveV7t4AzACm7NdmCvBQ8Ppx4ByL/eSnADPcvd7dVwHVwDiP2RW0zwq+PMRzEBE5oK11Dfz6uRV84mN9OXFIekzSFI8wg2MgsK7V+/XBsjbbuHsTsB3IP9i2ZpZhZm8Cm4A57r6wrYOb2dVmVm5m5TU1NQk4HRGRD/v18yvY1dDE9yaNjLqUDpVynePu3uzuY4FBwDgzO/YA7e5z9zJ3LyssPDruO4pIx3lv+14efGk1F40dyMh+3aIup0OFGRwbgNafghkULGuzjZllAj2AzfFs6+7bgHnE+kBERDrUXc9W0eLOt9NskqZ4hBkcrwGlZlZiZtnEOrtn7tdmJnB58PpiYK67e7B8avDUVQlQCrxqZoVm1hPAzLoA5wLLQzwHEZGPWFmzi8fK1/H58UMo6p0bdTkdLjOsHbt7k5l9A5gFZAAPuPsSM7sJKHf3mcA0YLqZVQNbiIULQbvHgKVAE3CNuzebWX/goeAJq07AY+7+VFjnICLSltvnVJKT2YlrzhoedSmRsNgf+OmtrKzMy8vLoy5DRNLAOxu2c+HdL/DNs4dz/Xnp2yluZovcvaytdSnXOS4iEqVbZlXQMzeLr0wcGnUpkVFwiIjE6aUVtcyvrOGaM4fTvXN6TtIUDwWHiEgcYpM0VdC/R2cuO2VI1OVESsEhIhKHOUvf581127j2nFI6Z6XvJE3xUHCIiBxCczBJ09CCPC4+cVDU5UROwSEicghPvrGBqk27+O6kkWSm+SRN8dBPQETkIOqbmrlzTiXHDezB+cf2i7qcpKDgEBE5iD8sXMuGbbFJmo6WYdMPRcEhInIAu+qb+OXcak4dls/pwwuiLidpKDhERA5g2oJVbK5r4IbJx+hqoxUFh4hIG7bUNfDbBSuZPLofY4t6Rl1OUlFwiIi04d551exuaOK7k46+YdMPJe7Rcc3sAmJzgHf+YJm73xRGUSIiUdqwbQ8Pv7KGz5wwiOF9jq5JmuIR1xWHmf0auAT4JmDAZ4Gj+zP3IpK27nqmEhyuOwonaYpHvLeqTnX3LwJb3f1HwCmAfqIiknaqN+3i8UXr+cLJQxjYs0vU5SSleINjT/B9t5kNABqB/uGUJCISndtnV9AlK4NrzhoWdSlJK97geCqYsvVW4HVgNfDHsIoSEYnC85U1/P2d9/jKxKHkd82JupykFW/n+C3uXg88YWZPEesg3xteWSIiHWtBVQ1XP1zOMf26cdWEo3eSpnjEe8Xx8gcv3L3e3be3XiYiksrmV9Zw1UPllBTk8YevnEzXnLgfOD0qHfSnY2b9gIFAFzM7ntgTVQDdgdyQaxMRCd3zlTV85eFyhhV25ZGrxtM7LzvqkpLeoWJ1EnAFMAi4o9XyHcAPQqpJRKRDzKvYxFenL2J4EBq9FBpxOWhwuPtDwENm9hl3f6KDahIRCd285bHQKO0bC42euQqNeMXbx/GimU0zs78DmNkoM7syxLpERELz7LL3+er0RYzs102hcRjiDY7fAbOAAcH7SuC6UCoSEQnRM0vf52u/X8Qx/bvx+ysVGocj3uAocPfHgBYAd28CmkOrSkQkBLOXvMe/PLKIUf27M/3K8fTIzYq6pJQUb3DUmVk+4ABmdjKwPbSqREQS7Ol33uPrj7zO6AE9mH7VeHp0UWgcrngfVv4OMBMYamYvAoXAxaFVJSKSQE+/s5Fv/OENjhvUg4e+PI7unRUaRyLe4FgKPAnsBnYC/0Osn0NEJKn9bfFGvvnHNxgThEY3hcYRizc4Hib22Y3/Dt7/MzCd2PDqIiJJ6a9vb+RbM97g+KKePPjlcfpEeILE+1M81t1HtXo/z8yWhlGQiEgiPPX2u1w7401OGNyT331JoZFI8XaOvx50iANgZuOB8nBKEhE5MjPfioXGiYN7KTRCcKixqhYTe5IqC3jJzNYG74cAy8MvT0Skff7y5ga+/eiblBX35ndXnESeQiPhDvUTvbBDqhARSYD/eWMD33nsTcaV9OaBK04iN1uhEYZDjVW1pqMKERE5En9+fT3f/dNbjC/JZ9oVZQqNEMXbx3FYzGyymVWYWbWZfb+N9Tlm9miwfqGZFbdad2OwvMLMJgXLisxsnpktNbMlZnZtmPWLSGp4fNF6rv/TW5w8NF9XGh0gtOAwswzgHuB8YBRwqZmN2q/ZlcBWdx8O3AncHGw7CpgKjAYmA/cG+2sCrg+e8DoZuKaNfYrIUeRP5ev43uNvcdqwAqZdfhJdsjOiLinthXnFMQ6odveV7t4AzACm7NdmCvBQ8Ppx4Bwzs2D5jGC2wVVANTDO3Te6++sA7r4TWEZsoikROQo99to6bnjibU4fXsD9l5cpNDpImMExEFjX6v16PvpLfl+bYODE7UB+PNsGt7WOBxa2dXAzu9rMys2svKam5rBPQkSS04xX13LDE28zobSQ336xjM5ZCo2OEmofR1jMrCvwBHCdu+9oq4273+fuZe5eVlhY2LEFikio/rBwLd//82LOGFHIfZedqNDoYGEGxwagqNX7QcGyNtuYWSbQA9h8sG3NLItYaDzi7n8OpXIRSVqPLFzDD55czFkjC/mNQiMSYQbHa0CpmZWYWTaxzu6Z+7WZCVwevL4YmOvuHiyfGjx1VQKUAq8G/R/TgGXufgciclSZ/soa/u3Jdzj7mD78WqERmdCeWXP3JjP7BrGZAzOAB9x9iZndBJS7+0xiITDdzKqBLcTChaDdY8RG5W0CrnH3ZjM7HbgMWGxmbwaH+oG7/y2s8xCR5DD95dX8x1+WcM4xfbj3CyeQk6nQiIrF/sBPb2VlZV5erqG1RFLVQy+t5oczl/CJj/Xlns8fr9DoAGa2yN3L2lqnT8mISFL73Yur+NH/LuXcUX25559PIDszJZ/pSSsKDhFJWtNeWMWPn1rKpNF9uftShUayUHCISFK6f8FK/v9fl3H+sf34xaXHk5Wh0EgWCg4RSTq/nb+Sn/xtGf/vuH7cNVWhkWwUHCKSVH7z/Ap++vflXPDx/vz8krEKjSSk4BCRpPGr51Zw89PL+eSYAdz5uTFkKjSSkoJDRJLCPfOquXVWBZ8aM4A7FBpJTcEhIpH75dwqbptdyZSxA7j9swqNZKfgEJFI/eLZKu6YU8lFxw/kts+OIaOTRV2SHIKCQ0Qic9czVdz5TCX/dPxAblVopAwFh4hE4s45ldz1bBWfOWEQt1z8cYVGClFwiEiHcnfufKaKXzxbxWdPHMTPPqPQSDUKDhHpMO7OHXMquXtuNZeUFfHTfzqOTgqNlKPgEJEO4e7cNruCe+atYOpJRfz3RQqNVKXgEJHQuTu3zKrgV8+t4NJxg/nJp49VaKQwBYeIhMrd+dnTy/nN8yv5/PjB/HiKQiPVKThEJDTuzk//vpz75q/kspOHcNOU0cRmgJZUpuAQkVC4Oz/56zLuf2EVl58yhP/6lEIjXSg4RCTh3J0fP7WMB15cxRWnFvPDT45SaKQRBYeIJJS786P/XcqDL63mS6cV858XKjTSjYJDRBKmdWh8+bQS/uPCjyk00pCCQ0QSwt354cwlPPzyGq46vYR/u0Chka4UHCJyxFpanP+c+Q6/f2UtV08cyo3nH6PQSGMKDhE5InX1Tfz4qaXMeG0dXz1jKN+frNBIdwoOETksexub+cPCtdwzr5rNdQ18/cxhfG/SSIXGUUDBISLt0tTcwuOL1nPXs1Vs3L6XU4fl891JIzlhcK+oS5MOouAQkbi0tDhPLd7InXMqWVVbx9iintz22TGcNrwg6tKkgyk4ROSg3J1nl23ittkVLH9vJyP7duO3XyzjEx/ro9tSRykFh4gc0Esrarl1VgVvrN1GcX4ud00dyyc/PkCDFB7lFBwi8hFvrN3KbbMreLF6M/26d+an/3QcF584iKyMTlGXJklAwSEi+yx/bwe3z65kztL36Z2XzX9cOIrPjx9M56yMqEuTJKLgEBFW19Zx5zOVzHzrXbpmZ3L9uSP40ukldM3Rrwj5KP2rEDmKbdy+h188W81j5evIyjC+dsYwvjpxKD1zs6MuTZKYgkPkKLR5Vz33PreC6a+swd35wvjBXHPWcPp07xx1aZICQg0OM5sM3AVkAPe7+8/2W58DPAycCGwGLnH31cG6G4ErgWbgW+4+K1j+AHAhsMndjw2zfpF0s2NvI/fPX8m0F1axp7GZz5wwiG+dU0pR79yoS5MUElpwmFkGcA9wLrAeeM3MZrr70lbNrgS2uvtwM5sK3AxcYmajgKnAaGAA8IyZjXD3ZuBB4JfEAkdE4rCnoZkHX1rNr59fwfY9jVxwXH++fe4IhvfpGnVpkoLCvOIYB1S7+0oAM5sBTAFaB8cU4L+C148Dv7TYJ4qmADPcvR5YZWbVwf5edvf5ZlYcYt0iaaOhqYUZr63l7rnV1Oys56yRhVx/3kiOHdgj6tIkhYUZHAOBda3erwfGH6iNuzeZ2XYgP1j+yn7bDmzPwc3sauBqgMGDB7ercJFU19TcwpNvbOCuZ6tYv3UP44p7c+/nT+Ck4t5RlyZpIG07x939PuA+gLKyMo+4HJEO0dLiPL3kPW6fXcGKmjqOG9iDn1x0HBNLCzQ8iCRMmMGxAShq9X5QsKytNuvNLBPoQayTPJ5tRSTg7jxXWcPtsyt4Z8MOhvfpyq+/cAKTRvdTYEjChRkcrwGlZlZC7Jf+VOCf92szE7gceBm4GJjr7m5mM4E/mNkdxDrHS4FXQ6xVJGUtXLmZ22ZX8NrqrRT17sIdnxvDlLEDydB4UhKS0IIj6LP4BjCL2OO4D7j7EjO7CSh395nANGB60Pm9hVi4ELR7jFhHehNwTfBEFWb2R+BMoMDM1gM/dPdpYZ2HSLJavH47t86uYH5lDX265fDjTx/LJWVFZGdqPCkJl7mn/+3/srIyLy8vj7oMkYSoen8nt8+u5Okl79EzN4uvnzmMy04upku2xpOSxDGzRe5e1ta6tO0cF0k367bs5s5nKvmfNzaQm53JteeUctWEErp1zoq6NDnKKDhEktz7O/Zy99wqHn1tHZ3MuGrCUL52xjB652k8KYmGgkMkSW2ta+DXz6/gwZdW09ziXHJSEd88u5R+PTSelERLwSGSZHbubWTaC6u4f8Eq6hqauGjsQK77xAgG52s8KUkOCg6RJLG3sZnpL6/h3ueq2bq7kUmj+3L9eSMZ0bdb1KWJfIiCQyRijc0tPPraOu6eW8X7O+qZUFrAd88byZiinlGXJtImBYdIRJpbnJlvbeDOOVWs3bKbE4f04ueXHM8pw/KjLk3koBQcIh1oa10DL66oZUFlLfOrati4fS+j+nfnd1ecxJkjCzU8iKQEBYdIiBqbW3hz3TbmV9Ywv6qWt9dvwx26dc7k9OEF/PsFAzj/2H500vAgkkIUHCIJtmZzHfOraplfWcPLKzazq76JTgZji3py7TmlTCgtZMygHmRmaGgQSU0KDpEjtHNvIy+t2MyCqhrmV9aydstuAAb27MInxwzgjBEFnDKsgB5d9AlvSQ8KDpF2am5xFm/YzvzKGhZU1fD62m00tzi52RmcOiyfK08vYeKIQorzc9VnIWlJwSESh3e37dl3RfFCdS3b9zRiBscO6MHXzhjKhNJCThjcSyPTylFBwSHSht0NTSxcuYX5VTUsqKqletMuAPp2z+HcUX2ZUFrA6cMLyO+aE3GlIh1PwSFCbMrVZe/tYEHQqV2+eisNzS3kZHZi/NB8pp5UxITSQkb07arbT3LUU3DIUWvTzr28UFXLguCrdlc9AMf068YVpxUzobSAk4p70zlL81yItKbgkKPG3sZmFq3Zuu8zFcs27gAgPy+b00sLmFBayITSAvp21+izIgej4JC05e5Ub9rF/KpaFlTV8MrKzextbCErwygb0psbJo9kYmkho/p31wfwRNpBwSFp5YMhPWKPytaycfteAIYW5jH1pMFMHFHA+JJ88nL0T1/kcOn/Hklpjc0tvLF2W/CobA1vb9iOO3TvnMnppQV8K7j9NKiX5rIQSRQFh6QUd2ftlt37+ik+GNIjo5Mxtqgn150zggkjChgzqCcZuv0kEgoFhySlrXUNrNpcx+ra2Neqzbv3vd5Z3wTAoF5d+NTYAUwsLeSUYfka0kOkgyg4JDI79jbGQiH4ah0Q2/c07mvXyWBgry4U5+dx0QkDKe3bjQnDCxiiIT1EIqHgkFDV1TfFQiG4elhVu3vf6811DfvamcGAHl0oLsjlwo/3p6Qgj+L8PIoL8ijq3YWcTH2WQiRZKDjkiO1paP5HMOy7vbSbVZvrqNlZ/6G2/bp3prggl/NG990XDCUFeQzunasP2omkCAWHxGVvYzPrtuzed/WwqnY3q2p3sbp2N+/t2PuhtgVdcxhakMdZIwtjwRAExJD8XHKz9U9OJNXp/2LZp6GphXVbd+/rd4hdRcTC4t3te3D/R9veedkU5+dy6vD8fcFQEoRDt87qpBZJZwqOo0xTcwvrt+5p84mlDdv20Nzyj3To0SWL4oI8TiruRXHBoA/1O+gJJpGjl4IjDTQ2t7BtdyNbdzewta4h9n13I1vqGtgWvK7dVc/azbtZu2U3Ta3CoVtOJsUFeYwp6smnxw6guCBv3+2lXnnZEZ6ViCQrBUeSqW9q3hcCsV/8/wiALXWNse9BGHwQEjv3Nh1wf12yMuidl02vvCyO6d+Nycf2o7ggj6FBQOTnZeuRVhFpFwVHiPY2Nh8wAGJXBR8OgK11DdQ1NB9wf11zMumZm0XvvGx65mZTkp9Lz9zsWDDkZtErL5teucFXXha9crP1pJKIJJyCIw7uzp7G5g8FwD9uCzW2GQBbdzeyp/HAIdCtc2bwCz6b/K7ZlPbpGoTARwOgd242PXKz9FkGEUkKCo4DcHcuvPsFNu+K3RpqaGo5YNseXT64CsiiX/fOHNOvO73zsj58NRCERK/cWLusDM1NLSKpScFxAGZGaZ+ujB7QqdUv/Q8HQK/cLHp0ySJTISAiR5FQg8PMJgN3ARnA/e7+s/3W5wAPAycCm4FL3H11sO5G4EqgGfiWu8+KZ5+J9POpx4e1axGRlBXan8pmlgHcA5wPjAIuNbNR+zW7Etjq7sOBO4Gbg21HAVOB0cBk4F4zy4hznyIiEqIw77GMA6rdfaW7NwAzgCn7tZkCPBS8fhw4x2LPhk4BZrh7vbuvAqqD/cWzTxERCVGYwTEQWNfq/fpgWZtt3L0J2A7kH2TbePYJgJldbWblZlZeU1NzBKchIiKtpW2vrrvf5+5l7l5WWFgYdTkiImkjzODYABS1ej8oWNZmGzPLBHoQ6yQ/0Lbx7FNEREIUZnC8BpSaWYmZZRPr7J65X5uZwOXB64uBue7uwfKpZpZjZiVAKfBqnPsUEZEQhfY4rrs3mdk3gFnEHp19wN2XmNlNQLm7zwSmAdPNrBrYQiwICNo9BiwFmoBr3L0ZoK19hnUOIiLyUeatJ1lIU2VlZV5eXh51GSIiKcPMFrl7WZvrjobgMLMaYM1hbl4A1CawnCily7mky3mAziUZpct5wJGdyxB3b/PJoqMiOI6EmZUfKHVTTbqcS7qcB+hcklG6nAeEdy5p+ziuiIiEQ8EhIiLtouA4tPuiLiCB0uVc0uU8QOeSjNLlPCCkc1Efh4iItIuuOEREpF0UHCIi0i4KjgMws8lmVmFm1Wb2/ajrOVxm9oCZbTKzd6Ku5UiZWZGZzTOzpWa2xMyujbqmw2Vmnc3sVTN7KziXH0Vd05EI5st5w8yeirqWI2Fmq81ssZm9aWYp/alhM+tpZo+b2XIzW2ZmpyRs3+rj+KhgwqhK4FxiQ7e/Blzq7ksjLewwmNlEYBfwsLsfG3U9R8LM+gP93f11M+sGLAI+naL/XQzIc/ddZsI7iQsAAAPtSURBVJYFvABc6+6vRFzaYTGz7wBlQHd3vzDqeg6Xma0Gytw95T8AaGYPAQvc/f5gbL9cd9+WiH3riqNtaTNhlLvPJzYOWMpz943u/nrweiewjAPMx5LsPGZX8DYr+ErJv+LMbBBwAXB/1LVIjJn1ACYSGw8Qd29IVGiAguNA4p4wSqJhZsXA8cDCaCs5fMHtnTeBTcAcd0/Vc/k5cAPQEnUhCeDAbDNbZGZXR13MESgBaoDfBbcQ7zezvETtXMEhKcfMugJPANe5+46o6zlc7t7s7mOJzSszzsxS7laimV0IbHL3RVHXkiCnu/sJwPnANcGt3lSUCZwA/MrdjwfqgIT11So42qYJo5JU0B/wBPCIu/856noSIbiFMA+YHHUth+E04FNB38AM4Gwz+320JR0+d98QfN8EPEnstnUqWg+sb3UV+zixIEkIBUfbNGFUEgo6lKcBy9z9jqjrORJmVmhmPYPXXYg9iLE82qraz91vdPdB7l5M7P+Tue7+hYjLOixmlhc8dEFwW+c8ICWfRnT394B1ZjYyWHQOsfmNEiK0iZxS2YEmoYq4rMNiZn8EzgQKzGw98EN3nxZtVYftNOAyYHHQNwDwA3f/W4Q1Ha7+wEPBE3ydgMfcPaUfZU0DfYEnY3+fkAn8wd2fjrakI/JN4JHgj9+VwJcStWM9jisiIu2iW1UiItIuCg4REWkXBYeIiLSLgkNERNpFwSEiIu2i4BBJIDP7t2C027eDEVbHm9l1ZpYbdW0iiaLHcUUSJBi2+g7gTHevN7MCIBt4iTQZcVUEdMUhkkj9gVp3rwcIguJiYAAwz8zmAZjZeWb2spm9bmZ/Csbe+mAuiFuC+SBeNbPhwfLPmtk7wdwd86M5NZF/0BWHSIIEAfACkAs8Azzq7s+3nuMhuAr5M3C+u9eZ2b8COe5+U9Dut+7+EzP7IvA5d7/QzBYDk919g5n1TOTw2CKHQ1ccIgkSzK9xInA1sSGtHzWzK/ZrdjIwCngxGDblcmBIq/V/bPX9gxnbXgQeNLOvEBsCRyRSGqtKJIHcvRl4DnguuFK4fL8mRmzujUsPtIv9X7v718xsPLHJkhaZ2YnuvjmxlYvET1ccIgliZiPNrLTVorHAGmAn0C1Y9gpwWqv+izwzG9Fqm0tafX85aDPM3Re6+38Su5JpPeS/SIfTFYdI4nQF7g6GS28CqondtroUeNrM3nX3s4LbV380s5xgu38nNsc9QC8zexuoD7YDuDUIJAOeBd7qkLMROQB1joskidad6FHXInIwulUlIiLtoisOERFpF11xiIhIuyg4RESkXRQcIiLSLgoOERFpFwWHiIi0y/8BvRkm8VnA4TIAAAAASUVORK5CYII=", 242 | "text/plain": [ 243 | "
" 244 | ] 245 | }, 246 | "metadata": { 247 | "needs_background": "light" 248 | }, 249 | "output_type": "display_data" 250 | } 251 | ], 252 | "source": [ 253 | "plt.plot(logbeta)\n", 254 | "plt.xlabel('Steps')\n", 255 | "plt.ylabel('beta')\n", 256 | "plt.savefig('32.png', dpi=200)" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [] 265 | } 266 | ], 267 | "metadata": { 268 | "interpreter": { 269 | "hash": "dc8f3ea9badafe9fc0f1b4faa22fba7f18b6863e43b73e6df2a27536033ffddf" 270 | }, 271 | "kernelspec": { 272 | "display_name": "Python 3.7.10 64-bit ('ml': conda)", 273 | "name": "python3" 274 | }, 275 | "language_info": { 276 | "name": "python", 277 | "version": "" 278 | }, 279 | "orig_nbformat": 2 280 | }, 281 | "nbformat": 4, 282 | "nbformat_minor": 2 283 | } -------------------------------------------------------------------------------- /20/1.py: -------------------------------------------------------------------------------- 1 | # Block Coordinate Descent 2 | # By ZincCat 3 | 4 | import numpy as np 5 | from matplotlib import pyplot as plt 6 | 7 | r = 5 8 | m = 200 9 | n = 300 10 | 11 | np.random.seed(1) 12 | 13 | U0 = np.random.normal(0, 1, (m, r)) 14 | VT0 = np.random.normal(0, 1, (r, n)) 15 | D = U0@VT0 16 | c = 0.1 # mask constant 17 | mask = np.random.uniform(0, 1, (m, n)) < c 18 | 19 | 20 | def bcd(D, maxIter, eps): 21 | U = np.random.normal(0, 1, (m, r)) 22 | VT = np.random.normal(0, 1, (r, n)) 23 | A = U@VT 24 | A[mask] = D[mask] 25 | value = np.linalg.norm(U@VT - A)**2/2 26 | oldvalue = 0 27 | count = 0 28 | for i in range(maxIter): 29 | value = np.linalg.norm(U@VT - A)**2/2 30 | if abs(value - oldvalue) < eps: 31 | count += 1 32 | if count >= 10: 33 | print("Converged at step", i, "Value:", value) 34 | break 35 | else: 36 | count = 0 37 | if i % 100 == 0: 38 | print("Step:", i, "Value:", value) 39 | U = A@np.linalg.pinv(VT) 40 | VT = np.linalg.pinv(U)@A 41 | A = U@VT 42 | A[mask] = D[mask] 43 | oldvalue = value 44 | return U, VT, A 45 | 46 | 47 | U, VT, A = bcd(D, 100000, 1e-20) 48 | print(np.linalg.norm(A-D)) 49 | -------------------------------------------------------------------------------- /20/2.py: -------------------------------------------------------------------------------- 1 | # pLADMPSAP 2 | # By ZincCat 3 | 4 | from scipy.optimize import minimize 5 | import numpy as np 6 | from matplotlib import pyplot as plt 7 | # from joblib import Parallel, delayed 8 | 9 | np.random.seed(19890817) 10 | 11 | n = 70 12 | s = 30 13 | x = np.random.normal(0, 1, (n, s)) 14 | y = np.random.choice([0, 1], s) 15 | w0 = np.random.normal(0, 1, n) 16 | 17 | 18 | def f(w): 19 | return np.sum(np.log(1+np.exp(-y*np.dot(w, x))))/s 20 | 21 | 22 | def gradient_f(w): 23 | temp = np.exp(-y*np.dot(w, x)) 24 | return np.sum(-temp*y/(1+temp)*x, axis=1)/s 25 | 26 | 27 | def descent(w, grad, value, mode='2', alpha=0.4, beta=0.8, eta=1e-7): 28 | # 梯度下降函数 29 | # 输入目前x取值, 梯度, 梯度的范数, 下降模式 30 | # 输出下降后x取值, 步长t 31 | # 下降模式为'2'时采用2范数, 为'inf'时采用无穷范数 32 | g = grad(w) 33 | grad_norm = np.linalg.norm(g) 34 | if grad_norm <= eta: 35 | return w, True 36 | normalized_grad = g/grad_norm 37 | t = 1.0 38 | if mode == '2': 39 | # l_2 norm 40 | while f(w - t*normalized_grad) > value - alpha*t*np.dot(g, normalized_grad): 41 | t *= beta 42 | w -= t*normalized_grad 43 | elif mode == 'inf': 44 | # l_infty norm 45 | while f(w - t*np.sign(normalized_grad)) > value - alpha*t*np.dot(g, np.sign(normalized_grad)): 46 | t *= beta 47 | w -= t*np.sign(normalized_grad) 48 | return w, False 49 | 50 | 51 | def gd(w0, eta=1e-5, maxIter=1000): 52 | w = w0.copy() 53 | timestep = 0 54 | while timestep <= maxIter: 55 | value = f(w) 56 | print("Iteration:", timestep, "Error", value) 57 | w, finished = descent(w, gradient_f, value, 58 | mode='2', eta=eta) # 此时使用2范数 59 | if finished: 60 | break 61 | timestep += 1 62 | return w 63 | 64 | 65 | def grad(i, w): 66 | temp = np.exp(-y[i]*np.dot(w, x[:, i])) 67 | return -temp*y[i]/(1+temp)*x[:, i]/s 68 | 69 | 70 | def pLADMAPSAP(w0, beta, eps1=1e-7, eps2=1e-5, maxBeta=1e1, maxIter=1e7, rho0=1.9): 71 | w = w0.copy() 72 | W = np.random.rand(s, n) 73 | newW = np.zeros_like(W) 74 | L = np.zeros_like(W) 75 | dL = np.zeros_like(W) 76 | Li = np.linalg.norm(x, axis=1)/4/s 77 | eta = s*np.ones_like(Li) 78 | tau = Li + beta*eta 79 | timestep = 0 80 | values = [] 81 | while timestep <= maxIter: 82 | if timestep % 1000 == 0: 83 | print(timestep, f(w)) 84 | values.append(f(w)) 85 | # naive multithreading, however, too slow when matrices are small 86 | # Parallel(n_jobs=-1, backend='threading')(delayed(update)(i, W, w, Lam, beta) 87 | # for i in range(s)) 88 | # sequential update 89 | for i in range(s): 90 | newW[i] = w - L[i]/tau - grad(i, W[i])/tau 91 | dL[i] = W[i]-w 92 | w = (np.sum(W, axis=0)+np.sum(L, axis=0)/tau)/s 93 | L += tau*dL 94 | crit = np.linalg.norm(dL) < eps1 95 | W = newW 96 | # if beta*np.max(np.sqrt(n)*dW/np.linalg.norm(w)) < eps2: 97 | # rho = rho0 98 | # crit2 = True 99 | # else: 100 | # rho = 1 101 | # crit2 = False 102 | # beta = min(maxBeta, beta*rho) 103 | # tau = Li + beta*eta 104 | if crit: # and crit2: 105 | print("Finished!!!") 106 | print(timestep, f(w)) 107 | break 108 | timestep += 1 109 | return w, values 110 | 111 | 112 | w3, values = pLADMAPSAP(w0, 0.001) 113 | print(f(w3)) 114 | plt.plot(values) 115 | plt.xlabel("Value") 116 | plt.ylabel("Steps") 117 | plt.savefig("pLADMPSAP") 118 | 119 | w = gd(w0) 120 | print(f(w)) 121 | 122 | """ 123 | w2 = minimize(f, w0, jac=gradient_f) 124 | print(w2.fun) 125 | """ 126 | -------------------------------------------------------------------------------- /4/hw41.py: -------------------------------------------------------------------------------- 1 | # HW4.1 2 | # Gradient calculator on given NN 3 | import numpy as np 4 | np.random.seed(19890817) 5 | 6 | # 卷积运算 7 | 8 | 9 | def conv(inputs: np.array, filter: np.array, padding=0): 10 | H, W = inputs.shape 11 | padded_input = np.zeros((H+2*padding, W+2*padding)) 12 | padded_input[padding:H+padding, padding:W+padding] = inputs 13 | fH, fW = filter.shape 14 | result = np.zeros((H-fH+1+2*padding, W-fW+1+2*padding)) 15 | for i in range(0, H-fH+1+2*padding): 16 | for j in range(0, W-fW+1+2*padding): 17 | input_now = padded_input[i:i + fH, j:j + fW] 18 | result[i, j] = np.sum(input_now * filter) 19 | return result 20 | 21 | 22 | class Op(object): 23 | def name(self): 24 | pass 25 | 26 | def __call__(self, inputs): 27 | pass 28 | 29 | def cal(self, inputs): 30 | pass 31 | 32 | def grad(self, inputs, gradback): 33 | pass 34 | 35 | 36 | class Node(object): 37 | 38 | # 初始化 39 | def __init__(self, op: Op, inputs, shape=None, weight=None, has_grad=False): 40 | self.op = op 41 | if not isinstance(inputs, Node): 42 | self.inputs = np.array(inputs) 43 | else: 44 | self.inputs = inputs 45 | self.name = op.name() 46 | if self.name in ['Add', 'Sub']: 47 | self.inputs_shape = self.inputs[0].value.squeeze().shape 48 | else: 49 | self.inputs_shape = self.inputsToValue().squeeze().shape 50 | self.input_grad = np.zeros(self.inputs_shape) 51 | self.value = None 52 | self.shape = shape 53 | self.has_grad = has_grad 54 | if self.has_grad: 55 | self.grad = np.zeros(self.shape) 56 | self.weight = weight 57 | self.gradback = None 58 | # 预计算给后续节点提供输出大小信息 59 | self.cal() 60 | 61 | # 前向计算 62 | def cal(self): 63 | self.value = self.op.cal(self.inputsToValue(), self.weight) 64 | # 此时已知输入大小, 初始化后向积累梯度 65 | self.gradback = np.zeros(self.value.shape) 66 | 67 | # 梯度计算 68 | def calgrad(self): 69 | if not self.has_grad: 70 | self.input_grad = self.op.grad(self.inputsToValue(), self.gradback) 71 | return self.input_grad 72 | else: 73 | tmp, self.input_grad = self.op.grad( 74 | self.inputsToValue(), self.weight, self.gradback) 75 | self.grad = tmp.squeeze() 76 | return self.input_grad 77 | 78 | # 把Node对象转成其数值用于计算 79 | def inputsToValue(self): 80 | lst = [] 81 | for i in self.inputs: 82 | if isinstance(i, Node): 83 | lst.append(i.value) 84 | else: 85 | lst.append(i) 86 | return np.array(lst) 87 | 88 | # 重载运算符 89 | def __add__(self, other): 90 | return add(self, other) 91 | 92 | def __radd__(self, other): 93 | return add(self, other) 94 | 95 | def __sub__(self, other): 96 | return sub(self, other) 97 | 98 | def __rsub__(self, other): 99 | return sub(other, self) 100 | 101 | # 加法 102 | 103 | 104 | class Add(Op): 105 | 106 | def name(self): 107 | return 'Add' 108 | 109 | def __call__(self, x, y): 110 | return Node(self, [x, y]) 111 | 112 | def cal(self, inputs, weight=None): 113 | assert(len(inputs) == 2) 114 | return inputs[0] + inputs[1] 115 | 116 | def grad(self, inputs, gradback): 117 | return np.array([gradback, gradback]) 118 | 119 | # 减法 120 | 121 | 122 | class Sub(Op): 123 | 124 | def name(self): 125 | return 'Sub' 126 | 127 | def __call__(self, x, y): 128 | return Node(self, [x, y]) 129 | 130 | def cal(self, inputs, weight=None): 131 | assert(len(inputs) == 2) 132 | return inputs[0] - inputs[1] 133 | 134 | def grad(self, inputs, gradback): 135 | return np.array([gradback, -gradback]) 136 | 137 | # 线性层 138 | 139 | 140 | class Linear(Op): 141 | def name(self): 142 | return 'Linear' 143 | 144 | def __init__(self, shape, weight=None): 145 | self.shape = shape 146 | if not type(weight) == np.ndarray: 147 | self.weight = np.random.rand(*shape) 148 | else: 149 | self.weight = weight 150 | 151 | def __call__(self, inputs: np.array): 152 | return Node(self, [inputs], self.shape, self.weight, True) 153 | 154 | def cal(self, inputs, weight): 155 | return weight@inputs 156 | 157 | def grad(self, inputs, weight, gradback): 158 | return gradback@inputs.T, weight.T@gradback 159 | 160 | # 卷积层 161 | 162 | 163 | class Conv(Op): 164 | def name(self): 165 | return 'Conv' 166 | 167 | def __init__(self, shape, padding=0, weight=None): 168 | self.shape = shape 169 | self.padding = padding 170 | if not type(weight) == np.ndarray: 171 | self.weight = np.random.rand(*shape) 172 | else: 173 | self.weight = weight 174 | 175 | def __call__(self, inputs: np.array): 176 | return Node(self, [inputs], self.shape, self.weight, True) 177 | 178 | def cal(self, inputs, weight): 179 | return conv(inputs.squeeze(), weight, padding=self.padding) 180 | 181 | def grad(self, inputs, weight, gradback): 182 | inputs = inputs.squeeze() 183 | gradback = gradback.squeeze() 184 | input_shape = inputs.shape 185 | input_grad = np.zeros(input_shape) 186 | shape = self.shape 187 | weight_grad = np.zeros(shape) 188 | # 计算输入梯度 189 | for i in range(input_shape[0]): 190 | for j in range(input_shape[1]): 191 | for a in range(gradback.shape[0]): 192 | for b in range(gradback.shape[1]): 193 | if i-a+self.padding < 0 or j-b+self.padding < 0 or i-a+self.padding >= shape[0] or j-b+self.padding >= shape[1]: 194 | continue 195 | input_grad[i, j] += gradback[a, 196 | b]*weight[i-a+self.padding, j-b+self.padding] 197 | # 计算卷积核梯度 198 | for i in range(shape[0]): 199 | for j in range(shape[1]): 200 | for a in range(gradback.shape[0]): 201 | for b in range(gradback.shape[1]): 202 | if i+a-self.padding < 0 or j+b-self.padding < 0 or i+a-self.padding >= input_shape[0] or j+b-self.padding >= input_shape[1]: 203 | continue 204 | weight_grad[i, j] += gradback[a, 205 | b]*inputs[i+a-self.padding, j+b-self.padding] 206 | return weight_grad, [input_grad] 207 | 208 | # MSE损失函数 209 | 210 | 211 | class MSE(Op): 212 | def name(self): 213 | return 'MSE' 214 | 215 | def __init__(self, shape, weight=None): 216 | self.shape = shape 217 | if not type(weight) == np.ndarray: 218 | self.weight = np.random.rand(*shape) 219 | else: 220 | self.weight = weight 221 | 222 | def __call__(self, inputs: np.array): 223 | return Node(self, [inputs], self.shape, self.weight, False) 224 | 225 | def cal(self, inputs: np.array, weight=None): 226 | return np.sum((inputs-self.weight) ** 2) 227 | 228 | def grad(self, inputs, gradback): 229 | return 2*(inputs-self.weight)*gradback 230 | 231 | # 输入变量 232 | 233 | 234 | class Holder(Op): 235 | def name(self): 236 | return 'Holder' 237 | 238 | def __call__(self, x): 239 | return Node(self, [x]) 240 | 241 | def cal(self, inputs, weight=None): 242 | return inputs[0] 243 | 244 | def grad(self, inputs, gradback): 245 | return gradback 246 | 247 | 248 | # 定义运算符 249 | add, sub, holder = Add(), Sub(), Holder() 250 | 251 | 252 | class ExecutionGraph(object): 253 | 254 | def __init__(self, root=None) -> None: 255 | if root == None: 256 | return 257 | self.root = root 258 | self.list = [] 259 | self.topoSort(root) 260 | # 去重 261 | self.topolist = [] 262 | for node in self.list: 263 | if not node in self.topolist: 264 | self.topolist.append(node) 265 | 266 | def cal(self, Print=False) -> float: 267 | for index, node in enumerate(self.topolist): 268 | node.cal() 269 | if Print: 270 | print(index, node.name, node.value) 271 | return self.root.value 272 | 273 | # 拓扑排序 274 | def topoSort(self, root): 275 | if type(root) is np.ndarray or root == None or not isinstance(root, Node): 276 | return 277 | for n in root.inputs: 278 | self.topoSort(n) 279 | self.list.append(root) 280 | 281 | def grad(self): 282 | reversed_lst = list(reversed(self.topolist)) 283 | reversed_lst[0].gradback = np.array([1.0]) 284 | for node in reversed_lst: 285 | node.calgrad() 286 | for inputNode, backGrad in zip(node.inputs, node.input_grad): 287 | if isinstance(inputNode, Node): 288 | if inputNode.name == 'Holder': 289 | backGrad = backGrad.squeeze() 290 | inputNode.gradback += backGrad 291 | -------------------------------------------------------------------------------- /4/hw41test.py: -------------------------------------------------------------------------------- 1 | # HW4.1 Test 2 | # 此处用于定义求导所用参数 3 | 4 | from hw41 import * 5 | 6 | # 先定义输入, 用holder构造 7 | x = holder(np.random.rand(5, 5)) 8 | 9 | # 再定义网络元素 10 | k = Conv((3, 3), padding=1) # 卷积, 需提供大小, 可以提供默认权重, 用weight制定 11 | l = Linear((5, 5)) # 线性, 需提供大小, 不指定weight时根据大小调用numpy.random.rand()生成随机矩阵 12 | y = np.random.rand(5, 5) # 正确输出值 13 | loss = MSE((5, 5), y) # 损失函数, 程序中提供了MSE 14 | 15 | # 构建网络, 此处定义了题目要求的resnet 16 | f = ExecutionGraph(loss(x+k(x))) 17 | 18 | # 进行计算 19 | print('------------------------------------------------------------------------') 20 | print("Forward:") 21 | res1 = f.cal() # 输出值 22 | print("Result =", res1) 23 | 24 | print('------------------------------------------------------------------------') 25 | print("Gradient:") 26 | f.grad() # 计算梯度 27 | 28 | print("x's grad", x.input_grad) # 每层网络输入的权重用input_grad表示 29 | print(f.topolist[1].name + "'s grad:", f.topolist[1].grad) # 每层网络自身的权重用grad表示 30 | 31 | # 数值测试 32 | print('------------------------------------------------------------------------') 33 | print("Numeric test on variables") 34 | x_grad = x.input_grad 35 | conv_grad = f.topolist[1].grad 36 | 37 | x.inputs[0][4, 2] += 10**(-8) 38 | res2 = f.cal() 39 | print("x[4,2]'s real grad:", x_grad[4, 2]) 40 | print("x[4,2]'s numeric grad:", (res2-res1)*(10**8)) 41 | x.inputs[0][4, 2] -= 10**(-8) 42 | 43 | f.topolist[1].weight[0, 1] += 10**(-8) 44 | res2 = f.cal() 45 | f.grad() 46 | print(f.topolist[1].name + "[0,1]'s real grad:", conv_grad[0, 1]) 47 | print(f.topolist[1].name + "[0,1]'s numeric grad:", (res2-res1)*(10**8)) 48 | f.topolist[1].weight[0, 1] -= 10**(-8) 49 | -------------------------------------------------------------------------------- /4/hw41test2.py: -------------------------------------------------------------------------------- 1 | # HW4.1 Test 2 | # 这是另一个测试程序 3 | # 此处用于定义求导所用参数 4 | 5 | from hw41 import * 6 | 7 | # 先定义输入, 用holder构造 8 | x = holder(np.random.rand(5, 5)) 9 | 10 | # 再定义网络元素 11 | k1 = Conv((3, 3), padding=1) # 卷积层, 需提供大小, 可以提供默认权重, 用weight制定 12 | k2 = Conv((3, 3), padding=1) 13 | k3 = Conv((3, 3), padding=1) 14 | k4 = Conv((3, 3), padding=0) 15 | k5 = Conv((3, 3), padding=1) 16 | l = Linear((5, 5)) # 线性, 需提供大小, 不指定weight时根据大小调用numpy.random.rand()生成随机矩阵 17 | y = np.random.rand(3, 3) # 正确输出值 18 | loss = MSE((3, 3), y) # 损失函数, 程序中提供了MSE 19 | 20 | # 构建网络 21 | f = ExecutionGraph(loss(k4(k3(x+l(k5(x)))+k2(x+k1(x))))) 22 | 23 | # 进行计算 24 | print('------------------------------------------------------------------------') 25 | print("Forward:") 26 | res1 = f.cal() # 输出值 27 | print("Result =", res1) 28 | 29 | print('------------------------------------------------------------------------') 30 | print("Gradient:") 31 | f.grad() # 计算梯度 32 | 33 | print("x's grad", x.input_grad) # 每层网络输入的权重用input_grad表示 34 | print(f.topolist[1].name + "'s grad:", f.topolist[1].grad) # 每层网络自身的权重用grad表示 35 | 36 | # 数值测试 37 | print('------------------------------------------------------------------------') 38 | print("Numeric test on variables") 39 | x_grad = x.input_grad 40 | conv_grad = f.topolist[4].grad 41 | 42 | x.inputs[0][4, 2] += 10**(-8) 43 | res2 = f.cal() 44 | print("x[4,2]'s real grad:", x_grad[4, 2]) 45 | print("x[4,2]'s numeric grad:", (res2-res1)*(10**8)) 46 | x.inputs[0][4, 2] -= 10**(-8) 47 | 48 | f.topolist[4].weight[0, 1] += 10**(-8) 49 | res2 = f.cal() 50 | f.grad() 51 | print(f.topolist[4].name + "[0,1]'s real grad:", conv_grad[0, 1]) 52 | print(f.topolist[4].name + "[0,1]'s numeric grad:", (res2-res1)*(10**8)) 53 | f.topolist[4].weight[0, 1] -= 10**(-8) 54 | -------------------------------------------------------------------------------- /4/hw42.in: -------------------------------------------------------------------------------- 1 | % 该文件为HW4.2的输入文件 2 | % 用%做注释 3 | % 先指明要求梯度的变量及数值 4 | x1 2.0 5 | x2 5.0 6 | x3 793484 7 | # 用#结束变量输入 8 | % 再输入要微分的式子, 此处使用题目中所给示例 9 | (sin(x1+1)+cos(2*x2))*tan(log(x3))+(sin(x2+1)+cos(2*x1))*exp(1+sin(x3)) 10 | % exp(log(x1)+x2/x3) 11 | %x1**(cos(x2)+log(x3)) -------------------------------------------------------------------------------- /4/hw42_autodiff.py: -------------------------------------------------------------------------------- 1 | # HW 4.2 Autodiff for multivariate formula 2 | 3 | import math 4 | from math import e, pi 5 | 6 | 7 | class Op(object): 8 | def name(self): 9 | pass 10 | 11 | def __call__(self, inputs): 12 | pass 13 | 14 | def cal(self, inputs): 15 | pass 16 | 17 | def grad(self, inputs, gradback): 18 | pass 19 | 20 | 21 | class Node(object): 22 | # 初始化 23 | def __init__(self, op: Op, inputs): 24 | self.op = op 25 | self.grad = 0 26 | self.inputs = inputs 27 | self.value = None 28 | self.name = op.name() 29 | 30 | # 计算 31 | def cal(self): 32 | self.value = self.op.cal(self.inputsToValue()) 33 | 34 | def inputsToValue(self): 35 | lst = [] 36 | for i in self.inputs: 37 | if isinstance(i, Node): 38 | lst.append(i.value) 39 | else: 40 | lst.append(i) 41 | return lst 42 | # 重载运算符 43 | 44 | def __add__(self, other): 45 | return add(self, other) 46 | 47 | def __radd__(self, other): 48 | return add(self, other) 49 | 50 | def __sub__(self, other): 51 | return sub(self, other) 52 | 53 | def __rsub__(self, other): 54 | return sub(other, self) 55 | 56 | def __mul__(self, other): 57 | return mul(self, other) 58 | 59 | def __rmul__(self, other): 60 | return mul(self, other) 61 | 62 | def __truediv__(self, other): 63 | return div(self, other) 64 | 65 | def __rtruediv__(self, other): 66 | return div(other, self) 67 | 68 | def __pow__(self, other): 69 | return pow(self, other) 70 | 71 | # 加法 72 | 73 | 74 | class Add(Op): 75 | 76 | def name(self): 77 | return 'Add' 78 | 79 | def __call__(self, x, y): 80 | return Node(self, [x, y]) 81 | 82 | def cal(self, inputs): 83 | assert(len(inputs) == 2) 84 | return inputs[0] + inputs[1] 85 | 86 | def grad(self, inputs, gradback): 87 | return [gradback, gradback] 88 | 89 | # 减法 90 | 91 | 92 | class Sub(Op): 93 | 94 | def name(self): 95 | return 'Sub' 96 | 97 | def __call__(self, x, y): 98 | return Node(self, [x, y]) 99 | 100 | def cal(self, inputs): 101 | assert(len(inputs) == 2) 102 | return inputs[0] - inputs[1] 103 | 104 | def grad(self, inputs, gradback): 105 | return [gradback, -gradback] 106 | 107 | # 乘法 108 | 109 | 110 | class Mul(Op): 111 | def name(self): 112 | return 'Mul' 113 | 114 | def __call__(self, x, y): 115 | return Node(self, [x, y]) 116 | 117 | def cal(self, inputs): 118 | assert(len(inputs) == 2) 119 | return inputs[0] * inputs[1] 120 | 121 | def grad(self, inputs, gradback): 122 | return [gradback * inputs[1], gradback * inputs[0]] 123 | 124 | 125 | # 除法 126 | class Div(Op): 127 | def name(self): 128 | return 'Div' 129 | 130 | def __call__(self, x, y): 131 | return Node(self, [x, y]) 132 | 133 | def cal(self, inputs): 134 | assert(len(inputs) == 2) 135 | try: 136 | assert(inputs[1] != 0) 137 | except: 138 | print("Divided by 0!") 139 | return inputs[0] / inputs[1] 140 | 141 | def grad(self, inputs, gradback): 142 | return [gradback / inputs[1], -gradback * inputs[0]/inputs[1]**2] 143 | 144 | # 对数 145 | 146 | 147 | class Log(Op): 148 | def name(self): 149 | return 'Log' 150 | 151 | def __call__(self, x): 152 | return Node(self, [x]) 153 | 154 | def cal(self, inputs): 155 | assert(len(inputs) == 1) 156 | assert(inputs[0] > 0) 157 | return math.log(inputs[0]) 158 | 159 | def grad(self, inputs, gradback): 160 | return [1.0/inputs[0]*gradback] 161 | 162 | 163 | class Sin(Op): 164 | def name(self): 165 | return 'Sin' 166 | 167 | def __call__(self, x): 168 | return Node(self, [x]) 169 | 170 | def cal(self, inputs): 171 | assert(len(inputs) == 1) 172 | return math.sin(inputs[0]) 173 | 174 | def grad(self, inputs, gradback): 175 | return [math.cos(inputs[0])*gradback] 176 | 177 | 178 | class Cos(Op): 179 | 180 | def name(self): 181 | return 'Cos' 182 | 183 | def __call__(self, x): 184 | return Node(self, [x]) 185 | 186 | def cal(self, inputs): 187 | assert(len(inputs) == 1) 188 | return math.cos(inputs[0]) 189 | 190 | def grad(self, inputs, gradback): 191 | return [-math.sin(inputs[0])*gradback] 192 | 193 | 194 | class Tan(Op): 195 | def name(self): 196 | return 'Tan' 197 | 198 | def __call__(self, x): 199 | return Node(self, [x]) 200 | 201 | def cal(self, inputs): 202 | assert(len(inputs) == 1) 203 | return math.tan(inputs[0]) 204 | 205 | def grad(self, inputs, gradback): 206 | return [gradback/((math.cos(inputs[0]))**2)] 207 | 208 | 209 | class Exp(Op): 210 | 211 | def name(self): 212 | return 'Exp' 213 | 214 | def __call__(self, x): 215 | return Node(self, [x]) 216 | 217 | def cal(self, inputs): 218 | assert(len(inputs) == 1) 219 | return math.exp(inputs[0]) 220 | 221 | def grad(self, inputs, gradback): 222 | return [gradback*math.exp(inputs[0])] 223 | 224 | 225 | class Pow(Op): 226 | def name(self): 227 | return 'Pow' 228 | 229 | def __call__(self, x, y): 230 | return Node(self, [x, y]) 231 | 232 | def cal(self, inputs): 233 | assert(len(inputs) == 2) 234 | return inputs[0]**inputs[1] 235 | 236 | def grad(self, inputs, gradback): 237 | return [gradback * inputs[1]*(inputs[0]**(inputs[1]-1)), gradback * math.log(inputs[0])*(inputs[0]**inputs[1])] 238 | 239 | 240 | class Holder(Op): 241 | def name(self): 242 | return 'Holder' 243 | 244 | def __call__(self, x): 245 | return Node(self, [x]) 246 | 247 | def cal(self, inputs): 248 | assert(len(inputs) == 1) 249 | return inputs[0] 250 | 251 | def grad(self, inputs, gradback): 252 | return [gradback] 253 | 254 | 255 | class ExecutionGraph(object): 256 | 257 | def __init__(self, root=None) -> None: 258 | if root == None: 259 | return 260 | self.root = root 261 | self.list = [] 262 | self.topoSort(root) 263 | # 去重 264 | self.topolist = [] 265 | for node in self.list: 266 | if not node in self.topolist: 267 | self.topolist.append(node) 268 | 269 | def cal(self, Print=False) -> float: 270 | for index, node in enumerate(self.topolist): 271 | node.cal() 272 | if Print: 273 | print(index, node.name, node.value) 274 | return self.root.value 275 | 276 | # 拓扑排序 277 | def topoSort(self, root): 278 | if root == None or not isinstance(root, Node): 279 | return 280 | for n in root.inputs: 281 | self.topoSort(n) 282 | self.list.append(root) 283 | 284 | def grad(self): 285 | reversed_lst = list(reversed(self.topolist)) 286 | reversed_lst[0].grad = 1.0 287 | for node in reversed_lst: 288 | grad = node.op.grad(node.inputsToValue(), node.grad) 289 | for inputNode, inputGrad in zip(node.inputs, grad): 290 | if isinstance(inputNode, Node): 291 | inputNode.grad += inputGrad 292 | 293 | 294 | # 定义运算符 295 | add, sub, mul, div, log, sin, cos, tan, exp, pow, holder = Add(), Sub( 296 | ), Mul(), Div(), Log(), Sin(), Cos(), Tan(), Exp(), Pow(), Holder() 297 | 298 | # 输入值 299 | # feedDict = {'x': 2.0, 'y': 5.0} 300 | # feedDict = {'x1': 2.0, 'x2': 5.0, 'x3': 7.0} 301 | feedDict = {} 302 | 303 | # 要计算的式子 304 | # formula = '3/x' 305 | # formula = 'log(x)+x**y-sin(y)' 306 | # formula = '(sin(x1+1)+cos(2*x2))*tan(log(x3))+(sin(x2+1)+cos(2*x1))*exp(1+sin(x3))' 307 | formula = '' 308 | 309 | # 从文件读入要积分的变量和表达式 310 | with open('hw42.in', 'r') as f: 311 | flag = False 312 | for line in f: 313 | line = line.strip() 314 | if not len(line): 315 | continue 316 | if line[0] == '#': 317 | flag = True 318 | continue 319 | if line[0] == '%': 320 | continue 321 | if not flag: 322 | tmp = line.strip().split() 323 | feedDict[tmp[0]] = float(tmp[1]) 324 | else: 325 | formula = line.strip() 326 | break 327 | 328 | for ele in feedDict: 329 | exec(ele + ' = holder(' + str(feedDict[ele]) + ')') 330 | 331 | graph = ExecutionGraph() 332 | 333 | try: 334 | exec('graph = ExecutionGraph(' + formula+')') 335 | except: 336 | print('Illegal Formula!') 337 | exit(0) 338 | 339 | 340 | # 打印计算图 341 | # for n in z.topolist: 342 | # print(n.name) 343 | 344 | print('------------------------------') 345 | print(formula) 346 | for ele in feedDict: 347 | print(ele, '=', feedDict[ele]) 348 | 349 | # 前向计算 350 | print('------------------------------') 351 | print("Forward:") 352 | result = graph.cal(Print=False) 353 | print('Result =', result) 354 | 355 | # 计算梯度 356 | graph.grad() 357 | print('------------------------------') 358 | print("Gradient:") 359 | for ele in feedDict: 360 | print(ele + "'s grad =", eval(ele).grad) 361 | print('------------------------------') 362 | 363 | # 数值测试 364 | print("Numeric test on variables") 365 | graph2 = ExecutionGraph() 366 | for ele in feedDict: 367 | exec(ele + '+= 10**(-7)') 368 | exec('graph2 = ExecutionGraph(' + formula+')') 369 | result2 = graph2.cal(Print=False) 370 | print(ele + "'s grad =", (result2-result)/10**(-7)) 371 | exec(ele + '-= 10**(-7)') 372 | print('------------------------------') 373 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 ZincCat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Convex Analysis Homework 2 | 2021 Spring 3 | 4 | ## List: 5 | - HW 4.1 Gradient calculator on given NN 6 | - HW 4.2 Autodiff for multivariate formula 7 | - HW 11 Gradient descent on given function 8 | - HW 12 Newton's method and conjugate gradient descent 9 | - HW 13.1 DFP Algorithm 10 | - HW 13.3 BFGS Algorithm 11 | - HW 13.4 L-BFGS Algorithm 12 | - HW 13.5 Majorization Minimization for LASSO 13 | - HW 17 Optimization with linear constraint 14 | - HW 18.1 Penalty Function 15 | - HW 18.2 Frank-Wolfe Algorithm 16 | - HW 19.1 Non-convergence of naive ADMM (failed) 17 | - HW 19.2 LADMAP 18 | - HW 19.3 LADMPSAP 19 | - HW 20.1 Block Coordinate Descent for a dictionary learning problem 20 | - HW 20.2 pLADMPSAP 21 | --------------------------------------------------------------------------------