├── .gitignore ├── LICENSE ├── README.md ├── chapter0.多项式求值 └── 霍纳法多项式求值.py ├── chapter1.求解方程根 ├── 不动点迭代.py ├── 二分法.py └── 牛顿法.py ├── chapter12.特征值和奇异值 ├── 12.1.幂迭代方法 │ ├── 占优特征值.py │ ├── 幂迭代法.py │ ├── 瑞利商迭代_结果与书上不符.py │ └── 逆向幂迭代.py ├── 12.2.QR算法 │ ├── PageRank算法.py │ ├── QR算法.py │ ├── 平移QR算法.py │ ├── 平移QR算法2.0.py │ └── 无移动QR算法.py ├── 12.3.奇异值分解 │ └── 奇异值分解.py └── 12.4.SVD的应用 │ ├── 12.4.3.图像压缩 │ ├── img.jpeg │ ├── result.png │ └── 图像压缩.py │ ├── 推荐系统应用.py │ └── 矩阵的低秩近似和降维.py ├── chapter13.最优化 ├── 13.1.不使用导数的无约束优化 │ └── 黄金分割搜索.py └── 13.2.使用导数的无约束优化 │ ├── 共轭梯度搜索.py │ ├── 最速下降法.py │ └── 牛顿法.py ├── chapter2.方程组 ├── 2.1.高斯消元 │ └── 朴素高斯消元.py ├── 2.2.LU分解 │ └── LU分解及回代.py ├── 2.4.PA=LU分解 │ ├── PA=LU分解及回代.py │ └── 部分主元法高斯消元.py ├── 2.5.迭代方法 │ ├── Gauss-Seidel方法.py │ ├── Jocobi迭代.py │ ├── SOR方法.py │ ├── 三种迭代方法对比.py │ └── 稀疏矩阵Jocobi迭代(没搞完).py ├── 2.6.用于对称正定矩阵的方法 │ ├── 共轭梯度法.py │ ├── 对角矩阵定义.py │ ├── 楚列斯基分解.py │ └── 预条件共轭梯度法.py └── 2.7 非线性方程组 │ ├── Broyden方法.py │ ├── Broyden方法2.py │ └── 多变量牛顿方法.py ├── chapter4.最小二乘 ├── QR分解_经典Gram正交化.py ├── QR分解实现最小二乘.py ├── 完全QR分解_经典Gram正交化.py ├── 改进Gram正交化.py ├── 最小二乘_多项式.py ├── 最小二乘_直线.py ├── 最小二乘_迭代法.py ├── 最小二乘_迭代法_带正则项.py ├── 范德蒙德实现最小二乘.py └── 范德蒙德矩阵.py ├── chapter5.数值微分和积分 └── 前向自动微分.py ├── chapter9.随机数和应用 ├── 最小标准生成器.py ├── 蒙特卡洛1型问题-随机数近似曲线下方面积.py ├── 蒙特卡洛2型问题-随机数近似图形面积.py └── 随机游走.py └── pic └── numericalanalysis_cover.jpeg /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Debug logs 10 | *.txt 11 | 12 | # Plot results 13 | *.png 14 | 15 | # IDE - VSCode 16 | .vscode/ 17 | 18 | # OS generated files 19 | .DS_Store 20 | 21 | # Distribution / packaging 22 | .Python 23 | build/ 24 | develop-eggs/ 25 | dist/ 26 | downloads/ 27 | eggs/ 28 | .eggs/ 29 | lib/ 30 | lib64/ 31 | parts/ 32 | sdist/ 33 | var/ 34 | wheels/ 35 | pip-wheel-metadata/ 36 | share/python-wheels/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | MANIFEST 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Unit test / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .nox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | *.py,cover 63 | .hypothesis/ 64 | .pytest_cache/ 65 | 66 | # Translations 67 | *.mo 68 | *.pot 69 | 70 | # Django stuff: 71 | *.log 72 | local_settings.py 73 | db.sqlite3 74 | db.sqlite3-journal 75 | 76 | # Flask stuff: 77 | instance/ 78 | .webassets-cache 79 | 80 | # Scrapy stuff: 81 | .scrapy 82 | 83 | # Sphinx documentation 84 | docs/_build/ 85 | 86 | # PyBuilder 87 | target/ 88 | 89 | # Jupyter Notebook 90 | .ipynb_checkpoints 91 | 92 | # IPython 93 | profile_default/ 94 | ipython_config.py 95 | 96 | # pyenv 97 | .python-version 98 | 99 | # pipenv 100 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 101 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 102 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 103 | # install all needed dependencies. 104 | #Pipfile.lock 105 | 106 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 107 | __pypackages__/ 108 | 109 | # Celery stuff 110 | celerybeat-schedule 111 | celerybeat.pid 112 | 113 | # SageMath parsed files 114 | *.sage.py 115 | 116 | # Environments 117 | .env 118 | .venv 119 | env/ 120 | venv/ 121 | ENV/ 122 | env.bak/ 123 | venv.bak/ 124 | 125 | # Spyder project settings 126 | .spyderproject 127 | .spyproject 128 | 129 | # Rope project settings 130 | .ropeproject 131 | 132 | # mkdocs documentation 133 | /site 134 | 135 | # mypy 136 | .mypy_cache/ 137 | .dmypy.json 138 | dmypy.json 139 | 140 | # Pyre type checker 141 | .pyre/ 142 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 HongYu Zhang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 9 |

10 | 11 | 12 | 13 |

14 | 15 |
16 | 17 | # 数值分析(第二版) 18 | 19 | 📚 天灭解析式，数值保平安~ 20 | 21 | [![Open Source Love](https://badges.frapsoft.com/os/v2/open-source.svg?v=103)](https://github.com/orion-orion/NumericalAnalysis) [![](https://img.shields.io/github/license/orion-orion/NumericalAnalysis)](https://github.com/orion-orion/NumericalAnalysis/blob/master/LICENSE) [![](https://img.shields.io/github/stars/orion-orion/NumericalAnalysis?style=social)](https://github.com/orion-orion/NumericalAnalysis) [![](https://img.shields.io/github/forks/orion-orion/NumericalAnalysis-Python?style=social)](https://github.com/orion-orion/NumericalAnalysis) 22 |
23 | [![](https://img.shields.io/github/directory-file-count/orion-orion/NumericalAnalysis-Python)](https://github.com/orion-orion/NumericalAnalysis) [![](https://img.shields.io/github/languages/code-size/orion-orion/NumericalAnalysis-Python)](https://github.com/orion-orion/NumericalAnalysis) 24 | 25 | 26 | 27 | 28 |
29 | 30 | ## 1 简介 31 | 本项目为《数值分析》(Timothy Sauer著) 第二版中的算法实现（使用Python+Numpy+Pytorch）。 32 | 33 | ## 2 目录 34 | - 第0章 多项式求值 35 | - 霍纳法多项式求值 36 | - 第1章 求解方程根 37 | - 不动点迭代 38 | - 二分法 39 | - 牛顿法 40 | - 第2章 方程组 41 | - 2.1 高斯消元 42 | - 朴素高斯消元 43 | - 2.2 LU分解 44 | - LU分解及回代 45 | - 2.4 PA=LU分解 46 | - 部分主元法高斯消元 47 | - PA=LU分解及回代 48 | - 2.5 迭代方法 49 | - 三种迭代方法对比 50 | - 稀疏矩阵Jocobi迭代（没搞完） 51 | - Gauss-Seidel方法 52 | - Jocobi迭代 53 | - SOR方法 54 | - 2.6 用于对称正定矩阵的方法 55 | - 楚列斯基分解 56 | - 对角矩阵定义 57 | - 共轭梯度法 58 | - 预条件共轭梯度法 59 | - 2.7 非线性方程组 60 | - 多变量牛顿方法 61 | - Broyden方法 62 | - Broyden方法2 63 | - 第4章 最小二乘 64 | - 解析法求解最小二乘（直线拟合） [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15887067.html) 65 | - 解析法求解最小二乘（多项式拟合） [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15887067.html) 66 | - 范德蒙德矩阵 67 | - 范德蒙德矩阵实现最小二乘 68 | - 迭代法求解最小二乘 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15887067.html) 69 | - 迭代法求解最小二乘（带正则项） [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15887067.html) 70 | - QR分解（经典Gram-Schmidt正交化） 71 | - QR分解实现最小二乘 72 | - 改进的Gram-Schmidt正交化 73 | - 第5章 数值微分和积分 74 | - 前向自动微分 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/17010353.html) 75 | - 第9章 随机数和应用 76 | - 蒙特卡洛1型问题-随机数近似曲线下方面积 77 | - 蒙特卡洛2型问题-随机数近似图形面积 78 | - 随机游走 79 | - 最小标准生成器 80 | - 第12章 特征值和特征向量 81 | - 12.1 幂迭代方法 82 | - 幂迭代法 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15405907.html) 83 | - 逆向幂迭代 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15405907.html) 84 | - 瑞利商迭代（结果与书上不符） [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15405907.html) 85 | - 占优特征值 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15405907.html) 86 | - 12.2 QR算法 87 | - 平移QR算法 88 | - 平移QR算法2 89 | - 无移动QR算法 90 | - PageRank算法 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15405907.html) 91 | - QR算法 92 | - 12.3 奇异值分解 93 | - 奇异值分解 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15415610.html) 94 | - 12.4 奇异值分解的应用 95 | - 矩阵的低秩近似和降维 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15415610.html) 96 | - 推荐系统应用 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15415610.html) 97 | - 图像压缩 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15415610.html) 98 | - 第13章 最优化 99 | - 不使用导数的无约束优化 100 | - 黄金分割搜索 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15418056.html) 101 | - 使用导数的无约束优化 102 | - 共轭梯度法 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15418056.html) 103 | - 牛顿法 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15418056.html) 104 | - 最速下降法 [[算法讲解]](https://www.cnblogs.com/orion-orion/p/15418056.html) 105 | 106 | -------------------------------------------------------------------------------- /chapter0.多项式求值/霍纳法多项式求值.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: horn法则多项式求值 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-07 11:40:20 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-03-07 11:59:29 8 | ''' 9 | import numpy as np 10 | 11 | def poly(order, coeff, x, basis): 12 | if basis is None: 13 | basis = np.zeros((order,), dtype=np.float32) 14 | # 初始递推值 15 | y = coeff[-1] 16 | for i in reversed(range(order)): 17 | y = y * (x - basis[i]) + coeff[i] 18 | return y 19 | 20 | if __name__ == '__main__': 21 | # 令所有基点为0的普通多项式 22 | res1 = poly(4, np.array([-1, 5,-3, 3, 2]), 1/2, np.array([0, 0, 0, 0])) 23 | print(res1) 24 | # 基点不为0的三阶插值多项式 25 | res2 = poly(3, np.array([1, 1/2, 1/2, -1/2]), 1, [0, 2, 3]) 26 | print(res2) -------------------------------------------------------------------------------- /chapter1.求解方程根/不动点迭代.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-08 15:13:40 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-03-08 15:21:11 8 | ''' 9 | import numpy as np 10 | import math 11 | def fpi(g, x0 ,k): #迭代k次,包括x0在内共k+1个数 12 | x = np.zeros(k+1,) 13 | x[0] = x0 14 | for i in range(1, k+1): 15 | x[i] = g(x[i-1]) 16 | return x[k] 17 | if __name__ == '__main__': 18 | res = fpi(lambda x: math.cos(x), 0, 10) 19 | print(res) -------------------------------------------------------------------------------- /chapter1.求解方程根/二分法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-08 14:48:29 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-03-08 15:01:59 8 | ''' 9 | import numpy as np 10 | import math 11 | def binary(f, a, b, tol): 12 | if f(a)*f(b) >=0 : 13 | raise ValueError("f(a)*f(b)<0 not satisfied!") 14 | while (b-a)/2 > tol: 15 | c=(a+b)/2 # 即使a和b是int,此处c自动转float了 16 | if f(c) == 0: #c是一个解，完成 17 | break 18 | if f(a)*f(c)<0 : 19 | b = c 20 | else: 21 | a = c 22 | return (a+b)/2 23 | if __name__ == '__main__': 24 | res = binary(lambda x: x**3+x-1, 0, 1, 5*pow(10, -5)) 25 | print(res) -------------------------------------------------------------------------------- /chapter1.求解方程根/牛顿法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-08 15:57:56 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-10-17 19:30:17 8 | ''' 9 | import numpy as np 10 | import math 11 | import torch 12 | #x.grad为dy/dx(假设dy为最后一个节点) 13 | def newton(x0, k, f): #迭代k次,包括x0在内共k+1个数 14 | # 初始化计算图参数 15 | x = torch.tensor([x0], requires_grad=True) 16 | for i in range(1, k+1): 17 | # 前向传播，注意x要用新的对象，否则后面y.backgrad后会释放 18 | y = f(x) 19 | y.backward() # y.grad是None 20 | # 更新参数 21 | with torch.no_grad(): 22 | x -= torch.divide(y, x.grad) 23 | x.grad.zero_() # 清空梯度，使下一轮建立新的计算图，否则因为backward释放资源下一轮再backward出错 24 | #注意x.grad不能是0，否则要出错使g(x)/x.grad变为none 25 | return x.detach().numpy()[0] 26 | if __name__ == '__main__': 27 | f = lambda x: x**3 + x - 1 28 | x0 = 1.0 29 | res = newton(x0, 10, f) 30 | print(res) -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.1.幂迭代方法/占优特征值.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 15:38:31 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-08 17:05:41 8 | ''' 9 | import numpy as np 10 | def prime_eigen(A, x, k): 11 | x_t = x.copy() 12 | for j in range(k): 13 | x_t = A.dot(x_t) 14 | return x_t 15 | if __name__ == '__main__': 16 | A = np.array( 17 | [ 18 | [1, 3], 19 | [2, 2] 20 | ] 21 | ) 22 | x = np.array([-5, 5]) 23 | k = 4 24 | r = prime_eigen(A, x, k) 25 | print(r) -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.1.幂迭代方法/幂迭代法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 15:38:31 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-08 17:19:23 8 | ''' 9 | import numpy as np 10 | 11 | # 幂迭代本质上每步进行归一化的不动点迭代 12 | # 毕竟x左乘一个矩阵A相当于g(*)作用于x 13 | def powerit(A, x, k): 14 | for j in range(k): 15 | # 为了让数据不失去控制 16 | # 每次迭代前先对x进行归一化 17 | u = x/np.linalg.norm(x) 18 | # 计算下一轮x，即将u乘以A, 为Auj-1 19 | x = A.dot(u) 20 | # 欲用最小二乘解特征方程x*lamb = Ax 21 | # x是特征向量的近似，lamb未知，系数矩阵是x，参数是lamb，b为Ax 22 | # 正规(法线)方程指出最小二乘解为:xT_x*lamb = xT(Ax)的解(方程两边同时乘AT的逆消掉) 23 | # 或者lamb = xTAx/xTx 24 | # 即lamb = uTAu，u.dot(A).dot(u) (又X=Au，避免重复计算这里乘x) 25 | # 故如下计算出本轮对应的特征值 26 | lam = u.dot(x) 27 | # 最后一次迭代得到的特征向量x需要归一化为u 28 | u = x / np.linalg.norm(x) 29 | return u, lam 30 | 31 | if __name__ == '__main__': 32 | A = np.array( 33 | [ 34 | [1, 3], 35 | [2, 2] 36 | ] 37 | ) 38 | x = np.array([-5, 5]) 39 | k = 10 40 | # 返回占优特征值和对应的特征值 41 | u, lam = powerit(A, x, k) 42 | # u为 [0.70710341 0.70711015]，指向特征向量[1, 1]的方向 43 | print("占优的特征向量:\n", u) 44 | print("占优的特征值:\n", lam) 45 | 46 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.1.幂迭代方法/瑞利商迭代_结果与书上不符.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 将瑞利商做为逆向幂迭代的特征值加速收敛 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 16:55:06 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-24 20:20:13 8 | ''' 9 | import numpy as np 10 | 11 | def Ray(A, x, k): 12 | for j in range(k): 13 | # 为了让数据不失去控制 14 | # 每次迭代前先对x进行归一化 15 | u = x/np.linalg.norm(x) 16 | 17 | #用瑞利商表示特征值的近似值lam 18 | lam = u.dot(A).dot(u) 19 | 20 | # 求解(A-lamj-1 I)Xj = uj-1 21 | # 收敛后A-lamj-1 I为奇异矩阵，不能继续迭代 22 | # 故在这种情况出现之前终止迭代 23 | try: 24 | # 单位矩阵I用np.eye实现 25 | x = np.linalg.solve(A - lam*np.eye(A.shape[0]), u) #逆向幂迭代 26 | except np.linalg.LinAlgError as e: 27 | print("A-lam I 为奇异矩阵，停止迭代!") 28 | break 29 | 30 | # 最后一次迭代得到的特征向量x需要归一化为u 31 | u = x / np.linalg.norm(x) 32 | # 瑞利商(这里容易漏掉) 33 | lam = u.dot(A).dot(u) 34 | return u, lam 35 | 36 | if __name__ == '__main__': 37 | A = np.array( 38 | [ 39 | [1, 3], 40 | [2, 2] 41 | ] 42 | ) 43 | x = np.array([-5, 5]) 44 | k = 10 45 | # 返回占优特征值和对应的特征值 46 | u, lam = Ray(A, x, k) 47 | # u为 [0.70710341 0.70711015]，指向特征向量[1, 1]的方向 48 | print("占优的特征向量:\n", u) 49 | print("占优的特征值:\n", lam) -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.1.幂迭代方法/逆向幂迭代.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 逆向幂迭代 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 16:55:06 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-22 17:18:21 8 | ''' 9 | import numpy as np 10 | 11 | # 幂迭代本质上每步进行归一化的不动点迭代 12 | def powerit(A, x, s, k): 13 | As = A-s*np.eye(A.shape[0]) 14 | for j in range(k): 15 | # 为了让数据不失去控制 16 | # 每次迭代前先对x进行归一化 17 | u = x/np.linalg.norm(x) 18 | 19 | # 求解(A-sI)Xj = uj-1 20 | x = np.linalg.solve(As, u) 21 | lam = u.dot(x) 22 | lam = 1/lam + s 23 | 24 | # 最后一次迭代得到的特征向量x需要归一化为u 25 | u = x / np.linalg.norm(x) 26 | return u, lam 27 | 28 | if __name__ == '__main__': 29 | A = np.array( 30 | [ 31 | [1, 3], 32 | [2, 2] 33 | ] 34 | ) 35 | x = np.array([-5, 5]) 36 | k = 10 37 | # 逆向幂迭代的平移值，可以通过平移值收敛到不同的特征值 38 | s = 2 39 | # 返回占优特征值和对应的特征值 40 | u, lam = powerit(A, x, s, k) 41 | # u为 [0.70710341 0.70711015]，指向特征向量[1, 1]的方向 42 | print("占优的特征向量:\n", u) 43 | print("占优的特征值:\n", lam) -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.2.QR算法/PageRank算法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-23 15:42:51 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2022-05-31 21:15:46 8 | ''' 9 | import numpy as np 10 | # 归一化同时迭代，k是迭代步数 11 | # 欲推往A特征值的方向，A肯定是方阵 12 | def PageRank(A, p, k, q): 13 | assert(A.shape[0]==A.shape[1]) 14 | n = A.shape[0] 15 | M = A.T.astype(np.float32) #注意要转为浮点型 16 | for i in range(n): 17 | M[:, i] = M[:, i]/np.sum(M[:, i]) 18 | G = (q/n)*np.ones((n,n)) + (1-q)*M 19 | #G_T = G.T 20 | p_t = p.copy() 21 | for i in range(k): 22 | y = G.dot(p_t) 23 | p_t = y/np.max(y) 24 | return p_t/np.sum(p_t) 25 | if __name__ == '__main__': 26 | A = np.array( 27 | [ 28 | [0, 1, 1], 29 | [0, 0, 1], 30 | [1, 0, 0] 31 | ] 32 | ) 33 | k = 20 34 | p = np.array([1, 1, 1]) 35 | q = 0.15 #概率为1移动到一个随机页面通常为0.15 36 | # 概率为1-q移动到与本页面链接的页面 37 | R= PageRank(A, p, k, q) 38 | print(R) 39 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.2.QR算法/QR算法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 21:00:46 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-22 21:18:09 8 | ''' 9 | import numpy as np 10 | # 归一化同时迭代，k是迭代步数 11 | # 欲求A特征值，A肯定是方阵 12 | def nsi(A, k): 13 | # Q0 = I 14 | # eye为方阵接收一个参数n 15 | Q = np.eye(A.shape[0]) 16 | for i in range(k): 17 | # 乘A, qr分解将其正交化后再乘A，以此迭代，这种写法很简洁，紧致 18 | Q, R = np.linalg.qr(A.dot(Q), mode="reduced") 19 | # 瑞利商 20 | # 对角线元素即为所有特征值 21 | lams = np.diag(Q.T.dot(A).dot(Q)) 22 | return Q, lams 23 | if __name__ == '__main__': 24 | A = np.array( 25 | [ 26 | [1, 3], 27 | [2, 2] 28 | ] 29 | ) 30 | k = 10 31 | Q, lams = nsi(A, k) 32 | 33 | # 特征向量为(-0.707, -0.707)，(-0.707, 0.707) 34 | # 特征值是4和-1 35 | # 占优特征值为4，占优特征向量为(-0.707, -0.707)，与幂迭代结果一致 36 | # 幂迭代占优特征向量为(0.7, 0.7)，占优特征值为4 37 | # 真实客观情况：占优特征向量(1, 1)，占优特征值为4 38 | # 一个特征值对应的特征向量(组)可以随意缩放长度，其实是无数个 39 | print(Q, "\n\n", lams) 40 | 41 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.2.QR算法/平移QR算法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-23 10:17:21 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-23 15:09:03 8 | ''' 9 | import numpy as np 10 | ''' 11 | @Description: 12 | @Param: 矩阵A 13 | @Return: 特征值lam 14 | @Author: ZhangHongYu 15 | @param {*} A 16 | ''' 17 | def shiftedqr0(A): 18 | tol = 1e-14 19 | m = A.shape[0] 20 | lam = np.zeros((m, )) 21 | n = m 22 | while n > 1: 23 | while max(abs(A[n-1, :n])) > tol: 24 | # 定义平移mu 25 | mu = A[n-1, n-1] 26 | Q, R = np.linalg.qr(A-mu*np.eye(n)) 27 | A = R.dot(Q) + mu * np.eye(n) 28 | # 声明特征值 29 | lam[n-1] = A[n-1, n-1] 30 | # 降低n 31 | n = n - 1 32 | # 收缩 33 | A = A[:n, :n] #.copy() 34 | lam[0] = A[0, 0] 35 | return lam 36 | 37 | if __name__ == '__main__': 38 | A = np.array( 39 | [ 40 | [1, 3], 41 | [2, 2] 42 | ] 43 | ) 44 | k = 10 45 | Q, lams = shiftedqr0(A) 46 | 47 | # 特征向量为(-0.707, -0.707)，(-0.707, 0.707) 48 | # 特征值是4和-1 49 | # 占优特征值为4，占优特征向量为(-0.707, -0.707)，与幂迭代结果一致 50 | # 幂迭代占优特征向量为(0.7, 0.7)，占优特征值为4 51 | # 真实客观情况：占优特征向量(1, 1)，占优特征值为4 52 | # 一个特征值对应的特征向量(组)可以随意缩放长度，其实是无数个 53 | print(Q, "\n\n", lams) 54 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.2.QR算法/平移QR算法2.0.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion:为了允许计算复数特征值，必须允许在实数舒尔形式的对角线上存在2*2的块 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-23 10:17:21 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-23 11:21:05 8 | ''' 9 | import numpy as np 10 | ''' 11 | @Description: 12 | @Param: 矩阵A 13 | @Return: 特征值lam 14 | @Author: ZhangHongYu 15 | @param {*} A 16 | ''' 17 | def shiftedqr0(A): 18 | tol = 1e-14 19 | m = A.shape[0] 20 | lam = np.zeros((A.shape[0], )) 21 | n = m 22 | while n > 1: 23 | while max(abs(A[n-1, :n])) > tol: 24 | # 定义平移mu 25 | mu = A[n-1, n-1] 26 | Q, R = np.linalg.qr(A-mu*np.eye(n)) 27 | A = R.dot(Q) + mu * np.eye(n) 28 | # 声明特征值 29 | lam[n-1] = A[n-1, n-1] 30 | # 降低n 31 | n = n - 1 32 | # 收缩 33 | A = A[:n, :n] #.copy() 34 | lam[0] = A[0, 0] 35 | return lam 36 | 37 | if __name__ == '__main__': 38 | A = np.array( 39 | [ 40 | [1, 3], 41 | [2, 2] 42 | ] 43 | ) 44 | k = 10 45 | Q, lams = shiftedqr0(A) 46 | 47 | # 特征向量为(-0.707, -0.707)，(-0.707, 0.707) 48 | # 特征值是4和-1 49 | # 占优特征值为4，占优特征向量为(-0.707, -0.707)，与幂迭代结果一致 50 | # 幂迭代占优特征向量为(0.7, 0.7)，占优特征值为4 51 | # 真实客观情况：占优特征向量(1, 1)，占优特征值为4 52 | # 一个特征值对应的特征向量(组)可以随意缩放长度，其实是无数个 53 | print(Q, "\n\n", lams) 54 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.2.QR算法/无移动QR算法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 算法证明见Golub, Van Loan《矩阵计算》 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 21:00:46 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-23 08:33:57 8 | ''' 9 | import numpy as np 10 | # 对比 11 | # def nsi(A, k): 12 | # # Q0 = I 13 | # # eye为方阵接收一个参数n 14 | # Q = np.eye(A.shape[0]) 15 | # for i in range(k): 16 | # # 乘A, qr分解将其正交化后再乘A，以此迭代，这种写法很简洁，紧致 17 | # Q, R = np.linalg.qr(A.dot(Q), mode="reduced") 18 | # # 瑞利商 19 | # # 对角线元素即为所有特征值 20 | # lams = np.diag(Q.T.dot(A).dot(Q)) 21 | # return Q, lams 22 | 23 | # 归一化同时迭代，k是迭代步数 24 | # 欲求A特征值，A肯定是方阵 25 | def unshiftedqr(A, k): 26 | # Q0 = I 27 | # eye为方阵接收一个参数n 28 | Q = np.eye(A.shape[0]) 29 | Qbar = Q.copy() 30 | R = A.copy() 31 | for i in range(k): 32 | # QR分解 33 | Q, R = np.linalg.qr(R.dot(Q), mode="reduced") 34 | # 累计Q 35 | Qbar = Qbar.dot(Q) 36 | 37 | # 对角线收敛到特征值 38 | lams = np.diag(R.dot(Q)) 39 | # 输出特征值lams和特征向量矩阵Q_bar 40 | return Qbar, lams 41 | if __name__ == '__main__': 42 | A = np.array( 43 | [ 44 | [1, 3], 45 | [2, 2] 46 | ] 47 | ) 48 | k = 10 49 | Q, lams = unshiftedqr(A, k) 50 | 51 | # 特征向量为(-0.707, -0.707)，(-0.707, 0.707) 52 | # 特征值是4和-1 53 | # 占优特征值为4，占优特征向量为(-0.707, -0.707)，与幂迭代结果一致 54 | # 幂迭代占优特征向量为(0.7, 0.7)，占优特征值为4 55 | # 真实客观情况：占优特征向量(1, 1)，占优特征值为4 56 | # 一个特征值对应的特征向量(组)可以随意缩放长度，其实是无数个 57 | print(Q, "\n\n", lams) 58 | 59 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.3.奇异值分解/奇异值分解.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-23 21:25:39 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-10-15 21:56:23 8 | ''' 9 | import numpy as np 10 | def svd(A): 11 | eigen_values, eigen_vectors = np.linalg.eig(A.T.dot(A)) 12 | singular_values = np.sqrt(eigen_values) 13 | #这里奇异值要从大到小排序，特征向量也要随之从大到小排 14 | val_vec = [] #存储奇异值-特征向量对 15 | for i in range(len(eigen_values)): 16 | val_vec.append((singular_values[i], eigen_vectors[:, i])) 17 | val_vec.sort(key = lambda x:-x[0]) 18 | singular_values = [ pair[0] for pair in val_vec] 19 | eigen_vectors = [ pair[1] for pair in val_vec] 20 | 21 | # 在计算左奇异向量之前，先要对右奇异向量 22 | # 也就是特征向量组成的基正交化，不过linalg.eig返回的是已经正交化的 23 | 24 | # 由等式Avi = siui(vi是右奇异向量, ui是左奇异向量) 25 | # 依次计算左奇异向量 26 | U = np.zeros((A.shape[0], A.shape[1])) 27 | for i in range(A.shape[1]): 28 | if singular_values[i] != 0: 29 | u = A.dot(eigen_vectors[i])/singular_values[i] 30 | else: 31 | # u = 与之前的u正交的向量 32 | pass #这里还没写好 33 | U[:, i] = u 34 | # 给U加上标准正交基去构造R3的基 35 | for i in range(A.shape[1], A.shape[0]): 36 | basis = np.zeros((A.shape[0], 1)) 37 | basis[i] = 1 38 | U = np.concatenate([U, basis], axis=1) 39 | # S = np.diag(singular_values) 40 | # S = np.concatenate([S, np.zeros((A.shape[0]-A.shape[1], A.shape[1]))], axis=0) 41 | eigen_vectors = [vec.reshape(-1, 1) for vec in eigen_vectors] 42 | eigen_vectors = np.concatenate(eigen_vectors, axis=1) 43 | return U, singular_values, eigen_vectors 44 | 45 | if __name__ == '__main__': 46 | # 例一：非方阵 47 | # A = np.array( 48 | # [ 49 | # [0, -1/2], 50 | # [3, 0], 51 | # [0, 0] 52 | # ] 53 | # ) 54 | # 例二：方阵 55 | # A = np.array( 56 | # [ 57 | # [0, 1], 58 | # [0, -1] 59 | # ] 60 | # ) 61 | # 例三：对称矩阵 62 | A = np.array( 63 | [ 64 | [0, 1], 65 | [1, 3/2] 66 | ] 67 | ) 68 | U, S, V = svd(A) 69 | print("我们实现的算法结果：") 70 | print(U, "\n", S, "\n", V) 71 | print("\n") 72 | print("调用库函数的计算结果：") 73 | # 调用api核对 74 | U2, S2, V2 = np.linalg.svd(A) 75 | print(U2, "\n", S2, "\n", V2) 76 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.4.SVD的应用/12.4.3.图像压缩/img.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orion-orion/NumericalAnalysis-Python/f1eab2e84658eca326b836336421af52a44b62ef/chapter12.特征值和奇异值/12.4.SVD的应用/12.4.3.图像压缩/img.jpeg -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.4.SVD的应用/12.4.3.图像压缩/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orion-orion/NumericalAnalysis-Python/f1eab2e84658eca326b836336421af52a44b62ef/chapter12.特征值和奇异值/12.4.SVD的应用/12.4.3.图像压缩/result.png -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.4.SVD的应用/12.4.3.图像压缩/图像压缩.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 as cv 3 | import matplotlib.pyplot as plt 4 | import matplotlib as mpl 5 | mpl.rcParams["font.sans-serif"] = [u"SimHei"] 6 | mpl.rcParams["axes.unicode_minus"] = False 7 | 8 | 9 | def approximation(A, p): 10 | B = np.zeros(A.shape) 11 | for c in range(A.shape[2]): 12 | U, s, V_T = np.linalg.svd(A[:, :, c]) 13 | for i in range(p): 14 | B[:, :, c] += s[i] * \ 15 | U[:, i].reshape(-1, 1).dot(V_T[i, :].reshape(1, -1)) 16 | return B 17 | 18 | 19 | if __name__ == '__main__': 20 | img = cv.imread( 21 | "chapter12.特征值和奇异值/12.4.SVD的应用/12.4.3.图像压缩/img.jpeg") 22 | # 将OpenCV采用的BGR格式转换到Matplotlib采用的RGB格式 23 | img = cv.cvtColor(img, cv.COLOR_BGR2RGB) 24 | # 图像必须归一化到[0 - 1]范围 25 | img = img.astype(np.float32) / 255.0 26 | img_output = img.copy() 27 | 28 | # p为近似矩阵的秩，秩p<=r，p越大图像压缩程度越小，越清晰 29 | p = 50 30 | img_output = approximation(img, p) 31 | 32 | fig, axs = plt.subplots(1, 2) 33 | axs[0].imshow(np.clip(img, 0, 1)) 34 | axs[0].set_title(u"原图") 35 | axs[1].imshow(np.clip(img_output, 0, 1)) 36 | axs[1].set_title(u"压缩后的图") 37 | plt.savefig( 38 | "chapter12.特征值和奇异值/12.4.SVD的应用/12.4.3.图像压缩/result.png", 39 | bbox_inches="tight") 40 | plt.show() 41 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.4.SVD的应用/推荐系统应用.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-07-09 09:07:46 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-10-16 10:43:47 8 | ''' 9 | import numpy as np 10 | 11 | 12 | if __name__ == '__main__': 13 | M = np.array( 14 | [ 15 | [0, 4.5, 2.0, 0], 16 | [4.0, 0, 3.5, 0], 17 | [0, 5.0, 0, 2.0], 18 | [0, 3.5, 4.0, 1.0] 19 | ] 20 | ) 21 | U, S, V_T = np.linalg.svd(M) 22 | k = 2 # 取前2个奇异值对应的隐向量 23 | # 分别打印物品向量和用户向量 24 | Vec_user, Vec_item = U[:,:k], V_T[:k, :].T 25 | print(Vec_user, "\n\n", Vec_item) 26 | -------------------------------------------------------------------------------- /chapter12.特征值和奇异值/12.4.SVD的应用/矩阵的低秩近似和降维.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-07-01 21:35:04 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-10-16 15:34:09 8 | ''' 9 | import numpy as np 10 | from sklearn.decomposition import PCA 11 | def approximation(A, p): 12 | U, s, V_T = np.linalg.svd(A) 13 | B = np.zeros(A.shape) 14 | for i in range(p): 15 | B += s[i]*U[:,i].reshape(-1, 1).dot(V_T[i, :].reshape(1, -1)) 16 | return B 17 | 18 | if __name__ == '__main__': 19 | # 例一： 20 | A = np.array( 21 | [ 22 | [0, 1], 23 | [1, 3/2], 24 | ] 25 | ) 26 | # 例二： 27 | # A = np.array( 28 | # [ 29 | # [3, 2, -2, -3], 30 | # [2, 4, -1, -5] 31 | # ] 32 | # ) 33 | 34 | # p为近似矩阵的秩，秩p<=r 35 | p = 1 36 | B = approximation(A, p) 37 | print(B) 38 | 39 | #可以看到最终得到的矩阵秩为1 40 | print(np.linalg.matrix_rank(B)) 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | # 调用api核对，和传统PCA比较 51 | # pca= PCA(n_components=2, svd_solver='auto') 52 | # B2 = pca.fit_transform(A) 53 | # print(B2) -------------------------------------------------------------------------------- /chapter13.最优化/13.1.不使用导数的无约束优化/黄金分割搜索.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-07-02 20:26:12 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-11 16:44:01 8 | ''' 9 | import numpy as np 10 | import math 11 | def gss(f, a, b, k): 12 | g = (math.sqrt(5)-1)/2 13 | # 计算x1和x2 14 | x1 = a + (1-g)*(b-a) 15 | x2 = a + g*(b-a) 16 | f1, f2 = f(x1), f(x2) 17 | for i in range(k): 18 | if f1 < f2 : 19 | # 依次更新b, x2, x1 20 | b = x2 21 | x2 = x1 22 | # 这里代码设计的很巧妙，b是已经更新后的新b 23 | x1 = a + (1-g)*(b-a) 24 | f2 = f1 25 | f1 = f(x1) 26 | else: 27 | a = x1 28 | x1 = x2 29 | x2 = a + g*(b-a) 30 | f1 = f2 31 | f2 = f(x2) 32 | y = (a+b)/2 33 | return(a, b), y 34 | if __name__ == '__main__': 35 | a, b = 0, 1 36 | k = 15 37 | (a,b), y = gss(lambda x: x**6-11*x**3+17*x**2-7*x+1, a, b, k) 38 | print("(%.4f, %.4f)"%(a, b), y) -------------------------------------------------------------------------------- /chapter13.最优化/13.2.使用导数的无约束优化/共轭梯度搜索.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-07-02 22:03:03 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-03 15:22:14 8 | ''' 9 | import numpy as np 10 | import math 11 | import torch 12 | 13 | #x.grad为Dy/dx(假设Dy为最后一个节点) 14 | def Conjugate_gradient_search(x0, k, f, alpha): #迭代k次,包括x0在内共k+1个数 15 | # 初始化计算图参数 16 | x = torch.tensor(x0, requires_grad=True) 17 | y = f(x) 18 | y.backward() 19 | r = -x.grad.detach().numpy() 20 | d = r.copy() 21 | for i in range(1, k+1): 22 | with torch.no_grad(): 23 | x.add_(alpha*torch.tensor(d)) 24 | y = f(x) 25 | y.backward() 26 | old_r = r.copy() 27 | r = -x.grad.detach().numpy() 28 | belta = r.dot(r)/old_r.dot(old_r) 29 | d = r + belta * d 30 | x_star = x.detach().numpy() 31 | minimum = f(x_star) 32 | # 最后得到的x即极值点的x 33 | return minimum, x_star 34 | 35 | # 多元函数，但非向量函数 36 | def f(x): 37 | return 5*x[0]**4 + 4*x[0]**2*x[1] - x[0]*x[1]**3 + 4*x[1]**4 - x[0] 38 | 39 | if __name__ == '__main__': 40 | x0 = np.array([1.0, -1.0]) 41 | k = 5 # k为迭代次数 42 | alpha = 0.01 # ita为迭代步长 43 | minimum, x_star = Conjugate_gradient_search(x0, k, f, alpha) 44 | print("the minimum is %.5f, the x_star is: ( %.5f, %.5f)"\ 45 | % (minimum, x_star[0], x_star[1])) -------------------------------------------------------------------------------- /chapter13.最优化/13.2.使用导数的无约束优化/最速下降法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 最速下降法，此处用到求多元函数的梯度 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-07-02 22:03:03 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-10-17 18:47:36 8 | ''' 9 | import numpy as np 10 | import math 11 | import torch 12 | 13 | #x.grad为Dy/dx(假设Dy为最后一个节点) 14 | def gradient_descent(x0, k, f, eta): #迭代k次,包括x0在内共k+1个数 15 | # 初始化计算图参数 16 | x = torch.tensor(x0, requires_grad=True) 17 | for i in range(1, k+1): 18 | y = f(x) 19 | y.backward() 20 | with torch.no_grad(): 21 | x -= eta*x.grad 22 | x.grad.zero_() #这里的梯度必须要清0，否则计算是错的 23 | x_star = x.detach().numpy() 24 | return f(x_star), x_star 25 | 26 | # 多元函数，但非向量函数 27 | def f(x): 28 | return 5*x[0]**4 + 4*x[0]**2*x[1] - x[0]*x[1]**3 + 4*x[1]**4 - x[0] 29 | 30 | if __name__ == '__main__': 31 | x0 = np.array([1.0, -1.0]) 32 | k = 25 # k为迭代次数 33 | eta = 0.01 # ita为迭代步长 34 | minimum, x_star = gradient_descent(x0, k, f, eta) 35 | print("the minimum is %.5f, the x_star is: ( %.5f, %.5f)"\ 36 | % (minimum, x_star[0], x_star[1])) -------------------------------------------------------------------------------- /chapter13.最优化/13.2.使用导数的无约束优化/牛顿法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 牛顿法，此处用到求多元函数的hessian矩阵 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-07-02 22:03:03 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-10-17 19:55:26 8 | ''' 9 | import numpy as np 10 | import math 11 | import torch 12 | from torch.autograd.functional import hessian 13 | from torch.autograd import grad 14 | # 多元函数，但非向量函数 15 | def f(x): 16 | return 5*x[0]**4 + 4*x[0]**2*x[1] - x[0]*x[1]**3 + 4*x[1]**4 - x[0] 17 | 18 | #x.grad为Dy/dx(假设Dy为最后一个节点) 19 | def gradient_descent(x0, k, f, alpha): #迭代k次,包括x0在内共k+1个数 20 | # 初始化计算图参数 21 | x = torch.tensor(x0, requires_grad=True) 22 | for i in range(1, k+1): 23 | y = f(x) 24 | y.backward() 25 | # 1阶导数可以直接访问x.grad 26 | # 高阶倒数我们需要调用functional.hession接口，这里返回hession矩阵 27 | # 注意，Hession矩阵要求逆 28 | H = hessian(f, x) 29 | with torch.no_grad(): 30 | # 如果为了避免求逆，也可以解线性方程组Hv = -x.grad，使x+v 31 | # v = np.linalg.solve(H, -x.grad) 32 | # x += torch.tensor(v) 33 | x -= torch.matmul(torch.inverse(H), x.grad) 34 | x.grad.zero_() 35 | x_star = x.detach().numpy() 36 | return f(x_star), x_star 37 | 38 | if __name__ == '__main__': 39 | x0 = np.array([1.0, 1.0]) 40 | k = 25 # k为迭代次数 41 | eta = 1 # 42 | alpha = 0 43 | # 基于牛顿法的推导，在最优解附近我们希望eta=1 44 | minimum, x_star = gradient_descent(x0, k, f, alpha) 45 | print("the minimum is %.5f, the x_star is: ( %.5f, %.5f)"\ 46 | % (minimum, x_star[0], x_star[1])) -------------------------------------------------------------------------------- /chapter2.方程组/2.1.高斯消元/朴素高斯消元.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 朴素法无法解决零主元和高斯消元问题，高斯消元都考虑方非奇异阵(肯定是方的) 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-08 17:26:11 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-29 16:22:24 8 | ''' 9 | import numpy as np 10 | eps = 1e-6 11 | # 消去步骤 12 | def gaussion_elimination(A, b): #假设A是方阵，A.shape[0] == A.shape[1], python中变量是传拷贝，数组等对象是传引用 13 | assert(A.shape[0] == A.shape[1]) 14 | for j in range(A.shape[1]): #消去第j列的数 15 | # abs(A[j ,j])为要消去的主元 16 | if abs(A[j, j]) < eps: 17 | raise ValueError("zero pivot encountered!") #无法解决零主元问题 18 | return 19 | # 消去主对角线以下的元素A[i, j] 20 | for i in range(j+1, A.shape[0]): 21 | mult_coeff = A[i, j]/A[j, j] 22 | # 对这A中这一行都进行更新 23 | for k in range(j, A.shape[1]): 24 | A[i, k] = A[i, k] - mult_coeff * A[j, k] 25 | b[i] = b[i] - mult_coeff * b[j] #二维的b取单dim1的索引即[1]这种含单个元素的列表 26 | 27 | def gaussion_putback(A, b): 28 | x = np.zeros((A.shape[0], 1)) 29 | for i in reversed(range(A.shape[0])): #算出第i个未知数 30 | for j in range(i+1, A.shape[1]): 31 | b[i] = b[i] - A[i, j] * x[j] 32 | x[i] = b[i] / A[i, i] 33 | return x 34 | 35 | if __name__ == '__main__': 36 | A = np.array( 37 | [ 38 | [1, 2, -1], 39 | [2, 1, -2], 40 | [-3, 1, 1] 41 | ] 42 | ) 43 | b = np.array( 44 | [ 45 | [3], 46 | [3], 47 | [-6] 48 | ] 49 | ) 50 | gaussion_elimination(A, b) 51 | x = gaussion_putback(A, b) 52 | print(x) 53 | #print(A, "\n", b) 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /chapter2.方程组/2.2.LU分解/LU分解及回代.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-05-22 21:53:43 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-25 09:45:50 8 | ''' 9 | import numpy as np 10 | from copy import deepcopy 11 | eps = 1e-6 12 | # 消去步骤 13 | def LU_decomposition(A): #假设A是方阵，A.shape[0] == A.shape[1], python中变量是传拷贝，数组等对象是传引用 14 | assert(A.shape[0] == A.shape[1]) 15 | U = deepcopy(A) 16 | L = np.zeros(A.shape, dtype=np.float32) 17 | for j in range(U.shape[1]): #消去第j列的数 18 | # abs(U[j ,j])为要消去的主元 19 | if abs(U[j, j]) < eps: 20 | raise ValueError("zero pivot encountered!") #无法解决零主元问题 21 | return 22 | L[j, j] = 1 23 | # 消去主对角线以下的元素A[i, j] 24 | for i in range(j+1, U.shape[0]): 25 | mult_coeff = U[i, j]/U[j, j] 26 | L[i, j] = mult_coeff 27 | # 对这A中这一行都进行更新 28 | for k in range(j, U.shape[1]): 29 | U[i, k] = U[i, k] - mult_coeff * U[j, k] 30 | 31 | return L, U 32 | 33 | #常规的上三角进行回代(此例中对角线不为0) 34 | def gaussion_putback_U(A, b): 35 | x = np.zeros((A.shape[0], 1), dtype=np.float32) 36 | for i in reversed(range(A.shape[0])): #算出第i个未知数 37 | for j in range(i+1, A.shape[1]): 38 | b[i] = b[i] - A[i, j] * x[j] 39 | x[i] = b[i] / A[i, i] 40 | return x 41 | 42 | #下三角进行回代(此例中对角线不为0) 43 | def gaussion_putback_L(A, b): 44 | x = np.zeros((A.shape[0], 1), dtype=np.float32) 45 | for i in range(A.shape[0]): #算出第i个未知数 46 | for j in range(i): 47 | b[i] = b[i] - A[i, j] * x[j] 48 | #草,如果b矩阵初始化时是整形，3-6.99999976 = ceil(-3.99999) = -3， 49 | # 直接给我向上取整(截断)约成整数了 50 | # if i == A.shape[0] - 1: 51 | # print(A[i, j], "----", x[j], "----", A[i, j]*x[j]) 52 | # print(b[i]) 53 | x[i] = b[i] / A[i, i] 54 | return x 55 | 56 | def LU_putback(L, U, b): 57 | # Ax = b => LUx = b ，令Ux = c 58 | # 解 Lc = b 59 | c = gaussion_putback_L(L, b) #上三角回代 60 | print(c) 61 | # 再解 Ux = c 62 | x = gaussion_putback_U(U, c) #下三角回代 63 | return x 64 | 65 | if __name__ == '__main__': 66 | A = np.array( 67 | [ 68 | [1, 2, -1], 69 | [2, 1, -2], 70 | [-3, 1, 1] 71 | ], 72 | dtype=np.float32 73 | ) 74 | b = np.array( 75 | [ 76 | [3], 77 | [3], 78 | [-6] 79 | ], 80 | dtype=np.float32 #注意，此处必须是浮点型，否则整形的话后面就自动舍入了 81 | ) 82 | # 单纯的LU分解过程不会对b有影响 83 | # 即消元与回代分离 84 | 85 | # 分解步骤 86 | L, U = LU_decomposition(A) # A=LU 87 | print(L) 88 | print(U) 89 | # 回代步骤 90 | x = LU_putback(L, U, b) 91 | print(x) 92 | #print(A, "\n", b) -------------------------------------------------------------------------------- /chapter2.方程组/2.4.PA=LU分解/PA=LU分解及回代.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: PA=LU 3 | 4 | Version: 1.0 5 | Author: ZhangHongYu 6 | Date: 2021-05-22 21:53:43 7 | LastEditors: ZhangHongYu 8 | LastEditTime: 2021-07-26 12:21:27 9 | ''' 10 | import numpy as np 11 | from copy import deepcopy 12 | eps = 1e-6 13 | 14 | # 消去第j个主元时，将第j列中最大的元素行和主元所在行交换位置 15 | # 以保证乘子A[i, j]/A[j, j]为小数 16 | def check_and_permute(P, A, j, b): 17 | max_row = j 18 | max_val = A[j, j] 19 | #每次置换对应左乘一个置换矩阵 20 | P_plus = np.eye(A.shape[0], dtype=np.float32) #初始化要更新的左乘置换矩阵 21 | for i in range(j+1, A.shape[0]): 22 | if abs(A[i, j]) > abs(max_val): 23 | max_val = A[i, j] 24 | max_row = i 25 | # 交换max_row行和主元所在的第j行 26 | # temp = deepcopy(A[max_row, :]) 27 | # A[max_row, :] = deepcopy(A[j, :]) 28 | # A[j, :] = temp 29 | 30 | # temp = deepcopy(b[max_row, :]) 31 | # b[max_row, :] = deepcopy(b[j, :]) 32 | # b[j, :] = temp 33 | # deepcopy()必须要，否则temp只是一个引用，改temp后A也会变 34 | A[j, :], A[max_row, :] = deepcopy(A[max_row, :]), deepcopy(A[j, :]) 35 | b[j, :], b[max_row, :] = deepcopy(b[max_row, :]), deepcopy(b[j, :]) 36 | #如果主元本身最大，即max_row = j，那相当于乘一个单位阵 37 | P_plus[j, j], P_plus[max_row, max_row] = 0, 0 38 | P_plus[j, max_row], P_plus[max_row, j] = 1, 1 39 | #print(P_plus) 40 | print("当次置换的矩阵为:", P_plus) 41 | P = np.matmul(P_plus, P) 42 | return P 43 | 44 | # 消去步骤 45 | def PA_LU_decomposition(A): #假设A是方阵，A.shape[0] == A.shape[1], python中变量是传拷贝，数组等对象是传引用 46 | assert(A.shape[0] == A.shape[1]) 47 | U = deepcopy(A) 48 | L = np.zeros(A.shape, dtype=np.float32) 49 | P = np.eye(A.shape[0], dtype=np.float32) #初始化置换矩阵为单位阵I 50 | for j in range(U.shape[1]): #消去第j列的数 51 | # abs(U[j ,j])为要消去的主元 52 | print("置换前的U：", U) 53 | P = check_and_permute(P, U, j, b) 54 | print("置换后的U", U) 55 | L[j, j] = 1 56 | # 消去主对角线以下的元素A[i, j] 57 | for i in range(j+1, U.shape[0]): 58 | mult_coeff = U[i, j]/U[j, j] 59 | L[i, j] = mult_coeff 60 | # 对这A中这一行都进行更新 61 | for k in range(j, U.shape[1]): 62 | U[i, k] = U[i, k] - mult_coeff * U[j, k] 63 | return P, L, U 64 | 65 | #常规的上三角进行回代(此例中对角线不为0) 66 | def gaussion_putback_U(A, b): 67 | x = np.zeros((A.shape[0], 1), dtype=np.float32) #注意float类型，否则又要截断成0 68 | for i in reversed(range(A.shape[0])): #算出第i个未知数 69 | for j in range(i+1, A.shape[1]): 70 | b[i] = b[i] - A[i, j] * x[j] 71 | x[i] = b[i] / A[i, i] 72 | return x 73 | 74 | #下三角进行回代(此例中对角线不为0) 75 | def gaussion_putback_L(A, b): 76 | x = np.zeros((A.shape[0], 1), dtype=np.float32) #注意float类型，否则又要截断成0 77 | for i in range(A.shape[0]): #算出第i个未知数 78 | for j in range(i): 79 | b[i] = b[i] - A[i, j] * x[j] 80 | #草,如果b矩阵初始化时是整形，3-6.99999976 = ceil(-3.99999) = -3， 81 | # 直接给我向上取整(截断)约成整数了 82 | # if i == A.shape[0] - 1: 83 | # print(A[i, j], "----", x[j], "----", A[i, j]*x[j]) 84 | # print(b[i]) 85 | x[i] = b[i] / A[i, i] 86 | return x 87 | 88 | def PA_LU_putback(P, L, U, b): 89 | 90 | # LUx = Pb ，令Ux = c 91 | # 解 Lc = Pb 92 | # 先要对b也进行置换 93 | b = np.matmul(P, b) #b存为矩阵的好处在此时就能体现 94 | 95 | c = gaussion_putback_L(L, b) #上三角回代 96 | # 再解 Ux = c 97 | x = gaussion_putback_U(U, c) #下三角回代 98 | return x 99 | 100 | if __name__ == '__main__': 101 | A = np.array( 102 | [ 103 | [1, 2, -1], 104 | [2, 1, -2], 105 | [-3, 1, 1] 106 | ], 107 | dtype=np.float32 108 | ) 109 | b = np.array( 110 | [ 111 | [3], 112 | [3], 113 | [-6] 114 | ], 115 | dtype=np.float32 #注意，此处必须是浮点型，否则整形的话后面就自动舍入了 116 | ) 117 | # PA=LU分解后对b也要置换 118 | # 还是尽量使消元与回代分离 119 | 120 | # 分解步骤 121 | # Ax = b => PAx = Pb => PA = LU 122 | P, L, U = PA_LU_decomposition(A) 123 | print(P) 124 | print(L) 125 | print(U) 126 | # 我感觉P, L, U都计算正确，但是x回代计算的结果就是不对 127 | # 回代步骤，要用P对b也进行置换 128 | x = PA_LU_putback(P, L, U, b) 129 | print(x) 130 | #print(A, "\n", b) -------------------------------------------------------------------------------- /chapter2.方程组/2.4.PA=LU分解/部分主元法高斯消元.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 部分主元法可解决淹没问题和0主元问题，高斯消元都考虑非奇异阵（肯定是方的） 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-05-29 15:49:30 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-29 21:05:20 8 | ''' 9 | import numpy as np 10 | from copy import deepcopy 11 | eps = 1e-6 12 | # 消去第j个主元时，将第j列中最大的元素行和主元所在行交换位置 13 | # 以保证乘子A[i, j]/A[j, j]为小数 14 | def check_and_permute(A, j, b): 15 | max_row = j 16 | max_val = A[j, j] 17 | for i in range(j+1, A.shape[0]): 18 | if abs(A[i, j]) > abs(max_val): 19 | max_val = A[i, j] 20 | max_row = i 21 | # 交换max_row行和主元所在的第j行 22 | # temp = deepcopy(A[max_row, :]) 23 | # A[max_row, :] = deepcopy(A[j, :]) 24 | # A[j, :] = temp 25 | 26 | # temp = deepcopy(b[max_row, :]) 27 | # b[max_row, :] = deepcopy(b[j, :]) 28 | # b[j, :] = temp 29 | # deepcopy()必须要，否则temp只是一个引用，改temp后A也会变 30 | A[j, :], A[max_row, :] = deepcopy(A[max_row, :]), deepcopy(A[j, :]) 31 | b[j, :], b[max_row, :] = deepcopy(b[max_row, :]), deepcopy(b[j, :]) 32 | # 消去步骤 33 | def gaussion_elimination(A, b): #假设A是方阵，A.shape[0] == A.shape[1], python中变量是传拷贝，数组等对象是传引用 34 | assert(A.shape[0] == A.shape[1]) 35 | for j in range(A.shape[1]): #消去第j列的数，即使用第j个主元 36 | print(A) 37 | check_and_permute(A, j, b) 38 | # 消去主对角线以下的元素A[i, j] 39 | for i in range(j+1, A.shape[0]): 40 | mult_coeff = A[i, j]/A[j, j] # 保证 0 < abs(multi_coeff) < 1，且主元系数A[j, j]决不为0 41 | # 对这A中这一行都进行更新 42 | for k in range(j, A.shape[1]): 43 | A[i, k] = A[i, k] - mult_coeff * A[j, k] 44 | b[i] = b[i] - mult_coeff * b[j] #二维的b取单dim1的索引即[1]这种含单个元素的列表 45 | 46 | def gaussion_putback(A, b): 47 | x = np.zeros((A.shape[0], 1)) 48 | for i in reversed(range(A.shape[0])): #算出第i个未知数 49 | for j in range(i+1, A.shape[1]): 50 | b[i] = b[i] - A[i, j] * x[j] 51 | x[i] = b[i] / A[i, i] #主元系数A[i, i]决不应为0，否则出错 52 | return x 53 | 54 | if __name__ == '__main__': 55 | A = np.array( 56 | [ 57 | [1, 2, -1], 58 | [2, 1, -2], 59 | [-3, 1, 1] 60 | ], 61 | dtype=np.float32 62 | ) 63 | b = np.array( 64 | [ 65 | [3], 66 | [3], 67 | [-6] 68 | ], 69 | dtype=np.float32 70 | ) 71 | gaussion_elimination(A, b) 72 | x = gaussion_putback(A, b) 73 | print(x) 74 | #　print(A, "\n", b) 75 | # 想答案是这样: 76 | # [[3. ] 77 | # [1.00000012] 78 | # [2.00000048]] 79 | # 需要换成float64提高精度或直接用decimal应对 -------------------------------------------------------------------------------- /chapter2.方程组/2.5.迭代方法/Gauss-Seidel方法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-08 17:26:11 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-30 16:31:29 8 | ''' 9 | import numpy as np 10 | import copy 11 | eps = 1e-6 12 | n = 6 13 | 14 | # 向量各元素之间互相依赖，不可并行/向量化 15 | def GaussSeidel(A, b): 16 | assert(A.shape[0] == A.shape[1] == b.shape[0]) 17 | x = np.zeros(b.shape, dtype=np.float32) 18 | # 迭代次数 19 | for t in range(n): 20 | for i in range(x.shape[0]): 21 | x[i] = b[i] 22 | for j in range(A.shape[1]): 23 | if j != i : 24 | x[i] -= A[i, j] * x[j] 25 | x[i] /= A[i][i] 26 | return x 27 | 28 | if __name__ == '__main__': 29 | # A一定要是主对角线占优矩阵 30 | A = np.array( 31 | [ 32 | [3, 1, -1], 33 | [2, 4, 1], 34 | [-1, 2, 5] 35 | ],dtype=np.float32 36 | ) 37 | b = np.array( 38 | [4, 1, 1],dtype=np.float32 39 | ) 40 | x = GaussSeidel(A, b) 41 | print(x) 42 | #print(A, "\n", b) 43 | -------------------------------------------------------------------------------- /chapter2.方程组/2.5.迭代方法/Jocobi迭代.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 小型矩阵向量化未必比naive快，但大型矩阵向量化远超naive 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-08 17:26:11 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-30 16:30:09 8 | ''' 9 | import numpy as np 10 | import time 11 | eps = 1e-6 12 | n = 6 #迭代次数 13 | #向量化实现 14 | def Jocobi(A, b): 15 | assert(A.shape[0] == A.shape[1] == b.shape[0]) 16 | x = np.zeros(b.shape, dtype=np.float32) 17 | d = np.diag(A) #diag即可提取A的对角线元素，也可构建对角阵 18 | R = A - np.diag(d) #r为余项 19 | # U = np.triu(R) #如果想获取不含对角线的L和U需如此，直接np.triu()得到的是含对角线的 20 | # L = np.tril(R) 21 | # 迭代次数 22 | for t in range(n): 23 | x = (b-np.matmul(R, x))/d 24 | return x 25 | 26 | #普通实现 27 | def Jocobi_naive(A, b): 28 | assert(A.shape[0] == A.shape[1] == b.shape[0]) 29 | x = np.zeros(b.shape, dtype=np.float32) 30 | # 迭代次数 31 | for t in range(n): 32 | #普通实现 33 | for i in range(x.shape[0]): 34 | val = b[i] 35 | for j in range(A.shape[1]): 36 | if j != i : 37 | val -= A[i, j] * x[j] 38 | x[i] = val/A[i][i] 39 | return x 40 | 41 | if __name__ == '__main__': 42 | # A一定要是主对角线占优矩阵 43 | # A = np.array( 44 | # [ 45 | # [3, 1, -1], 46 | # [2, 4, 1], 47 | # [-1, 2, 5] 48 | # ],dtype=np.float32 49 | # ) 50 | A = np.eye(1000, dtype=np.float32) 51 | 52 | # b = np.array( 53 | # [4, 1, 1],dtype=np.float32 54 | # ) 55 | b = np.zeros((1000,), np.float32) 56 | 57 | start1 = time.time() 58 | x1 = Jocobi_naive(A, b) 59 | end1 = time.time() 60 | print("time: %.10f" % (end1-start1)) 61 | print(x1) 62 | 63 | 64 | start2 = time.time() 65 | x2 = Jocobi(A, b) 66 | end2 = time.time() 67 | print("time: %.10f" % (end2 - start2)) 68 | print(x2) 69 | #print(A, "\n", b) 70 | -------------------------------------------------------------------------------- /chapter2.方程组/2.5.迭代方法/SOR方法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-05-30 14:54:36 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-30 16:31:41 8 | ''' 9 | ''' 10 | Descripttion: 11 | Version: 1.0 12 | Author: ZhangHongYu 13 | Date: 2021-05-30 14:48:57 14 | LastEditors: ZhangHongYu 15 | LastEditTime: 2021-05-30 14:53:27 16 | ''' 17 | ''' 18 | Descripttion: 19 | Version: 1.0 20 | Author: ZhangHongYu 21 | Date: 2021-03-08 17:26:11 22 | LastEditors: ZhangHongYu 23 | LastEditTime: 2021-05-30 14:47:51 24 | ''' 25 | import numpy as np 26 | eps = 1e-6 27 | n = 6 28 | # 向量各元素之间互相依赖，不可并行/向量化 29 | def SOR(A, b, w): 30 | assert(A.shape[0] == A.shape[1] == b.shape[0]) 31 | x = np.zeros(b.shape, dtype=np.float32) 32 | # 迭代次数 33 | for t in range(n): 34 | for i in range(x.shape[0]): 35 | val = b[i] 36 | for j in range(A.shape[1]): 37 | if j != i : 38 | val -= A[i, j] * x[j] 39 | x[i] = (1-w)*x[i] + w*(val/A[i][i]) 40 | print(x) 41 | return x 42 | 43 | if __name__ == '__main__': 44 | # A一定要是主对角线占优矩阵 45 | A = np.array( 46 | [ 47 | [3, 1, -1], 48 | [2, 4, 1], 49 | [-1, 2, 5] 50 | ],dtype=np.float32 51 | ) 52 | b = np.array( 53 | [4, 1, 1],dtype=np.float32 54 | ) 55 | w = 1.1 56 | x = SOR(A, b, w) 57 | print(x) 58 | #print(A, "\n", b) 59 | -------------------------------------------------------------------------------- /chapter2.方程组/2.5.迭代方法/三种迭代方法对比.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-05-30 14:57:56 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-30 15:01:49 8 | ''' 9 | import numpy as np 10 | eps = 1e-6 11 | n = 6 12 | # 消去步骤 13 | def Jocobi(A, b): 14 | assert(A.shape[0] == A.shape[1] == b.shape[0]) 15 | x = np.zeros(b.shape, dtype=np.float32) 16 | # 迭代次数 17 | for t in range(n): 18 | for i in range(x.shape[0]): 19 | val = b[i] 20 | for j in range(A.shape[1]): 21 | if j != i : 22 | val -= A[i, j] * x[j] 23 | x[i] = val/A[i][i] 24 | return x 25 | 26 | 27 | def GaussSeidel(A, b): 28 | assert(A.shape[0] == A.shape[1] == b.shape[0]) 29 | x = np.zeros(b.shape, dtype=np.float32) 30 | # 迭代次数 31 | for t in range(n): 32 | for i in range(x.shape[0]): 33 | x[i] = b[i] 34 | for j in range(A.shape[1]): 35 | if j != i : 36 | x[i] -= A[i, j] * x[j] 37 | x[i] /= A[i][i] 38 | return x 39 | 40 | 41 | def SOR(A, b, w): 42 | assert(A.shape[0] == A.shape[1] == b.shape[0]) 43 | x = np.zeros(b.shape, dtype=np.float32) 44 | # 迭代次数 45 | for t in range(n): 46 | for i in range(x.shape[0]): 47 | val = b[i] 48 | for j in range(A.shape[1]): 49 | if j != i : 50 | val -= A[i, j] * x[j] 51 | x[i] = (1-w)*x[i] + w*(val/A[i][i]) 52 | return x 53 | 54 | 55 | if __name__ == '__main__': 56 | # A一定要是主对角线占优矩阵 57 | A = np.array( 58 | [ 59 | [3, -1, 0, 0, 0, 1/2], 60 | [-1, 3, -1, 0, 1/2, 0], 61 | [0, -1, 3, -1, 0, 0], 62 | [0, 0, -1, 3, -1, 0], 63 | [0, 1/2, 0, -1, 3, -1], 64 | [1/2, 0, 0, 0, -1, 3] 65 | ],dtype=np.float32 66 | ) 67 | b = np.array( 68 | [5/2, 3/2, 1, 1, 3/2, 5/2],dtype=np.float32 69 | ) 70 | w = 1.1 71 | x1 = Jocobi(A, b) 72 | x2 = GaussSeidel(A, b) 73 | x3 = SOR(A, b, w) 74 | print(x1) 75 | print(x2) 76 | print(x3) 77 | #print(A, "\n", b) -------------------------------------------------------------------------------- /chapter2.方程组/2.5.迭代方法/稀疏矩阵Jocobi迭代(没搞完).py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 稀疏矩阵sparse库似乎只定义了稀疏矩阵.dot(普通向量) 3 | 以及一些稀疏矩阵的方法(triu,eye之类) 4 | Version: 1.0 5 | Author: ZhangHongYu 6 | Date: 2021-03-08 17:26:11 7 | LastEditors: ZhangHongYu 8 | LastEditTime: 2021-05-30 17:10:42 9 | ''' 10 | import numpy as np 11 | from scipy.sparse import csr_matrix 12 | from scipy.sparse import diags 13 | import time 14 | eps = 1e-6 15 | n = 6 #迭代次数 16 | #向量化普通实现 17 | def Jocobi(A, b): 18 | assert(A.shape[0] == A.shape[1] == b.shape[0]) 19 | x = np.zeros(b.shape, dtype=np.float32) 20 | d = np.diag(A) #diag即可提取A的对角线元素，也可构建对角阵 21 | R = A - np.diag(d) #r为余项 22 | # U = np.triu(R) #如果想获取不含对角线的L和U需如此，直接np.triu()得到的是含对角线的 23 | # L = np.tril(R) 24 | # 迭代次数 25 | for t in range(n): 26 | x = (b-np.matmul(R, x))/d 27 | return x 28 | 29 | #向量化稀疏矩阵实现 30 | def Jocobi_sparse(A:csr_matrix, b): #此处csr_matrix不会执行，仅仅参数提醒而已 31 | #assert(A.shape[0] == A.shape[1] == b.shape[0]) 32 | x = np.zeros(b.shape, dtype=np.float32) 33 | d = np.diag(A) #diag即可提取A的对角线元素，也可构建对角阵 34 | R = A - np.diag(d) #r为余项 35 | R = csr_matrix(R) #采用(row, col，val)形式的元组存放 36 | # 迭代次数 37 | for t in range(n): 38 | x = (b-R.dot(x))/d 39 | return x 40 | 41 | if __name__ == '__main__': 42 | 43 | A = np.eye(10000, dtype=np.float32,) #可选择k让对角线偏移 44 | 45 | b = np.ones((10000,), np.float32) 46 | 47 | start1 = time.time() 48 | x1 = Jocobi(A, b) 49 | end1 = time.time() 50 | print("time: %.10f" % (end1-start1)) 51 | print(x1) 52 | 53 | 54 | start2 = time.time() 55 | x2 = Jocobi_sparse(A, b) 56 | 57 | end2 = time.time() 58 | print("time: %.10f" % (end2 - start2)) 59 | print(x2) 60 | 61 | -------------------------------------------------------------------------------- /chapter2.方程组/2.6.用于对称正定矩阵的方法/共轭梯度法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-05 18:56:58 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-06 16:31:46 8 | ''' 9 | import numpy as np 10 | from copy import deepcopy 11 | A = np.array( 12 | [ 13 | [2, 2], 14 | [2, 5], 15 | ] 16 | ) 17 | assert(A.shape[0] == A.shape[1]) 18 | n = A.shape[0] 19 | b = np.array([6, 3]) 20 | x = [0, 0] #初始估计解向量 21 | d = r = b - A.dot(x) 22 | for k in range(n): 23 | # 余项为0，可以返回 24 | if r.any() == 0 : 25 | break 26 | A_d = np.matmul(A, d) #先把Ad先计算出来 27 | # 计算更新步长参数alpha 28 | alpha = r.dot(r) / np.matmul(d, A_d) 29 | # 更新解向量 30 | x = x + alpha * d 31 | pre_r = np.copy(r) 32 | # 更新余项 33 | r = r - alpha * A_d 34 | # 计算更新步长参数belta 35 | belta = r.dot(r)/pre_r.dot(pre_r) 36 | # 更新d 37 | d = r + belta * d 38 | print(x) 39 | 40 | -------------------------------------------------------------------------------- /chapter2.方程组/2.6.用于对称正定矩阵的方法/对角矩阵定义.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-06 16:45:50 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-06 16:55:19 8 | ''' 9 | import numpy as np 10 | n = 10 11 | # 前者乃对角线往上数的第10阶 12 | # 后者乃对角线往下数的第10阶(也就是-10阶) 13 | A = np.diag(np.sqrt(range(1, n+1)))\ 14 | + np.diag(np.cos(range(1, n-10+1)),10)\ 15 | + np.diag(np.cos(range(1, n-10+1)), -10) 16 | print(A) -------------------------------------------------------------------------------- /chapter2.方程组/2.6.用于对称正定矩阵的方法/楚列斯基分解.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-05 18:56:58 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-05 19:33:08 8 | ''' 9 | import numpy as np 10 | from copy import deepcopy 11 | A = np.array( 12 | [ 13 | [4, -2, 2], 14 | [-2, 2, -4], 15 | [2, -4, 11] 16 | ] 17 | ) 18 | R = np.zeros(A.shape) 19 | n = 3 20 | for k in range(n): 21 | if A[k, k]<0: 22 | raise ValueError("对角线元素不能为负!") 23 | break 24 | R[k, k] = np.sqrt(A[k, k]) 25 | u_T = deepcopy(A[k, k+1:n]/R[k, k]).reshape(1, -1) 26 | R[k, k+1:n] = u_T 27 | A[k+1:n, k+1:n] = A[k+1:n, k+1:n] - (u_T.T).dot(u_T) 28 | print(R) 29 | 30 | print(R.T.dot(R)) 31 | 32 | -------------------------------------------------------------------------------- /chapter2.方程组/2.6.用于对称正定矩阵的方法/预条件共轭梯度法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-05 18:56:58 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-06 16:37:42 8 | ''' 9 | import numpy as np 10 | from copy import deepcopy 11 | A = np.array( 12 | [ 13 | [2, 2], 14 | [2, 5], 15 | ] 16 | ) 17 | assert(A.shape[0] == A.shape[1]) 18 | n = A.shape[0] 19 | b = np.array([6, 3]) 20 | x = [0, 0] # 初始估计解向量 21 | r = b - A.dot(x) 22 | M = np.diag(np.diag(A)) # M 为A的对角矩阵 23 | z = np.matmul(np.matrix(M).I.A, r) 24 | d = np.copy(z) 25 | for k in range(n): 26 | # 余项为0，可以返回 27 | if r.any() == 0 : 28 | break 29 | A_d = np.matmul(A, d) 30 | alpha = r.dot(z)/d.dot(A_d) 31 | x = x + alpha * d 32 | r_pre = np.copy(r) 33 | r = r - alpha * A_d 34 | z_pre = np.copy(z) 35 | #如果用matrix dot ndarray会返回二维matrix 36 | #如此对结果需用 .A取出ndarray并reshape到一维 37 | z = np.matmul(np.matrix(M).I.A, r) 38 | belta = r.dot(z)/r_pre.dot(z_pre) 39 | d = z + belta*d 40 | print(x) 41 | 42 | -------------------------------------------------------------------------------- /chapter2.方程组/2.7 非线性方程组/Broyden方法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: Broyden方法 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-19 19:22:08 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-24 19:36:09 8 | ''' 9 | import numpy as np 10 | import math 11 | import torch 12 | import scipy 13 | from scipy import linalg 14 | # 注意这里x表x(k+1)，x0表x(k)，迭代小技巧可以使用 15 | # 一定要注意x0和x分开存，且若是直接赋值不能用x=x0，一定要用x = x0.copy()!!! 16 | # 最后还要注意，如果是向量求外积一定要先reshape成矩阵形式 17 | def Broyden(x0, K, F): #迭代k次,包括x0在内共k+1个数 18 | # 初始向量 19 | x0 = x0.copy() 20 | x = x0.copy() 21 | # 初始矩阵，我们将其初始化为对角阵(保证可逆) 22 | A = np.eye(x0.shape[0]) 23 | for k in range(K): 24 | # 这里A做为Jocobian矩阵的估计 25 | # 此处A必须可逆，不可逆那么算法就会出问题 26 | if np.linalg.det(A)==0: 27 | raise RuntimeError("A is a singular matrix!") 28 | x = x0 - np.matmul(np.matrix(A).I.A, F(x0)) 29 | delta_f = F(x) - F(x0) 30 | delta_x = x - x0 31 | A = A + (delta_f - A.dot(delta_x)).reshape(-1, 1).dot(delta_x.reshape(1, -1))/delta_x.dot(delta_x) 32 | x0 = x.copy() 33 | return x 34 | def F(x:np.ndarray): 35 | return np.array([x[1]-x[0]**3, x[0]**2+x[1]**2-1]) 36 | if __name__ == '__main__': 37 | # 初始解，初始值要设成(1, 1)才能结果正确 38 | x0 = np.array([1, 1], dtype=np.float32) 39 | # 迭代次数 40 | K = 10 41 | res = Broyden(x0, K, F) 42 | print(res) -------------------------------------------------------------------------------- /chapter2.方程组/2.7 非线性方程组/Broyden方法2.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 多变量牛顿方法 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-08 15:57:56 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-24 19:01:22 8 | ''' 9 | import numpy as np 10 | import math 11 | import scipy 12 | from scipy import linalg 13 | # >>> matrix = np.array([[2, 3], [4, 5]]) 14 | # >>> a = np.array([2, 3]) (2, )可以右乘矩阵，自动做为(1, 2)对齐 15 | # >>> a.dot(matrix) 16 | # array([16, 21]) 17 | # 但是，如果是求外积一定要先reshape成矩阵形式 18 | # 这个方法有一个好，就是不用求逆 19 | # 注意这里x表x(k+1)，x0表x(k)，迭代小技巧可以使用 20 | # 一定要注意x0和x分开存，且赋值不能x=x0，一定要用x = x0.copy()!!! 21 | # 否则你这就变成失败的类SOR方法了，即错误的异步处理，算出来的值还一样? 22 | def Broyden_2(x0, K, F): #迭代k次,包括x0在内共k+1个数 23 | # 初始向量 24 | x0 = x0.copy() 25 | x = x0.copy() 26 | # 初始矩阵，难以计算导数，用单位矩阵I初始化 27 | B = np.eye(x0.shape[0], dtype=np.float32) 28 | for k in range(K): 29 | x = x0 - np.matmul(B, F(x0)) 30 | delta_f = F(x) - F(x0) 31 | delta_x = x - x0 32 | B = B + (delta_x - B.dot(delta_f)).reshape(-1, 1).dot(delta_x.reshape(1, -1)).dot(B)/delta_x.dot(B).dot(delta_f) 33 | x0 = x.copy() 34 | return x 35 | def F(x:np.ndarray): 36 | return np.array([x[1]-x[0]**3, x[0]**2+x[1]**2-1]) 37 | if __name__ == '__main__': 38 | # 初始解为[1, 1]结果才正确 39 | x0 = np.array([1, 1], dtype=np.float32) 40 | # 迭代次数 41 | K = 10 42 | res = Broyden_2(x0, K, F) 43 | print(res) -------------------------------------------------------------------------------- /chapter2.方程组/2.7 非线性方程组/多变量牛顿方法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 多变量牛顿方法 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-03-08 15:57:56 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-10-17 19:54:23 8 | ''' 9 | import numpy as np 10 | import math 11 | import torch 12 | import scipy 13 | from scipy import linalg 14 | from torch.autograd.functional import jacobian 15 | def multi_variable_newton(x0, K, F): #迭代k次,包括x0在内共k+1个数 16 | # 初始x张量 17 | x = torch.tensor(x0, requires_grad=True) 18 | for k in range(K): 19 | y = F(x) 20 | # 因为只能由标量函数backward，此处需要传入参数 21 | y.backward(torch.ones_like(x), retain_graph=True) 22 | # 计算Jocobian矩阵 23 | J = jacobian(F, x) 24 | with torch.no_grad(): 25 | # 如果为了避免求逆，也可以解线性方程组Jv = -y，使x+v 26 | # 注意，此处y是一维，则返回的v也是一维 27 | v = np.linalg.solve(J, -y) 28 | x.add_(torch.tensor(v)) 29 | # 这等价于 30 | # x.sub_(torch.matmul(torch.inverse(J), y)) 31 | x.grad.zero_() 32 | return x.detach().numpy() 33 | 34 | def F(x): 35 | # 书上定义的向量函数F(x): F(u, v)=(v-u**3, u**2+v**2-1) 36 | # 注意，这里不能重新定义torch.tensor对象，否则梯度无法传播 37 | # 故我们这里重新定义F(x): F(u, v)=(v-5u, 2u+2v) 38 | # 可以写成矩阵乘的形式，可以使梯度正常传播 39 | A = torch.tensor([[-5, 1], [2, 2]], dtype=torch.float32) 40 | return torch.matmul(A, x) 41 | 42 | if __name__ == '__main__': 43 | # 初始解 44 | x0 = np.array([1, 2], dtype=np.float32) 45 | # 迭代次数 46 | K = 10 47 | res = multi_variable_newton(x0, K, F) 48 | print(res) -------------------------------------------------------------------------------- /chapter4.最小二乘/QR分解_经典Gram正交化.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 09:45:36 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-22 10:26:58 8 | ''' 9 | import numpy as np 10 | def QR_Gram_Schmidt(A): 11 | R = np.zeros((A.shape[1], A.shape[1])) 12 | Q = np.zeros(A.shape) 13 | for j in range(A.shape[1]): 14 | # 默认copy为深拷贝，此处y为A的一维向量切片 15 | y = A[:, j].copy() 16 | # 生成R中对角线以上的元素 17 | for i in range(j): 18 | R[i, j] = Q[:, i].dot(A[:, j]) 19 | y = y - R[i, j] * Q[:, i] 20 | R[j, j] = np.linalg.norm(y) 21 | Q[:, j] = y/R[j, j] 22 | return Q, R 23 | 24 | if __name__ == '__main__': 25 | # 前提: A中列向量线性无关 26 | A = np.array( 27 | [ 28 | [1, -4], 29 | [2, 3], 30 | [2, 2] 31 | ] 32 | ) 33 | Q, R = QR_Gram_Schmidt(A) 34 | print(Q, "\n\n", R) 35 | -------------------------------------------------------------------------------- /chapter4.最小二乘/QR分解实现最小二乘.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 09:45:36 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-22 11:25:20 8 | ''' 9 | import numpy as np 10 | 11 | 12 | if __name__ == '__main__': 13 | # 前提: A中列向量线性无关 14 | A = np.array( 15 | [ 16 | [1, -4], 17 | [2, 3], 18 | [2, 2] 19 | ] 20 | ) 21 | b = np.array([-3, 15, 9]) 22 | Q, R = np.linalg.qr(A, mode="complete") 23 | # 最小二乘误差||e||2 = ||(0, 0, 3)||2 = 3 24 | e = Q.T.dot(b)[-1] 25 | print("最小二乘误差", e) 26 | # 注意在解方程的时候一定要使“上部分”相等 27 | x = np.linalg.solve(R[:A.shape[1], :], Q.T.dot(b)[:A.shape[1]]) 28 | print(x) 29 | -------------------------------------------------------------------------------- /chapter4.最小二乘/完全QR分解_经典Gram正交化.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 09:45:36 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-22 10:55:51 8 | ''' 9 | import numpy as np 10 | def Complete_QR_Gram_Schmidt(A): 11 | # R只是大小扩充了，剩余部分仍然是0没有计算 12 | R = np.zeros(A.shape) 13 | # Q的剩余部分用dummy向量An+1, An+2, ..., Am计算 14 | Q = np.zeros((A.shape[0], A.shape[0])) 15 | # 先找到正交单位向量q1, q2, ..., qn 16 | for j in range(A.shape[1]): 17 | # 默认copy为深拷贝，此处y为A的一维向量切片 18 | y = A[:, j].copy() 19 | # 生成R中对角线以上的元素 20 | for i in range(j): 21 | R[i, j] = Q[:, i].dot(A[:, j]) 22 | y = y - R[i, j] * Q[:, i] 23 | R[j, j] = np.linalg.norm(y) 24 | Q[:, j] = y/R[j, j] 25 | 26 | # 使用扩充的dummy向量An+1, An+2, ..., Am 27 | # 来扩充找到qn+1, qn+2, ..., qm，对于扩充的Q没有对应的r值 28 | for j in range(A.shape[1], A.shape[0]): 29 | # 加上处于第 j + 1 位置的dummy向量Aj， 30 | # 一般是(1, 0, 0), (0, 1, 0)... 31 | A_plus = np.zeros((A.shape[0],)) 32 | A_plus[j-A.shape[1]] = 1 33 | y = A_plus.copy() 34 | for i in range(j): 35 | y = y - Q[:, i].dot(A_plus)*Q[:, i] 36 | Q[:, j] = y/np.linalg.norm(y) 37 | 38 | return Q, R 39 | 40 | if __name__ == '__main__': 41 | # 前提: A中列向量线性无关 42 | A = np.array( 43 | [ 44 | [1, -4], 45 | [2, 3], 46 | [2, 2] 47 | ] 48 | ) 49 | Q, R = Complete_QR_Gram_Schmidt(A) 50 | print(Q, "\n\n", R) 51 | -------------------------------------------------------------------------------- /chapter4.最小二乘/改进Gram正交化.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-22 09:45:36 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-22 15:15:21 8 | ''' 9 | import numpy as np 10 | def QR_Gram_Schmidt(A): 11 | R = np.zeros((A.shape[1], A.shape[1])) 12 | Q = np.zeros(A.shape) 13 | for j in range(A.shape[1]): 14 | # 默认copy为深拷贝，此处y为A的一维向量切片 15 | y = A[:, j].copy() 16 | # 生成R中对角线以上的元素 17 | for i in range(j): 18 | R[i, j] = Q[:, i].dot(A[:, j]) 19 | y = y - R[i, j] * Q[:, i] 20 | R[j, j] = np.linalg.norm(y) 21 | Q[:, j] = y/R[j, j] 22 | return Q, R 23 | 24 | def Modified_QR_Gram_Schmidt(A): 25 | R = np.zeros((A.shape[1], A.shape[1])) 26 | Q = np.zeros(A.shape) 27 | for j in range(A.shape[1]): 28 | # 默认copy为深拷贝，此处y为A的一维向量切片 29 | y = A[:, j].copy() 30 | # 生成R中对角线以上的元素 31 | for i in range(j): 32 | # 这里直接算的是qi在y上的投影，而不是之前的Aj 33 | # y是已经减掉投影后的 34 | R[i, j] = Q[:, i].dot(y) 35 | y = y - R[i, j] * Q[:, i] 36 | R[j, j] = np.linalg.norm(y) 37 | Q[:, j] = y/R[j, j] 38 | return Q, R 39 | epsilon = 10e-10 40 | 41 | if __name__ == '__main__': 42 | # 前提: A中列向量线性无关 43 | # A = np.array( 44 | # [ 45 | # [1, -4], 46 | # [2, 3], 47 | # [2, 2] 48 | # ] 49 | # ) 50 | # Q, R = Modified_QR_Gram_Schmidt(A) 51 | # print(Q, "\n\n", R) 52 | # 前提: A中列向量线性无关 53 | A = np.array( 54 | [ 55 | [1, 1, 1], 56 | [epsilon, 0, 0], 57 | [0, epsilon, 0], 58 | [0, 0, epsilon] 59 | ] 60 | ) 61 | 62 | Q1, R1 = Modified_QR_Gram_Schmidt(A) 63 | Q, R = QR_Gram_Schmidt(A) 64 | print("改进施密特方法的Q和R:") 65 | print(Q1, "\n\n", R1) 66 | print("经典施密特方法的Q和R:") 67 | print(Q, "\n\n", R) 68 | print("改进施密特方法条件数:\n") 69 | print(np.linalg.cond(Q1)) 70 | print("经典施密特方法条件数:\n") 71 | print(np.linalg.cond(Q)) 72 | -------------------------------------------------------------------------------- /chapter4.最小二乘/最小二乘_多项式.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-09-19 19:53:53 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2022-02-13 09:26:45 8 | ''' 9 | import numpy as np 10 | if __name__ == '__main__': 11 | x = np.array( 12 | [ 13 | [-1], 14 | [0], 15 | [1], 16 | [2] 17 | ] 18 | ) 19 | y = [1, 0, 0, -2] 20 | # 对数据矩阵x预处理，从第三列开始依次计算出第二列的次方值(还是拟合平面上的点，不过扩充了) 21 | # 此处A一共三列，最高次数有2次，即抛物线 22 | A = np.concatenate([np.ones([x.shape[0], 1]), x, x**2], axis=1) 23 | AT_A = A.T.dot(A) 24 | AT_y = A.T.dot(y) 25 | c_bar = np.linalg.solve(AT_A, AT_y) # 该API AT_y是一维/二维的都行 26 | print("最小二乘估计得到的参数:", c_bar) 27 | # 条件数 28 | print("条件数:", np.linalg.cond(AT_A)) 29 | -------------------------------------------------------------------------------- /chapter4.最小二乘/最小二乘_直线.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-09-19 19:53:53 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2022-02-13 09:26:03 8 | ''' 9 | import numpy as np 10 | if __name__ == '__main__': 11 | x = np.array( 12 | [ 13 | [-1], 14 | [0], 15 | [1], 16 | [2] 17 | ] 18 | ) 19 | y = [1, 0, 0, -2] 20 | # 对数据矩阵x预处理，即扩充常数1的列 21 | # 此处A一共两列，最高次数只有1次 22 | A = np.concatenate([np.ones([x.shape[0], 1]), x], axis=1) 23 | AT_A = A.T.dot(A) 24 | AT_y = A.T.dot(y) 25 | c_bar = np.linalg.solve(AT_A, AT_y) 26 | print("最小二乘估计得到的参数:", c_bar) 27 | # 条件数 28 | print("条件数:", np.linalg.cond(AT_A)) -------------------------------------------------------------------------------- /chapter4.最小二乘/最小二乘_迭代法.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2022-02-12 18:34:54 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2022-02-12 18:36:15 8 | ''' 9 | import numpy as np 10 | import torch 11 | 12 | def mse_loss(y_pred, y): 13 | m = y.shape[0] 14 | return 1/m*torch.square(y-y_pred).sum() 15 | 16 | def linear_f(X, w): 17 | return torch.matmul(X, w) 18 | 19 | # 之前实现的梯度下降法，做了一些小修改 20 | def gradient_descent(X, w, y, n_iter, eta, loss_func, f): 21 | # 初始化计算图参数，注意：这里是创建新对象，非参数引用 22 | w = torch.tensor(w, requires_grad=True) 23 | X = torch.tensor(X) 24 | y = torch.tensor(y) 25 | for i in range(1, n_iter+1): 26 | y_pred = f(X, w) 27 | loss_v = loss_func(y_pred, y) 28 | loss_v.backward() 29 | with torch.no_grad(): 30 | w.sub_(eta*w.grad) 31 | w.grad.zero_() 32 | w_star = w.detach().numpy() 33 | return w_star 34 | 35 | # 本模型按照多分类架构设计 36 | def linear_model( 37 | X, y, n_iter=200, eta=0.001, loss_func=mse_loss, optimizer=gradient_descent): 38 | # 初始化模型参数 39 | # 我们使w和b融合，X后面添加一维 40 | X = np.concatenate([np.ones([X.shape[0], 1]), X], axis=1) 41 | w = np.zeros((X.shape[1],), dtype=np.float64) 42 | # 调用梯度下降法对函数进行优化 43 | # 这里采用单次迭代对所有样本进行估计，后面我们会介绍小批量法减少时间复杂度 44 | w_star = optimizer(X, w, y, n_iter, eta, mse_loss, linear_f) 45 | return w_star 46 | 47 | if __name__ == '__main__': 48 | X = np.array( 49 | [ 50 | [-1], 51 | [0], 52 | [1], 53 | [2] 54 | ], dtype=np.float32 55 | ) 56 | y = np.array([1, 0, 0, -2], dtype=np.float32) 57 | n_iter, eta = 200, 0.1 58 | w_star = linear_model(X, y, n_iter, eta, mse_loss, gradient_descent) 59 | print("最小二乘估计得到的参数:", w_star) -------------------------------------------------------------------------------- /chapter4.最小二乘/最小二乘_迭代法_带正则项.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-20 11:37:16 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2022-02-12 18:44:17 8 | ''' 9 | import numpy as np 10 | import torch 11 | 12 | def mse_loss(y_pred, y, w, lam, reg): 13 | # 这里y是创建新的对象，这里将y变为(batch_size. 1)形式 14 | m = y.shape[0] 15 | if reg == 'L2': 16 | reg_item = lam*torch.square(w).sum() 17 | elif reg == 'L1': 18 | reg_item = lam*torch.norm(w, p=1).sum() 19 | else: 20 | reg_item = 0 21 | return 1/m*torch.square(y-y_pred).sum() + reg_item 22 | 23 | def linear_f(X, w): 24 | return torch.matmul(X, w) 25 | 26 | # 之前实现的梯度下降法，做了一些小修改 27 | def gradient_descent(X, w, y, n_iter, eta, lam, reg, loss_func, f): 28 | # 初始化计算图参数，注意：这里是创建新对象，非参数引用 29 | w = torch.tensor(w, requires_grad=True) 30 | X = torch.tensor(X) 31 | y = torch.tensor(y) 32 | for i in range(1, n_iter+1): 33 | y_pred = f(X, w) 34 | loss_v = loss_func(y_pred, y, w, lam, reg) 35 | loss_v.backward() 36 | with torch.no_grad(): 37 | w.sub_(eta*w.grad) 38 | w.grad.zero_() 39 | w_star = w.detach().numpy() 40 | return w_star 41 | 42 | # 本模型按照多分类架构设计 43 | def linear_model( 44 | X, y, n_iter=200, eta=0.001, lam=0.01, reg="L2", loss_func=mse_loss, optimizer=gradient_descent): 45 | # 初始化模型参数 46 | # 我们使w和b融合，X后面添加一维 47 | X = np.concatenate([np.ones([X.shape[0], 1]), X], axis=1) 48 | w = np.zeros((X.shape[1],), dtype=np.float64) 49 | # 调用梯度下降法对函数进行优化 50 | # 这里采用单次迭代对所有样本进行估计，后面我们会介绍小批量法减少时间复杂度 51 | w_star = optimizer(X, w, y, n_iter, eta, lam, reg, mse_loss, linear_f) 52 | return w_star 53 | 54 | if __name__ == '__main__': 55 | X = np.array( 56 | [ 57 | [-1], 58 | [0], 59 | [1], 60 | [2] 61 | ], dtype=np.float32 62 | ) 63 | y = np.array([1, 0, 0, -2], dtype=np.float32) 64 | n_iter, eta, lam = 200, 0.1, 0.1 65 | reg = "L2" 66 | w_star = linear_model(X, y, n_iter, eta, lam, reg, mse_loss, gradient_descent) 67 | print("最小二乘估计得到的参数:", w_star) 68 | -------------------------------------------------------------------------------- /chapter4.最小二乘/范德蒙德实现最小二乘.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-20 15:09:04 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-06-22 14:42:23 8 | ''' 9 | import numpy as np 10 | if __name__ == '__main__': 11 | 12 | x = np.array([(2+i/5) for i in range(11)]).reshape(-1, 1) 13 | y = 1 + x + x**2 + x**3 + x**4 + x**5 + x**6 + x**7 #(1被广播到向量的每个维度) 14 | A = np.concatenate([x**i for i in range(8)], axis=1) 15 | # print(A) 16 | # AT_A = A.T.dot(A) 17 | # AT_y = A.T.dot(y) 18 | # c_bar = np.linalg.solve(AT_A, AT_y) 19 | # print("最小二乘估计得到的参数:", c_bar) 20 | # # 条件数 21 | # print("条件数:", np.linalg.cond(AT_A)) 22 | 23 | Q, R = np.linalg.qr(A, mode="complete") 24 | # 最小二乘误差||e||2 = ||(0, 0, 3)||2 = 3 25 | e = Q.T.dot(y)[-1] 26 | print("最小二乘误差", e[0], "\n") 27 | # 注意在解方程的时候一定要使“上部分”相等 28 | x = np.linalg.solve(R[:A.shape[1], :], Q.T.dot(y)[:A.shape[1]]) 29 | print("最小二乘解\n:", x) 30 | print("条件数:", np.linalg.cond(R.T.dot(R))) 31 | -------------------------------------------------------------------------------- /chapter4.最小二乘/范德蒙德矩阵.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-06-20 15:09:04 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-07-24 15:21:12 8 | ''' 9 | import numpy as np 10 | if __name__ == '__main__': 11 | 12 | x = np.array([(2+i/5) for i in range(11)]).reshape(-1, 1) 13 | y = 1 + x + x**2 + x**3 + x**4 + x**5 + x**6 + x**7 #(1被广播到向量的每个维度) 14 | A = np.concatenate([x**i for i in range(8)], axis=1) 15 | AT_A = A.T.dot(A) 16 | AT_y = A.T.dot(y) 17 | c_bar = np.linalg.solve(AT_A, AT_y) 18 | print("最小二乘估计得到的参数:", c_bar) 19 | # 条件数 20 | print("条件数:", np.linalg.cond(AT_A)) -------------------------------------------------------------------------------- /chapter5.数值微分和积分/前向自动微分.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | class Var: 4 | def __init__(self, val, deriv=1.0): 5 | self.val = val 6 | self.deriv = deriv 7 | 8 | def __add__(self, other): 9 | if isinstance(other, Var): 10 | val = self.val + other.val 11 | deriv = self.deriv + other.deriv 12 | else: 13 | val = self.val + other 14 | deriv = self.deriv 15 | return Var(val, deriv) 16 | 17 | def __radd__(self, other): 18 | return self + other 19 | 20 | def __sub__(self, other): 21 | if isinstance(other, Var): 22 | val = self.val - other.val 23 | deriv = self.deriv - other.deriv 24 | else: 25 | val = self.val - other 26 | deriv = self.deriv 27 | return Var(val, deriv) 28 | 29 | def __rsub__(self, other): 30 | val = other - self.val 31 | deriv = - self.deriv 32 | return Var(val, deriv) 33 | 34 | def __mul__(self, other): 35 | if isinstance(other, Var): 36 | val = self.val * other.val 37 | deriv = self.val * other.deriv + self.deriv * other.val 38 | else: 39 | val = self.val * other 40 | deriv = self.deriv * other 41 | return Var(val, deriv) 42 | 43 | def __rmul__(self, other): 44 | return self * other 45 | 46 | def __truediv__(self, other): 47 | if isinstance(other, Var): 48 | val = self.val / other.val 49 | deriv = (self.deriv * other.val - self.val * other.deriv)/other.val**2 50 | else: 51 | val = self.val / other 52 | deriv = self.deriv / other 53 | return Var(val, deriv) 54 | 55 | def __rtruediv__(self, other): 56 | val = other / self.val 57 | deriv = other * 1/self.val**2 58 | return Var(val, deriv) 59 | 60 | def __repr__(self): 61 | return "value: {}\t deriv: {}".format(self.val, self.deriv) 62 | 63 | 64 | def exp(f: Var): 65 | return Var(math.exp(f.val), math.exp(f.val) * f.deriv) 66 | 67 | 68 | fx = lambda x: exp(x*x - x)/x 69 | 70 | df = fx(Var(2.0)) 71 | print(df) 72 | 73 | # value: 3.694528049465325 deriv: 9.236320123663312 74 | -------------------------------------------------------------------------------- /chapter9.随机数和应用/最小标准生成器.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-05-22 20:57:46 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-22 21:04:04 8 | ''' 9 | seed = 1 10 | n = 100 #生成100个随机数 11 | x = seed #初始化整数种子,不算在随机数内 12 | a = int(pow(7, 5)) 13 | m = int(pow(2, 31))-1 14 | for i in range(n): 15 | x = a * x % m 16 | ui = x/m 17 | print(x, "---", ui) 18 | 19 | -------------------------------------------------------------------------------- /chapter9.随机数和应用/蒙特卡洛1型问题-随机数近似曲线下方面积.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-05-22 21:09:23 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-22 21:13:00 8 | ''' 9 | #用随机数近似 y = x**2在[0, 1]下的面积 10 | 11 | import random 12 | sum_v = 0.0 13 | n = 30 14 | for i in range(n): 15 | ui = random.random() 16 | sum_v += ui**2 17 | print("近似面积为:", sum_v/n) 18 | 19 | 20 | # 用随机数近似 -------------------------------------------------------------------------------- /chapter9.随机数和应用/蒙特卡洛2型问题-随机数近似图形面积.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-05-22 21:15:21 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-22 21:19:19 8 | ''' 9 | import random 10 | n= 10000 11 | cnt = 0 12 | for i in range(n): 13 | x = random.random() 14 | y = random.random() 15 | if 4*(2*x-1)**4+8*(2*y-1)**8 < 1 + 2*(2*y-1)**3*(3*x-2)**2: #注意python连乘性和优先级 16 | cnt += 1 17 | print("面积近似值为:", cnt/n) -------------------------------------------------------------------------------- /chapter9.随机数和应用/随机游走.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Descripttion: 3 | Version: 1.0 4 | Author: ZhangHongYu 5 | Date: 2021-05-22 21:20:28 6 | LastEditors: ZhangHongYu 7 | LastEditTime: 2021-05-22 21:24:14 8 | ''' 9 | import random 10 | import matplotlib.pyplot as plt 11 | # 随机游走步数 12 | #一维随机游走 13 | t = 10 14 | w = 1 15 | plots = [w] 16 | for i in range(1, t+1): 17 | if random.random() > 0.5: 18 | w = w + 1 19 | else: 20 | w = w- 1 21 | plots.append(w) 22 | plt.plot(range(t+1), plots) 23 | plt.show() 24 | 25 | -------------------------------------------------------------------------------- /pic/numericalanalysis_cover.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orion-orion/NumericalAnalysis-Python/f1eab2e84658eca326b836336421af52a44b62ef/pic/numericalanalysis_cover.jpeg --------------------------------------------------------------------------------