├── B站-刘二大人课件 ├── Lecture_01_Overview.pdf ├── Lecture_02_Linear_Model.pdf ├── Lecture_03_Gradient_Descent.pdf ├── Lecture_04_Back_Propagation.pdf ├── Lecture_05_Linear_Regression_with_PyTorch.pdf ├── Lecture_06_Logistic_Regression.pdf ├── Lecture_07_Multiple_Dimension_Input.pdf ├── Lecture_08_Dataset_and_Dataloader.pdf ├── Lecture_09_Softmax_Classifier.pdf ├── Lecture_10_Basic_CNN.pdf ├── Lecture_11_Advanced_CNN.pdf ├── Lecture_12_Basic_RNN.pdf └── Lecture_13_RNN_Classifier.pdf ├── CNN ├── CNN.md └── CNN_MINST.ipynb ├── ConvLSTM.py ├── LSTM ├── LSTM_Regression.py └── LSTM_airplane_forcast.ipynb ├── LinearNetwork └── multiple_dimension_diabetes.ipynb ├── README.md ├── RNN ├── GRU_Classifier.ipynb ├── GRU_Classifier.py ├── RNN.md ├── RNN_Regression.py └── RNNcell.ipynb ├── Transformer.md ├── dataset ├── airplane_data.csv ├── diabetes.csv.gz ├── mnist │ ├── .DS_Store │ ├── processed │ │ ├── test.pt │ │ └── training.pt │ └── raw │ │ ├── t10k-images-idx3-ubyte │ │ ├── t10k-labels-idx1-ubyte │ │ ├── train-images-idx3-ubyte │ │ └── train-labels-idx1-ubyte ├── names_test.csv.gz └── names_train.csv.gz └── softmax_classifier.ipynb /B站-刘二大人课件/Lecture_01_Overview.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_01_Overview.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_02_Linear_Model.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_02_Linear_Model.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_03_Gradient_Descent.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_03_Gradient_Descent.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_04_Back_Propagation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_04_Back_Propagation.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_05_Linear_Regression_with_PyTorch.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_05_Linear_Regression_with_PyTorch.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_06_Logistic_Regression.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_06_Logistic_Regression.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_07_Multiple_Dimension_Input.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_07_Multiple_Dimension_Input.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_08_Dataset_and_Dataloader.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_08_Dataset_and_Dataloader.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_09_Softmax_Classifier.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_09_Softmax_Classifier.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_10_Basic_CNN.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_10_Basic_CNN.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_11_Advanced_CNN.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_11_Advanced_CNN.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_12_Basic_RNN.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_12_Basic_RNN.pdf -------------------------------------------------------------------------------- /B站-刘二大人课件/Lecture_13_RNN_Classifier.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/B站-刘二大人课件/Lecture_13_RNN_Classifier.pdf -------------------------------------------------------------------------------- /CNN/CNN.md: -------------------------------------------------------------------------------- 1 | # 卷积神经网络 2 | 3 | ## 概述 4 | 5 | 卷积神经网络——Conventional Neural Network，一般用于图像分类、图像检索、图像语义分割、物体检测等计算机视觉问题 6 | 7 | ### 前馈运算 8 | 9 | 输入原始数据（RGB图像、原始音频数据等），经过**卷积、池化、激活函数**等一系列层叠操作，最后输出目标结果（分类、回归等结果）『或者说得到一个目标函数』。 10 | 11 | 前馈运算 12 | 13 | ### 反馈运算 14 | 15 | 然后计算预测结果和实际结果的Loss，凭借**反向传播算法**将误差由最后一层逐层向前**反馈**，更新每层参数，并在更新参数之后再次前馈，如此往复，直到网络模型收敛。 16 | 17 | > 最小化损失函数更新参数：随机梯度下降（Stochastic Gradient Descent，SDG）、误差反向传播 18 | > 19 | > 对于大规模的问题，批处理随机梯度下降（mini-batch SGD），随机选择n个样本处理完成前馈运算和反馈运算就是一个“**批处理过程**”（mini-batch），不同批处理之间无放回的处理完所有样本就是“**一轮**”（epoch），其中批处理样本的大小（batch-size）不宜过小，上限取决于硬件资源。 20 | 21 | ### 基本术语 22 | 23 | - **Channel**：输入数据（图像）个数 24 | - **Convolution Layer**：经过卷积之后得到的中间结果 25 | - **Pooling Layer**：经过池化之后得到的中间结果 26 | - **Flatten**：将每个图像比如6*6，展开成一个vector（size = 36\*1） 27 | - **Padding and Stride**：图像周围补充0，每次filter移动步长，涉及到超参数的调整 28 | - **Softmax**：激活函数$S_i = \frac{e^i}{\Sigma{e^j}}$：将一些输入映射为0-1之间的实数，并且归一化保证和为1 29 | - **Fully Connected Layer** 30 | 31 | 32 | 33 | ## 基本操作 34 | 35 | 输入数据一般是三维张量**tensor**，比如n个video frames（也可以叫做channel），每个frame大小是6\*6，然后经过**filter**（比如是3\*3大小，这参数是通过训练得到的，可以使用比如SGD），在经过**Max/ Average Pooling**（比如大小是2*2），经过多次处理最后**Flatten**成vector，然后输入到一个全连接神经网络中，其中可以使用**softmax**，然后输出结果 36 | 37 | CNN结构 38 | 39 | ### 卷积 40 | 41 | 这个过程就是**特征提取**，通俗点说就是对于冗余信息压缩提纯，进行这个过 程我们需要学习一些参数（这些参数组合成一个**卷积核**，Filter Matrix），然后对输入的图像进行扫描，每次移动步长**stride**，另外如果需要充分利用图像信息也可以**padding** 42 | 43 | convolution 44 | 45 | 46 | 47 | - 每个filter可以探测到一些小特征，这也是选择其中参数的依据 48 | - 原始图像经过filter之后可以看到一些特征，对应在灰度图中可能是一条斜杠 49 | 50 | property 51 | 52 | - filter的个数决定了卷积操作输出的层数，所以每次经过卷积之后输出结果一般都会变胖 53 | 54 | 卷积层逐渐变胖 55 | 56 | - 卷积过程中的参数是共享的，**Shared Weights**，所以需要的参数就会变少 57 | 58 | Shared Weights 59 | 60 | 61 | 62 | - [一图理解卷积操作](https://yuzy007.github.io/2019/02/14/CNN/) 63 | 64 | CONV_layer_02 65 | 66 | ### 池化 67 | 68 | 对卷积层输出的特征图**进一步特征提取**，主要分为**Max Pooling**、**Average Pooling** 69 | 70 | Pooling 71 | 72 | > 池化实际上是一种”降采样“（down-sampling）操作。一般有三种功能： 73 | > 74 | > 1. 特征不变性 75 | > 2. 特征降维 76 | > 3. 防止过拟合 77 | 78 | ### 输出 79 | 80 | 之前得到的所有特征需要使用Flatten将特征矩阵转化成$vector$，最后将$vector$接到全连接神经网络上，全连接层则起到将学到的特征表示映射到样本的标记空间，使用激活函数**softmax**得到分类或者回归的结果 81 | 82 | Flatten 83 | 84 | 85 | 86 | ## 参考文献 87 | 88 | 1. b站 李宏毅 https://www.bilibili.com/video/BV1JE411g7XF?p=17 89 | 90 | 2. b站 [阿力阿哩哩](https://space.bilibili.com/299585150) https://www.bilibili.com/video/BV1j7411f7Ru?t=665 91 | 92 | 3. 魏秀参《解析深度学习——卷积神经网络原理与视觉实践》（开源电子版） 93 | 94 | -------------------------------------------------------------------------------- /CNN/CNN_MINST.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.6.12-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3.6.12 64-bit ('pytorch_dl': conda)", 19 | "metadata": { 20 | "interpreter": { 21 | "hash": "6b962ed846ab7a2f9c4286c5c9f447c6ba721d3b496bd62adf19906831909870" 22 | } 23 | } 24 | } 25 | }, 26 | "nbformat": 4, 27 | "nbformat_minor": 2, 28 | "cells": [ 29 | { 30 | "source": [ 31 | "来源：[b站刘二大人](https://www.bilibili.com/video/BV1Y7411d7Ys)\n", 32 | "# 1. prepare dataset" 33 | ], 34 | "cell_type": "markdown", 35 | "metadata": {} 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 1, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "import torch\n", 44 | "from torchvision import transforms\n", 45 | "from torchvision import datasets\n", 46 | "from torch.utils.data import DataLoader\n", 47 | "import torch.nn.functional as F\n", 48 | "import torch.optim as optim\n", 49 | " \n", 50 | "batch_size = 64\n", 51 | "transform = transforms.Compose([\n", 52 | " transforms.ToTensor(), \n", 53 | " transforms.Normalize((0.1307,), (0.3081,))\n", 54 | "])\n", 55 | " \n", 56 | "train_dataset = datasets.MNIST(root='../dataset/mnist/', train=True, download=True, transform=transform)\n", 57 | "train_loader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size)\n", 58 | "test_dataset = datasets.MNIST(root='../dataset/mnist/', train=False, download=True, transform=transform)\n", 59 | "test_loader = DataLoader(test_dataset, shuffle=False, batch_size=batch_size)" 60 | ] 61 | }, 62 | { 63 | "source": [ 64 | "# 2. design model using class" 65 | ], 66 | "cell_type": "markdown", 67 | "metadata": {} 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 2, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "class Net(torch.nn.Module):\n", 76 | " def __init__(self):\n", 77 | " super(Net, self).__init__()\n", 78 | " # Conv2d 三个参数分别是输入通道数、输出通道数，卷积核大小\n", 79 | " self.conv1 = torch.nn.Conv2d(1, 10, kernel_size=5)\n", 80 | " self.conv2 = torch.nn.Conv2d(10, 20, kernel_size=5)\n", 81 | " self.pooling = torch.nn.MaxPool2d(2)\n", 82 | " self.fc = torch.nn.Linear(320, 10)\n", 83 | " \n", 84 | " \n", 85 | " def forward(self, x):\n", 86 | " # flatten data from (n,1,28,28) to (n, 784)\n", 87 | " batch_size = x.size(0)\n", 88 | " x = F.relu(self.pooling(self.conv1(x)))\n", 89 | " x = F.relu(self.pooling(self.conv2(x)))\n", 90 | " x = x.view(batch_size, -1) # -1 此处自动算出的是320\n", 91 | " x = self.fc(x)\n", 92 | " \n", 93 | " return x\n", 94 | " \n", 95 | " \n", 96 | "model = Net()" 97 | ] 98 | }, 99 | { 100 | "source": [ 101 | "# 3. construct loss and optimizer" 102 | ], 103 | "cell_type": "markdown", 104 | "metadata": {} 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 3, 109 | "metadata": {}, 110 | "outputs": [ 111 | { 112 | "output_type": "stream", 113 | "name": "stdout", 114 | "text": [ 115 | "Net(\n (conv1): Conv2d(1, 10, kernel_size=(5, 5), stride=(1, 1))\n (conv2): Conv2d(10, 20, kernel_size=(5, 5), stride=(1, 1))\n (pooling): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n (fc): Linear(in_features=320, out_features=10, bias=True)\n)\nconv1_weight: torch.Size([10, 1, 5, 5])\nconv2_bias: torch.Size([10])\n" 116 | ] 117 | } 118 | ], 119 | "source": [ 120 | "criterion = torch.nn.CrossEntropyLoss()\n", 121 | "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)\n", 122 | "\n", 123 | "print(model)\n", 124 | "print(\"conv1_weight: \", model.conv1.weight.shape)\n", 125 | "print(\"conv2_bias: \", model.conv1.bias.shape)" 126 | ] 127 | }, 128 | { 129 | "source": [ 130 | "# 4. training cycle forward, backward, update" 131 | ], 132 | "cell_type": "markdown", 133 | "metadata": {} 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 4, 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "output_type": "stream", 142 | "name": "stdout", 143 | "text": [ 144 | "[1, 300] loss: 0.656\n", 145 | "[1, 600] loss: 0.191\n", 146 | "[1, 900] loss: 0.146\n", 147 | "accuracy on test set: 96 % \n", 148 | "[2, 300] loss: 0.118\n", 149 | "[2, 600] loss: 0.102\n", 150 | "[2, 900] loss: 0.088\n", 151 | "accuracy on test set: 97 % \n", 152 | "[3, 300] loss: 0.083\n", 153 | "[3, 600] loss: 0.078\n", 154 | "[3, 900] loss: 0.074\n", 155 | "accuracy on test set: 97 % \n", 156 | "[4, 300] loss: 0.070\n", 157 | "[4, 600] loss: 0.062\n", 158 | "[4, 900] loss: 0.064\n", 159 | "accuracy on test set: 98 % \n", 160 | "[5, 300] loss: 0.054\n", 161 | "[5, 600] loss: 0.061\n", 162 | "[5, 900] loss: 0.055\n", 163 | "accuracy on test set: 98 % \n", 164 | "[6, 300] loss: 0.051\n", 165 | "[6, 600] loss: 0.052\n", 166 | "[6, 900] loss: 0.050\n", 167 | "accuracy on test set: 98 % \n", 168 | "[7, 300] loss: 0.047\n", 169 | "[7, 600] loss: 0.048\n", 170 | "[7, 900] loss: 0.042\n", 171 | "accuracy on test set: 98 % \n", 172 | "[8, 300] loss: 0.042\n", 173 | "[8, 600] loss: 0.043\n", 174 | "[8, 900] loss: 0.041\n", 175 | "accuracy on test set: 98 % \n", 176 | "[9, 300] loss: 0.040\n", 177 | "[9, 600] loss: 0.037\n", 178 | "[9, 900] loss: 0.040\n", 179 | "accuracy on test set: 98 % \n", 180 | "[10, 300] loss: 0.037\n", 181 | "[10, 600] loss: 0.039\n", 182 | "[10, 900] loss: 0.035\n", 183 | "accuracy on test set: 98 % \n" 184 | ] 185 | } 186 | ], 187 | "source": [ 188 | "def train(epoch):\n", 189 | " running_loss = 0.0\n", 190 | " for batch_idx, data in enumerate(train_loader, 0):\n", 191 | " inputs, target = data\n", 192 | " optimizer.zero_grad()\n", 193 | " \n", 194 | " outputs = model(inputs)\n", 195 | " loss = criterion(outputs, target)\n", 196 | " loss.backward()\n", 197 | " optimizer.step()\n", 198 | " \n", 199 | " running_loss += loss.item()\n", 200 | " if batch_idx % 300 == 299:\n", 201 | " print('[%d, %5d] loss: %.3f' % (epoch+1, batch_idx+1, running_loss/300))\n", 202 | " running_loss = 0.0\n", 203 | " \n", 204 | "ACC = []\n", 205 | "def test():\n", 206 | " correct = 0\n", 207 | " total = 0\n", 208 | " with torch.no_grad():\n", 209 | " for data in test_loader:\n", 210 | " images, labels = data\n", 211 | " outputs = model(images)\n", 212 | " _, predicted = torch.max(outputs.data, dim=1)\n", 213 | " total += labels.size(0)\n", 214 | " correct += (predicted == labels).sum().item()\n", 215 | " print('accuracy on test set: %d %% ' % (100*correct/total))\n", 216 | " ACC.append(100*correct/total)\n", 217 | " \n", 218 | "EPOCH = 10\n", 219 | "if __name__ == '__main__':\n", 220 | " for epoch in range(EPOCH):\n", 221 | " train(epoch)\n", 222 | " test()" 223 | ] 224 | }, 225 | { 226 | "source": [ 227 | "# 5. show ACC_EPOCH" 228 | ], 229 | "cell_type": "markdown", 230 | "metadata": {} 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": 5, 235 | "metadata": {}, 236 | "outputs": [ 237 | { 238 | "output_type": "display_data", 239 | "data": { 240 | "text/plain": "
", 241 | "image/svg+xml": "\n\n\n\n \n \n \n \n 2021-03-12T17:47:22.727640\n image/svg+xml\n \n \n Matplotlib v3.3.2, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", 242 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtnElEQVR4nO3deXxV9Z3/8dcnYU+QLRAIEcKqIAoYpKKCQemGjhbGtaOjVqVVtNDpr63tdFptpx3r2MWp1g23VmtQ0WqrdSkCiooYEBBBSgz7vgRCQkK2z++Pe5AkhCUmJ+cmeT8fjzzuuWe7n/MluW/O+jV3R0RE5HglRF2AiIg0LQoOERGpEwWHiIjUiYJDRETqRMEhIiJ10irqAhpDSkqKZ2RkRF1GvRQVFZGUlBR1GXFD7XGI2qI6tUd19WmPRYsW7XT37jXHt4jgyMjIICcnJ+oy6mXu3LlkZWVFXUbcUHscoraoTu1RXX3aw8zW1TZeh6pERKROFBwiIlInCg4REakTBYeIiNSJgkNEROpEwSEiInWi4BARkTppEfdxiIi0NFv2FvPQW3mMatfwXWcoOEREmpGte0v4w9xcshduwHFOGN6GCxr4MxQcIiLNwLaCEu6f+yl/Xrieykrn0lHpTB0/kNylCxv8sxQcIiJN2PZ9QWC8v57ySueS09O55byBnNi1AwC5IXymgkNEpAnase8AD877lD8tWEd5pTNpZG9uPW8gfbuF/4BHBYeISBOyq/AAD76Vxx/fW0tpeSVfG9mbb583iIyUxnsisIJDRKQJ2F1UyoNvfcof313HgfIKLh4R28Po3z250WtRcIiIxLH8olIefjuPJ95dy/6yCi4ansat5w1iYI/GD4yDFBwiInFoz/5SZry9hsffXUtRaTkXnNqLaecPYlBqx6hLU3CIiMSTvfvLeGR+Ho+9s5Z9B4LAmDCIwXEQGAcpOERE4kBBSRmPzl/DI/PXsK+knK8O68m0CYM4uecJUZd2mFCDw8ymATcCBjzs7r8zsxHAA0A7oBy42d0X1ljuJGBmlVH9gZ8Ey98erHNHMO1H7v5KmNshIhKWfSVlPPbOWma8nUdBSTlfGprK9AmDGZoWf4FxUGjBYWbDiH3BjwZKgVfN7GXgLuAOd/+7mU0M3mdVXdbdVwEjgvUkApuAF6rM8lt3vzus2kWk6XF3Zq/czjsby+iwZjf9UpJISW6DmUVdWq0KD5Tz+DtrePjtNewtLmPCkFSmTxjEsN6doi7tmMLc4xgCLHD3/QBmNg+YBDhwMEo7AZuPsZ7zgU/dvdZO00VENu8p5sd/Wc6bn2wH4LHl7wHQsW0r+nVPol/KoZ/+KclkpHSgY7vWkdRadKCcx99dy8Nv57Fnfxnnn9yD6RMGc2p6/AfGQebe8E9OBDCzIcCLwBigGJgN5AB/AF4jdvgqATjraKFgZo8Ci9393uD97cC1QEGwvu+6e34ty00BpgCkpqZmZmdnN9SmRaKwsJDk5Oguv4s3ao9DWnJbVLozb0M5M1eVUglcMqgNg5MPsM/bsXW/s7Wokm1Fztb9lewqdqp+23Vqa/TsYKQmJdAzyejZIYGeSQl072C0Tmj4vZQD5c7s9WX8fU0Z+8rgtO6JfG1ga/p3Smzwz6qqPr8f48ePX+Tuo2qODy04AMzsemAqUAisIBYgicA8d59lZpcBU9x9whGWb0Nsj+QUd98WjEsFdhLbc/k50Mvdv3G0OkaNGuU5OTkNtFXRmDt3LllZWVGXETfUHoe01LZYt6uIH8xaxoK83Zw1oBt3Tj6NPt06HLE9SsoqWL97P3k7ilizs4g1OwuD1yJ2FpZ+Nl+CQXqXDof2UII9loxuSaR1bk9iHUOluLSCPy1Yy4Pz8thVVMq4wd35zoRBjOzTpb5NcFzq8/thZrUGR6gnx939EeCRoIBfAhuB/wGmBbM8C8w4yiq+SmxvY1uVdX42bGYPA39r4LJFJI5VVDqPvbOGu19fReuEBO6cfCqXn3HiMc9ltGudyODUjrVe1rq3uIy1QYjk7TwULDlrd1NUWvHZfG1aJZDR7WCoJNM/JemzQ2HdkqqfTykureCp99fxwLxP2VlYythBKUyfMJjMvo0TGGEK+6qqHu6+3cz6AJOJHba6FTgXmAucB6w+yiquBJ6usc5e7r4leDsJWN7QdYtIfFq9bR/fn7WMD9fv4fyTe/Dfk4bRq1P7eq+3U/vWDD+xM8NP7FxtvLuzY9+BKmFSRN6OInK3F/LmJ9spqzh0xKZju1axIElJonvHtvxlyWZ27DvA2QO7cf+EwZyR0bXedcaLsO/jmGVm3YAyYKq755vZjcA9ZtYKKCE4D2FmacAMd58YvO8AfBH4Zo113hVc0uvA2lqmi0gzU1ZRyQNzP+X3b+aS1DaRe64YwUXD00K/YsrM6HFCO3qc0I4z+3erNq28opJNe4pjobLjULB8sDafzXuL+UK/rtx75Ui+UGO55iDsQ1Vjaxk3H8isZfxmYGKV9/uBw1rc3a9u4DJFJI4t37SX7z23jJVbCrjwtF7cftEppCS3jbosWiUm0LdbEn27JTH+pOrTyisqaZWYEE1hjUB3jotIXCopq+Ce2at56K08uia14cGrM/nyKT2jLuu4NOfQAAWHiMShnLW7+f6sZeTtKOLSzHR+fMFQOnWI5r4LOZyCQ0TiRtGBcv73tVU88d5a0jq154/fGM24wd2jLktqUHCISFyYv3ontz2/jI35xVwzpi/f/8rJJLXVV1Q80r+KiERqb3EZv3x5JTNzNtA/JYlnvzWmWV262hwpOEQkMm+s2MaP//IRO/Yd4FvnDmD6hEG0ax3uIzik/hQcItLodhUe4Pa/ruCvSzdzcs+OPPzvozgtvXPUZclxUnCISKNxd/66bAu3v/Qx+0rK+M6EwdyUNYA2rZr35avNjYJDRBrFtoIS/vOF5fxj5TaGp3firkvO5KSe8dMdqhw/BYeIhMrdeSZnA//98kpKyyv50cST+cbZ/Zr9TXLNmYJDpIkqq6hk6YY9zN9URqvVO+ndpT29OrWLq5PLG3bv54fPf8T83J2M7teVX/3rafRLSYq6LKknBYdIE+Hu5G4vZH7uTt7J3cmCvN0UHigHYMZH7382X0pyW3p3bkda5/b07tyetOAnvUvstUuH1qE/HLCy0nnivbXc9eoqEgx+/rVh/NvoPiSE0EGSND4Fh0gc215Qwjuf7uTt1bGw2FZwAIC+3Tpw0Yg0xg5MYc+6lWQMGc7mPcVs2lP82euqbfuYs2o7JWWV1dbZvnUiaUGwpHdpT1qnWKD07hILmp6d2tG6HoeRcrcX8oNZy1i0Lp9zB3fnl5NPpXfn+j/6XOKHgkMkjhQeKGfhml3MX72L+bk7+Oe2QgC6dGjNWQNTOCf4ObFrh8+WmbtrFWMG1P7obncnf38Zm/JjYXIwWA6Gy8otBdV6vwMwg9SO7Ujr3I7eXTrEXmvsvXRqf/hzo8orKnno7Tx+94/VtG+dyK8vHc7k03uHvncjjU/BIRKh8opKlm7cw/zVu3gndyeL1+dTXum0bZXA6H5dmXx6OucMTGForxM+12EeM6NrUhu6JrXh1PROtc5TUlYRhEkJm/bsZ9Oekliw5BezbOMeXlteQmlF9b2Wjm1bfbaXkta5Hb06tefvy7ewfFMBXx3WkzsuPoUeHdt9rjaR+KfgEGlE7s6nO4qYv3oH83N38X7eLvYdKMcMhqV14sZx/TlnYAqZfbs02knudq0T6d89mf7dk2udXlnp7Cw8UGOPpYSN+bHhRevy2VtcRkpyG/7wb6cz8dRejVK3RCfsrmOnATcCBjzs7r8Leu97AGgHlAM3u/vCWpZdC+wDKoDygx2mm1lXYCaQQawHwMvcPT/M7RCpj+37Sng3d9dn5ym2FpQA0KdrBy4cnsbYQSmM6d+NLkltIq60dgkJh3rBG9mn9v6yCw+U0yYxQTfytRChBYeZDSMWGqOBUuBVM3sZuAu4w93/bmYTg/dZR1jNeHffWWPcbcBsd7/TzG4L3v8gjG0Q+TyKDpSzcM3uz65++mTrPgA6d2jN2QNSODs4T9GnW4djrKnpSNZTbFuUMP+1hwALgi5gMbN5wCRifYWfEMzTCdhcx/VezKGgeQKYi4JDIlReUcmyTXuZv3on83N38uH6fMoqnDatEjgjows/+MrJsfMUaSeQqMtRpRkwdw9nxWZDgBeBMUAxMBvIAf4AvEbs8FUCcJa7r6tl+TVAPrGgedDdHwrG73H3zlXmy3f3w/afzWwKMAUgNTU1Mzs7u0G3r7EVFhaSnFz7MeiWKOr2OFDhvLOpnOU7K1i5u4Li8tgvdJ8TEjilWyKndEtkUJcE2iSGHxRRt0W8UXtUV5/2GD9+/KKDpwmqCi04AMzsemAqUAisIBYgicA8d59lZpcBU9x9Qi3Lprn7ZjPrAbwB3Orubx1vcFQ1atQoz8nJabgNi8DcuXPJysqKuoy4EWV7LN+0l2nZH/LpjiLSu7Rn7KDY4aezBqTQNYLzFPrdqE7tUV192sPMag2OUA9MuvsjwCNBAb8ENgL/A0wLZnkWmHGEZTcHr9vN7AVi50reAraZWS9332JmvYDtYW6DyEGVlc6M+Xn872ur6JrUhj9dP5qxg9StqbQ8oV4CEewtYGZ9gMnA08TOaZwbzHIesLqW5ZLMrOPBYeBLwPJg8kvANcHwNcQOh4mEauveEq5+9H1++connHdyD16dNk6hIS1W2JdCzDKzbkAZMNXd883sRuAeM2sFlBCchzCzNGCGu08EUoEXgjtOWwF/dvdXg3XeCTwTHAZbD1wa8jZIC/fq8q3c9vwyDpRVcufkU7n8jBN1N7S0aGEfqhpby7j5QGYt4zcDE4PhPGD4Eda5Czi/YSsVOdz+0nJ+9tcVZH+wgVN7d+KeK0Yc8SY5kZZEF1+L1GLZxj1Mz17Cml1F3JQ1gO9MGKyb20QCCg6RKioqnQff+pTfvP5Pundsy59vOPOIDxAUaakUHCKBzXuK+Y9nlrAgbzcXnNqLX046lU4dDn8KrEhLp+AQAV5etoUfPr+M8krnfy85jUsy03UCXOQIFBzSohUeKOf2lz7muUUbGX5iZ+65fAQZ6tpU5KgUHNJifbg+n+kzl7Bh935uPW8g3z5/UL16vhNpKRQc0uJUVDp/mJPL72avpucJ7cieMobR/bpGXZZIk6HgkBZlY/5+vjNzCR+szeei4Wn8/GvDau0GVUSOTMEhLcaLSzbx4xeW48BvLx/O10aoP2yRz0PBIc1eQUkZP33xY174cBOn9+nMPVeM5MSuzacTJZHGpuCQZm3Rut1My17C5j3FTJ8wiFvGD6SVToCL1IuCQ5ql8opKfv9mLr9/czW9u7Tn2W+NIbOvToCLNAQFhzQ763ftZ/rMD1m8fg+TR/bmjotPoWM7nQAXaSgKDmk23J0XPtzET178GDO454oRXDyid9RliTQ7Cg5pFvYWl/Hjvyznr0s3MzqjK7+5fDjpXXQCXCQMCg5p8hau2c13Zi5ha0EJ/+9Lg7kpayCJCbrMViQsYXcdO83MlpvZx2Y2PRg3wswWmNkSM8sxs9G1LHeimc0xs5XBstOqTLvdzDYFyy8xs4lhboPEr7KKSu5+bRVXPPQerRKN5741hlvOG6TQEAlZaHscZjYMuBEYDZQCr5rZy8BdwB3u/vfgS/8uIKvG4uXAd919cdD3+CIze8PdVwTTf+vud4dVu8S/bUWVXPLAeyzdsIdLM9P56UWnkNxWO9AijSHMv7QhwAJ33w9gZvOASYADJwTzdAI211zQ3bcAW4LhfWa2EugNrKg5r7Qs7s6zizbyk3eLadu6jPu+fjoXnNYr6rJEWhRz93BWbDYEeBEYAxQDs4Ec4A/Aa4ARO1R2lruvO8p6MoC3gGHuXmBmtwPXAgXB+r7r7vm1LDcFmAKQmpqamZ2d3VCbFonCwkKSk1t2f9cHKpw/flzKO5vLGdTJ+daIDnRrr5v59LtRndqjuvq0x/jx4xe5+6ia40MLDgAzux6YChQS21soBhKBee4+y8wuA6a4+4QjLJ8MzAN+4e7PB+NSgZ3E9lx+DvRy928crY5Ro0Z5Tk5OA21VNObOnUtWVlbUZURmzc4ibnpyEau27ePb5w1ieKtNnDd+fNRlxYWW/rtRk9qjuvq0h5nVGhyh/nfN3R9x99PdfRywG1gNXAM8H8zyLLFzIIcxs9bALOCpg6ERrHObu1e4eyXw8JGWl+bj1eVbuej389laUMJj157Bd744mAQ9nFAkMmFfVdUjeO0DTAaeJnZO49xglvOIhUnN5Qx4BFjp7r+pMa3qAe1JwPKGr1ziQXlFJb98ZSXfenIR/bon8bdbzyHrpB5RlyXS4oV9GcosM+sGlAFT3T3fzG4E7jGzVkAJwXkIM0sDZrj7ROBs4GrgIzNbEqzrR+7+CnCXmY0gdqhqLfDNkLdBIrC9oIRbnv6QhWt2c9WZffivC4fStlVi1GWJCCEHh7uPrWXcfCCzlvGbgYlV5qn1WIS7X93AZUqceT9vF1P//CGFB8r47eXDmTQyPeqSRKQKXfguccPdefjtPH716ir6du3AUzd8gZN6doy6LBGpQcEhcaGgpIzvPbuU1z7exleH9eSuS07TE21F4pSCQyK3cksBNz25iA35xfz4giFcf04/dekqEscUHBKp5xZt5Md/+YgT2rUme8qZnJGhzpZE4p2CQyJRUlbBHX9dwdML13Nm/67835Uj6dGxXdRlichxUHBIo9uwez83PbWI5ZsK+Na5A/h/XxqsfsBFmhAFhzSqOZ9sZ/rMJVS68/C/j+KLQ1OjLklE6kjBIY2iotL53T/+ye/fzGVIrxN44KrT6dstKeqyRORzUHBI6HYVHmBa9hLm5+7kslHp/OziYbRrrbvARZoqBYeEatG6fG7582J2FZXyq389lcvP6BN1SSJSTwoOCYW78/i7a/nFyytJ69ye5286i2G9O0Vdlog0AAWHNLiiA+X8YNYy/rZsCxOG9ODXl46gUwfdBS7SXCg4pEHlbt/Ht55cTN6OQr735ZO46dwBJCToLnCR5kTBIQ3mpaWbuW3WMtq3TuTJ67/AWQNToi5JREKg4JB6Ky2Pdbj0+Ltryezbhfu+fjo9O+kucJHmSsEh9bJlbzE3P7WYD9fv4fpz+nHbV0+mte4CF2nWwu46dpqZLTezj81sejBuhJktMLMlZpZjZkfqc/wrZrbKzHLN7LYq47ua2Rtmtjp47RLmNsiRzV+9kwv+bz7/3LqP+75+Ov914VCFhkgLENpfuZkNA24ERgPDgQvNbBBwF3CHu48AfhK8r7lsInAf8FVgKHClmQ0NJt8GzHb3QcDs4L00ospK5943V3P1o++TktyGl249hwtO63XsBUWkWQjzUNUQYIG77wcws3nAJGJ9hZ8QzNMJ2FzLsqOBXHfPC5bNBi4GVgSvWcF8TwBzgR+EsgVymD37S/mPZ5by5ifbuXhEGv8z+VQ6tNERT5GWJMy/+OXAL8ysG1BMrD/xHGA68JqZ3U1sj+esWpbtDWyo8n4j8IVgONXdtwC4+xYz6xFO+VLTRxv3ctNTi9hWUMLPLz6Fq87sqw6XRFogc/fwVm52PTAVKCS2t1AMJALz3H2WmV0GTHH3CTWWuxT4srvfELy/Ghjt7rea2R5371xl3nx3P+w8h5lNAaYApKamZmZnZ4eyjY2lsLCQ5OTkyD4/b28Fdy4sIbm1MXVEWwZ0jvZZU1G3RzxRW1Sn9qiuPu0xfvz4Re4+6rAJ7t4oP8AvgZuBvRwKLAMKapl3DPBalfc/BH4YDK8CegXDvYBVx/rszMxMb+rmzJkT2Wev3Vnop//sdT/7ztm+bW9xZHVUFWV7xBu1RXVqj+rq0x5AjtfynRr2VVU9gtc+wGTgaWLnNM4NZjkPWF3Loh8Ag8ysn5m1Aa4AXgqmvQRcEwxfA7wYTvUCsSfbXvPoQirceeIbo+lxgu7PEGnpwj6rOSs4x1EGTHX3fDO7EbjHzFoBJQSHk8wsDZjh7hPdvdzMbgFeI3Zo61F3/zhY553AM8FhsPXApSFvQ4tVXFrB9U/ksGVvCX++8QsM6K7dfxE5zuAws0nAm+6+N3jfGchy978cbTl3H1vLuPlAZi3jNxM7gX7w/SvAK7XMtws4/3jqls+vvKKSW59ezLKNe7j/qkwy+3aNuiQRiRPHe6jqpwdDA8Dd9wA/DaUiiZy785OXPuYfK7dzx0Wn8OVTekZdkojEkeMNjtrm08X7zdR9c3L58/vruSlrAFePyYi6HBGJM8cbHDlm9hszG2Bm/c3st8CiMAuTaDy3aCN3v/5PJo3szfe/fFLU5YhIHDre4LgVKAVmAs8Qux9jalhFSTTe+ucObpu1jHMGpvCrfz1NN/eJSK2O63CTuxehZ0I1a8s37eWmJxcxKLUj9191Om1a6WGFIlK74/p2CJ5C27nK+y5m9lpoVUmj2rB7P9c9/gGdO7Th8evOoGM7dfMqIkd2vP+tTAmupALA3fMBPSOqGcgvKuWaxxZyoKyCx687g1Td4Ccix3C8wVEZ3P0NgJllEHvKrTRhJWUVXP/EB2zML2bGNWcwKLVj1CWJSBNwvJfU/icwP3g0OsA4gju+pWmqqHS+/fSHfLhhD/d9/XRG99MNfiJyfI735PirZjaKWFgsIfZ8qOIQ65IQuTt3/PVjXl+xjZ9cOJSJp6oTJhE5fsf7yJEbgGlAOrHgOBN4j9hDCqWJeWBeHn98bx03ju3HN87pF3U5ItLEHO85jmnAGcA6dx8PjAR2hFaVhOYvH27iV69+wr8MT+OHXx0SdTki0gQdb3CUuHsJgJm1dfdPAN1W3MS8k7uT7z23lDP7d+XuS08jIUE3+IlI3R3vyfGNwX0cfwHeMLN8au8rXOLUis0FfPNPi+ifksyDV4+ibatoe/ATkabreE+OTwoGbzezOUAn4NXQqpIGtWlPMdc9vpDktq147Loz6NReN/iJyOdX5yfcuvu8Y88l8WLv/jKueXQh+w9U8OxNY0jr3D7qkkSkiQu769hpZrbczD42s+nBuJlmtiT4WWtmS2pZ7qQq8ywxs4Iqy99uZpuqTJtYc3mJKSmr4MY/5rB+134e/PdMTu55QtQliUgzEFqfGmY2DLgRGE3sybqvmtnL7n55lXl+Deytuay7rwJGBPMkApuAF6rM8lt3vzus2puDykrnu88sZeHa3fzflSM5a0BK1CWJSDMR5h7HEGCBu+9393JgHnDwXAkWe2b3ZcDTx1jP+cCn7r4utEqbof9+eSUvf7SF/5w4hIuGp0Vdjog0I2EGx3JgnJl1M7MOxPoTP7HK9LHANndffYz1XMHh4XKLmS0zs0fNrEvDldw8zHg7j0ffWcN1Z2dww1jd4CciDcvcw3tWoZldT6zDp0JgBVDs7t8Jpt0P5Lr7r4+yfBtil/2e4u7bgnGpwE5iD1n8OdDL3b9Ry7JTCJ6nlZqampmdnd2Qm9boCgsLSU5OPuZ8728p5/6lBxiVmsjNI9qS0Ew7Yzre9mgJ1BbVqT2qq097jB8/fpG7jzpsgrs3yg/wS+DmYLgVsA1IP8YyFwOvH2V6BrD8WJ+dmZnpTd2cOXOOOc+7uTt90I9e8Uvvf9eLS8vDLypCx9MeLYXaojq1R3X1aQ8gx2v5Tg37qqoewWsfYDKHDjlNAD5x943HWMWV1DhMZWZVn8g3idghsRZv1dZ9TPlTDn26deChf8+kXWvd4Cci4QjtqqrALDPrBpQBUz3WARTUct7CzNKAGe4+MXjfAfgi8M0a67zLzEYQO1S1tpbpLc6WvcVc+9hC2rdO5IlvjKZzhzZRlyQizVioweHuY48w/tpaxm0mdgL94Pv9QLda5ru6AUts8gpKyrjusQ/YV1LOM98cQ2/d4CciIQv1UJWE60B5Bd/84yJytxfywFWZDE3TDX4iEr6wD1VJSCorne89u4z38nbx28uHc84g3eAnIo1DexxN1K9e/YSXlm7m+185iUkj06MuR0RaEAVHE/TYO2t48K08rj6zLzedOyDqckSkhVFwNDF//2gLP/vbCr40NJXbLzoFa6Y3+IlI/FJwNCEL1+xm2swljDyxM/935UgS1YOfiERAwdFEbC6s5MY/5pDeuT2PXHOGbvATkcgoOJqAbQUl/DqnhNaJCTzxjdF0SdINfiISHV2O2wRMy/6QwjLnuRvO4MSuHaIuR0RaOO1xxLn383axIG83lwxuw6npnaIuR0REexzx7t45uaQkt+XcdJ3TEJH4oD2OOLZ0wx7eXr2TG8b2o02irqASkfig4Ihj987JpVP71lx1Zt+oSxER+YyCI059srWAN1Zs47qzM0huqyOKIhI/FBxx6r45n5LUJpFrz8qIuhQRkWoUHHFozc4iXl62mavG9FWnTCISdxQccej+ubm0TkzghnP6R12KiMhhwu5zfJqZLTezj81sejBuppktCX7WmtmSIyy71sw+CubLqTK+q5m9YWarg9cuYW5DY9uYv5/nF2/iytF96N6xbdTliIgcJrTgMLNhwI3AaGA4cKGZDXL3y919hLuPAGYBzx9lNeODeUdVGXcbMNvdBwGzg/fNxkNv5WEGU8Zpb0NE4lOYexxDgAXuvt/dy4F5wKSDEy32PPDLgKfruN6LgSeC4SeAr9W/1PiwfV8J2R9sYPLIdNLUd7iIxClz93BWbDYEeBEYAxQT2zvIcfdbg+njgN/U2JuouvwaIB9w4EF3fygYv8fdO1eZL9/dDztcZWZTgCkAqampmdnZ2Q24deGYuaqUV9eUcefY9qQmVc/0wsJCkpOTI6os/qg9DlFbVKf2qK4+7TF+/PhFtX1Hh3aDgLuvNLNfAW8AhcBSoLzKLFdy9L2Ns919s5n1AN4ws0/c/a06fP5DwEMAo0aN8qysrLpuQqPKLyrl5jff5KIRaVx+wcjDps+dO5d434bGpPY4RG1RndqjujDaI9ST4+7+iLuf7u7jgN3AagAzawVMBmYeZdnNwet24AVi50oAtplZr2A9vYDt4W1B43ns3bXsL63g5qyBUZciInJUYV9V1SN47UMsKA7uYUwAPnH3jUdYLsnMOh4cBr4ELA8mvwRcEwxfQ+xwWJO2r6SMx99Zw5eGpnJSz45RlyMiclRhP8tilpl1A8qAqe6eH4y/ghqHqcwsDZjh7hOBVOCFoD/tVsCf3f3VYNY7gWfM7HpgPXBpyNsQuicXrKegpJxbztPehojEv1CDw93HHmH8tbWM2wxMDIbziF3CW9uyu4DzG67KaBWXVjDj7TzGDe7Oaemdoy5HROSYdOd4xLI/WM+uolJuGa+9DRFpGhQcESotr+Sht/IYndGV0f26Rl2OiMhxUXBE6PnFG9myt0TnNkSkSVFwRKS8opI/zP2U09I7MXZQStTliIgcNwVHRP62bAvrd+9n6viBBFePiYg0CQqOCFRWOvfNyWVwajJfHJIadTkiInWi4IjA6yu2sXp7IVPHDyQhQXsbItK0KDgambtz75zVZHTrwIWnpUVdjohInSk4Gtm8f+5g+aYCbsoaQKL2NkSkCVJwNLL75uSS1qkdk0amR12KiMjnouBoRO/n7eKDtfl889wBtGmlpheRpknfXo3o3jm5pCS35fIzToy6FBGRz03B0UiWbtjD26t3csPYfrRrnRh1OSIin5uCo5HcOyeXTu1bc9WZfaMuRUSkXhQcjeCTrQW8sWIb152dQXLbsLtAEREJl4KjEdw351OS2iRy7VkZUZciIlJvYXcdO83MlpvZx2Y2PRg308yWBD9rzWxJLcudaGZzzGxlsOy0KtNuN7NNVdYxMcxtqK81O4t4edlmrhrTl84d2kRdjohIvYV23MTMhgE3AqOBUuBVM3vZ3S+vMs+vgb21LF4OfNfdFwd9jy8yszfcfUUw/bfufndYtTek++fm0joxgRvO6R91KSIiDSLMPY4hwAJ33+/u5cA8YNLBiRZ7JOxl1Oh7HMDdt7j74mB4H7AS6B1iraHYmL+f5xdv4srRfejesW3U5YiINAhz93BWbDYEeBEYAxQDs4Ecd781mD4O+I27jzrGejKAt4Bh7l5gZrcD1wIFQA6xPZP8WpabAkwBSE1NzczOzm6YDauDP604wNwN5dw1rj3d2tcvowsLC0lOTm6gypo+tcchaovq1B7V1ac9xo8fv6jW72h3D+0HuB5YTOyL/wFih5gOTruf2Jf+0ZZPBhYBk6uMSwUSie0t/QJ49Fh1ZGZmemPbVlDsg/7zFf/+s0sbZH1z5sxpkPU0F2qPQ9QW1ak9qqtPexD7z/5h36mhnhx390fc/XR3HwfsBlYDmFkrYDIw80jLmllrYBbwlLs/X2Wd29y9wt0rgYeJnUOJO4+8vYbyikpuyhoQdSkiIg0q7KuqegSvfYgFxcHzGROAT9x94xGWM+ARYKW7/6bGtF5V3k4Cljd03fWVX1TKnxas41+Gp5GRkhR1OSIiDSrsu9FmmVk3oAyY6ofORVxBjZPiZpYGzHD3icDZwNXAR1Uu1/2Ru78C3GVmIwAH1gLfDHkb6uyxd9eyv7SCm7MGRl2KiEiDCzU43H3sEcZfW8u4zcDEYHg+UGtnFe5+dQOW2OD2lZTx+Dtr+NLQVE7q2THqckREGpzuHG9gTy5YT0FJObecp70NEWmeFBwNqLi0ghlv5zFucHdOS+8cdTkiIqFQcDSg7A/Ws6uolFvGa29DRJovBUcDKS2v5KG38hid0ZXR/bpGXY6ISGgUHA3k+cUb2bK3hKk6tyEizZyCowGUV1Tyh7mfclp6J8YNSom6HBGRUCk4GsDflm1h/e79TB0/kNi9iyIizZeCo54qK5375uQyODWZLw5JjbocEZHQKTjq6fUVW1m9vZCp4weSkKC9DRFp/hQc9eDu3Dsnl4xuHbjg1F7HXkBEpBlQcNTDvH/uYPmmAm7KGkCrRDWliLQM+rb7nNyde9/MJa1TOyaNTI+6HBGRRqPg+JzeX7ObnHX5fPPcAbRppWYUkZZD33if031zcklJbsvlZ5wYdSkiIo1KwfE5LN2wh7dX7+SGsf1o1zox6nJERBqVguNzuHdOLp3at+aqM/tGXYqISKMLu+vYaWa23Mw+NrPpwbiZZrYk+FlbpYe/mst+xcxWmVmumd1WZXxXM3vDzFYHr13C3IaaPtlawBsrtnHd2Rkktw27A0URkfgTWnCY2TDgRmA0MBy40MwGufvl7j7C3UcAs4Dna1k2EbgP+CowFLjSzIYGk28DZrv7IGB28L7R3DfnU5LaJHLtWRmN+bEiInEjzD2OIcACd9/v7uXAPGDSwYkWe6jTZdToezwwGsh19zx3LwWygYuDaRcDTwTDTwBfC6f8w63ZWcTLyzZz1Zi+dO7QprE+VkQkroR5rGU58Asz6wYUE+tPPKfK9LHANndfXcuyvYENVd5vBL4QDKe6+xYAd99iZj1q+3AzmwJMAUhNTWXu3Ln12JSYRz46QKLB0IStzJ27rd7rq4vCwsIG2YbmQu1xiNqiOrVHdWG0R2jB4e4rzexXwBtAIbAUKK8yy5XUvrcBUNtDn7yOn/8Q8BDAqFGjPCsrqy6LH2Zj/n7ee30uV52ZwcVfPqVe6/o85s6dS323oTlRexyitqhO7VFdGO0R6slxd3/E3U9393HAbmA1gJm1AiYDM4+w6Eag6g0S6cDmYHibmfUK1tML2B5G7TU99FYeZjBlXP/G+DgRkbgV9lVVPYLXPsSC4uAexgTgE3ffeIRFPwAGmVk/M2sDXAG8FEx7CbgmGL4GeDGM2qvavq+E7A82MHlkOmmd24f9cSIicS3s60lnBec4yoCp7p4fjL+CGoepzCwNmOHuE9293MxuAV4DEoFH3f3jYNY7gWfM7HpgPXBpyNvAI2+vobyikpuyBoT9USIicS/U4HD3sUcYf20t4zYTO4F+8P0rwCu1zLcLOL/hqjy6/KJS/rRgHf8yPI2MlKTG+lgRkbilO8eP4bF317K/tIKbswZGXYqISFxQcBzFvpIyHn9nDV8amspJPTtGXY6ISFxQcBzFkwvWU1BSzi3naW9DROQgBcdRdO/Ylksz0zktvXPUpYiIxA09pe8oLslM55JM9e4nIlKV9jhERKROFBwiIlInCg4REakTBYeIiNSJgkNEROpEwSEiInWi4BARkTpRcIiISJ2Ye5061muSzGwHsC7qOuopBdgZdRFxRO1xiNqiOrVHdfVpj77u3r3myBYRHM2BmeW4+6io64gXao9D1BbVqT2qC6M9dKhKRETqRMEhIiJ1ouBoOh6KuoA4o/Y4RG1RndqjugZvD53jEBGROtEeh4iI1ImCQ0RE6kTBEefM7EQzm2NmK83sYzObFnVNUTOzRDP70Mz+FnUtUTOzzmb2nJl9EvyOjIm6pqiY2XeCv5HlZva0mbWLuqbGZGaPmtl2M1teZVxXM3vDzFYHr10a4rMUHPGvHPiuuw8BzgSmmtnQiGuK2jRgZdRFxIl7gFfd/WRgOC20XcysN/BtYJS7DwMSgSuirarRPQ58pca424DZ7j4ImB28rzcFR5xz9y3uvjgY3kfsi6F3tFVFx8zSgQuAGVHXEjUzOwEYBzwC4O6l7r4n0qKi1Qpob2atgA7A5ojraVTu/hawu8boi4EnguEngK81xGcpOJoQM8sARgLvR1xKlH4HfB+ojLiOeNAf2AE8Fhy6m2FmSVEXFQV33wTcDawHtgB73f31aKuKC6nuvgVi/wkFejTEShUcTYSZJQOzgOnuXhB1PVEwswuB7e6+KOpa4kQr4HTgfncfCRTRQIcimprg2P3FQD8gDUgys6uirar5UnA0AWbWmlhoPOXuz0ddT4TOBi4ys7VANnCemT0ZbUmR2ghsdPeDe6DPEQuSlmgCsMbdd7h7GfA8cFbENcWDbWbWCyB43d4QK1VwxDkzM2LHsFe6+2+iridK7v5Dd0939wxiJz7fdPcW+79Kd98KbDCzk4JR5wMrIiwpSuuBM82sQ/A3cz4t9EKBGl4CrgmGrwFebIiVtmqIlUiozgauBj4ysyXBuB+5+yvRlSRx5FbgKTNrA+QB10VcTyTc/X0zew5YTOxKxA9pYY8eMbOngSwgxcw2Aj8F7gSeMbPriYXrpQ3yWXrkiIiI1IUOVYmISJ0oOEREpE4UHCIiUicKDhERqRMFh4iI1ImCQyTOmVmWngQs8UTBISIidaLgEGkgZnaVmS00syVm9mDQb0ihmf3azBab2Wwz6x7MO8LMFpjZMjN74WA/CWY20Mz+YWZLg2UGBKtPrtLvxlPB3dEikVBwiDQAMxsCXA6c7e4jgArg34AkYLG7nw7MI3Y3L8AfgR+4+2nAR1XGPwXc5+7DiT1raUswfiQwHRhK7Km4Z4e8SSJHpEeOiDSM84FM4INgZ6A9sQfKVQIzg3meBJ43s05AZ3efF4x/AnjWzDoCvd39BQB3LwEI1rfQ3TcG75cAGcD80LdKpBYKDpGGYcAT7v7DaiPN/qvGfEd7xs/RDj8dqDJcgf52JUI6VCXSMGYDl5hZD/isr+e+xP7GLgnm+Tow3933AvlmNjYYfzUwL+hnZaOZfS1YR1sz69CYGyFyPPS/FpEG4O4rzOzHwOtmlgCUAVOJda50ipktAvYSOw8CsUdcPxAEQ9Wn2l4NPGhmPwvW0SBPMxVpSHo6rkiIzKzQ3ZOjrkOkIelQlYiI1In2OEREpE60xyEiInWi4BARkTpRcIiISJ0oOEREpE4UHCIiUif/H5Rs1cnnGVP3AAAAAElFTkSuQmCC\n" 243 | }, 244 | "metadata": { 245 | "needs_background": "light" 246 | } 247 | } 248 | ], 249 | "source": [ 250 | "import matplotlib.pyplot as plt\n", 251 | "import numpy as np\n", 252 | "\n", 253 | "x_epoch = np.arange(1, EPOCH + 1)\n", 254 | "plt.plot(x_epoch, ACC)\n", 255 | "plt.xlabel('epoch')\n", 256 | "plt.ylabel('acc')\n", 257 | "plt.grid()\n", 258 | "plt.show()" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [] 267 | } 268 | ] 269 | } -------------------------------------------------------------------------------- /ConvLSTM.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | import torch.nn.functional as F 4 | 5 | class ConvLSTMCell(nn.Module): 6 | 7 | def __init__(self, input_dim, hidden_dim, kernel_size, bias): 8 | """ 9 | Initialize ConvLSTM cell. 10 | 11 | Parameters 12 | ---------- 13 | input_dim: int 14 | Number of channels of input tensor. 15 | hidden_dim: int 16 | Number of channels of hidden state. 17 | kernel_size: (int, int) 18 | Size of the convolutional kernel. 19 | bias: bool 20 | Whether or not to add the bias. 21 | """ 22 | 23 | super(ConvLSTMCell, self).__init__() 24 | 25 | self.input_dim = input_dim 26 | self.hidden_dim = hidden_dim 27 | 28 | self.kernel_size = kernel_size 29 | self.padding = kernel_size[0] // 2, kernel_size[1] // 2 30 | self.bias = bias 31 | 32 | self.conv = nn.Conv2d(in_channels=self.input_dim + self.hidden_dim, 33 | out_channels=4 * self.hidden_dim, 34 | kernel_size=self.kernel_size, 35 | padding=self.padding, 36 | bias=self.bias) 37 | 38 | 39 | def forward(self, input_tensor, cur_state): 40 | h_cur, c_cur = cur_state 41 | 42 | combined = torch.cat([input_tensor, h_cur], dim=1) # concatenate along channel axis 43 | 44 | combined_conv = self.conv(combined) 45 | cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_dim, dim=1) 46 | i = torch.sigmoid(cc_i) 47 | f = torch.sigmoid(cc_f) 48 | o = torch.sigmoid(cc_o) 49 | g = torch.tanh(cc_g) 50 | 51 | c_next = f * c_cur + i * g 52 | h_next = o * torch.tanh(c_next) 53 | 54 | return h_next, c_next 55 | 56 | def init_hidden(self, batch_size, image_size): 57 | height, width = image_size 58 | return (torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv.weight.device), 59 | torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv.weight.device)) 60 | 61 | 62 | class ConvLSTM(nn.Module): 63 | """ 64 | Parameters: 65 | input_dim: Number of channels in input 66 | hidden_dim: Number of hidden channels 67 | kernel_size: Size of kernel in convolutions 68 | num_layers: Number of LSTM layers stacked on each other 69 | batch_first: Whether or not dimension 0 is the batch or not 70 | bias: Bias or no bias in Convolution 71 | return_all_layers: Return the list of computations for all layers 72 | Note: Will do same padding. 73 | 74 | Input: 75 | A tensor of size B, T, C, H, W or T, B, C, H, W 76 | Output: 77 | A tuple of two lists of length num_layers (or length 1 if return_all_layers is False). 78 | 0 - layer_output_list is the list of lists of length T of each output 79 | 1 - last_state_list is the list of last states 80 | each element of the list is a tuple (h, c) for hidden state and memory 81 | Example: 82 | >> x = torch.rand((32, 10, 64, 128, 128)) 83 | >> convlstm = ConvLSTM(64, 16, 3, 1, True, True, False) 84 | >> _, last_states = convlstm(x) 85 | >> h = last_states[0][0] # 0 for layer index, 0 for h index 86 | """ 87 | 88 | def __init__(self, input_dim, hidden_dim, kernel_size, num_layers, 89 | batch_first=False, bias=True, return_all_layers=False): 90 | super(ConvLSTM, self).__init__() 91 | 92 | self._check_kernel_size_consistency(kernel_size) 93 | 94 | # Make sure that both `kernel_size` and `hidden_dim` are lists having len == num_layers 95 | kernel_size = self._extend_for_multilayer(kernel_size, num_layers) 96 | hidden_dim = self._extend_for_multilayer(hidden_dim, num_layers) 97 | if not len(kernel_size) == len(hidden_dim) == num_layers: 98 | raise ValueError('Inconsistent list length.') 99 | 100 | self.input_dim = input_dim 101 | self.hidden_dim = hidden_dim 102 | self.kernel_size = kernel_size 103 | self.num_layers = num_layers 104 | self.batch_first = batch_first 105 | self.bias = bias 106 | self.return_all_layers = return_all_layers 107 | 108 | cell_list = [] 109 | for i in range(0, self.num_layers): 110 | cur_input_dim = self.input_dim if i == 0 else self.hidden_dim[i - 1] 111 | 112 | cell_list.append(ConvLSTMCell(input_dim=cur_input_dim, 113 | hidden_dim=self.hidden_dim[i], 114 | kernel_size=self.kernel_size[i], 115 | bias=self.bias)) 116 | 117 | 118 | self.cell_list = nn.ModuleList(cell_list) 119 | 120 | 121 | def forward(self, input_tensor, hidden_state=None): 122 | """ 123 | Parameters 124 | ---------- 125 | input_tensor: construct the input shape by yourself 126 | 5-D Tensor either of shape (t, b, c, h, w) or (b, t, c, h, w) 127 | hidden_state: hidden layer could be set by yourself 128 | None. 129 | 130 | Returns 131 | ---------- 132 | last_state_list, layer_output 133 | """ 134 | if not self.batch_first: 135 | # (t, b, c, h, w) -> (b, t, c, h, w) 136 | input_tensor = input_tensor.permute(1, 0, 2, 3, 4) 137 | 138 | b, _, _, h, w = input_tensor.size() 139 | 140 | # Implement stateful ConvLSTM 141 | if hidden_state is not None: 142 | raise NotImplementedError() 143 | else: 144 | # Since the init is done in forward. Can send image size here 145 | hidden_state = self._init_hidden(batch_size=b, 146 | image_size=(h, w)) 147 | 148 | layer_output_list = [] 149 | last_state_list = [] 150 | 151 | seq_len = input_tensor.size(1) 152 | cur_layer_input = input_tensor 153 | 154 | for layer_idx in range(self.num_layers): 155 | 156 | h, c = hidden_state[layer_idx] 157 | output_inner = [] 158 | for t in range(seq_len): 159 | h, c = self.cell_list[layer_idx](input_tensor=cur_layer_input[:, t, :, :, :], cur_state=[h, c]) 160 | output_inner.append(h) 161 | 162 | layer_output = torch.stack(output_inner, dim=1) 163 | cur_layer_input = layer_output 164 | 165 | layer_output_list.append(layer_output) # 所有时间点的隐层状态以及cell状态 166 | last_state_list.append([h, c]) # 最后一层的隐层状态以及cell状态 167 | 168 | if not self.return_all_layers: 169 | layer_output_list = layer_output_list[-1:] 170 | last_state_list = last_state_list[-1:] 171 | 172 | return layer_output_list, last_state_list 173 | 174 | def _init_hidden(self, batch_size, image_size): 175 | init_states = [] 176 | for i in range(self.num_layers): 177 | init_states.append(self.cell_list[i].init_hidden(batch_size, image_size)) 178 | return init_states 179 | 180 | @staticmethod 181 | def _check_kernel_size_consistency(kernel_size): 182 | if not (isinstance(kernel_size, tuple) or 183 | (isinstance(kernel_size, list) and all([isinstance(elem, tuple) for elem in kernel_size]))): 184 | raise ValueError('`kernel_size` must be tuple or list of tuples') 185 | 186 | @staticmethod 187 | def _extend_for_multilayer(param, num_layers): 188 | if not isinstance(param, list): 189 | param = [param] * num_layers 190 | return param 191 | 192 | 193 | class outputCNN(nn.Module): 194 | def __init__(self, input_dim): 195 | super(outputCNN, self).__init__() 196 | self.conv1 = nn.Conv2d(in_channels=input_dim, out_channels=128, kernel_size=(5, 5), padding=(2, 2)) 197 | self.pool = nn.MaxPool2d(2, 2, return_indices=True) 198 | self.conv2 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=(5, 5), padding=(2, 2)) 199 | self.unpool = nn.MaxUnpool2d(2, 2) 200 | 201 | self.conv3 = nn.ConvTranspose2d(in_channels=128, out_channels=64, kernel_size=(5, 5), padding=(2, 2)) 202 | self.conv4 = nn.ConvTranspose2d(in_channels=64, out_channels=1, kernel_size=(5, 5), padding=(2, 2)) 203 | 204 | def forward(self,x): 205 | x = F.relu(self.conv1(x)) 206 | output_size = x.shape 207 | x, i = self.pool(x) 208 | x = F.relu(self.conv2(x)) 209 | x = self.unpool(x, i, output_size=output_size) 210 | x = F.relu(self.conv3(x)) 211 | # x = torch.sigmoid(self.conv4(x)) 212 | x = torch.relu(self.conv4(x)) 213 | return x 214 | 215 | 216 | 217 | class ConvLSTM_model(nn.Module): 218 | def __init__(self, VGG_indim, VGG_outdim, input_dim, hidden_dim, kernel_size, num_layers, 219 | batch_first=False, bias=True, return_all_layers=False, block_nums=None): 220 | super(ConvLSTM_model, self).__init__() 221 | 222 | self.input_dim = input_dim 223 | self.hidden_dim = hidden_dim 224 | self.kernel_size = kernel_size 225 | self.num_layers = num_layers 226 | self.batch_first = batch_first 227 | self.bias = bias 228 | self.return_all_layers = return_all_layers 229 | 230 | self.conv_lstm = ConvLSTM(self.input_dim, self.hidden_dim, self.kernel_size, self.num_layers, 231 | self.batch_first, self.bias, self.return_all_layers) 232 | self.outputCNN = outputCNN(self.hidden_dim) 233 | 234 | def initialize_weight(self): 235 | for m in self.modules(): 236 | if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d): 237 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 238 | if m.bias is not None: 239 | nn.init.constant_(m.bias, 0) 240 | elif isinstance(m, nn.BatchNorm2d): 241 | nn.init.constant_(m.weight, 1) 242 | nn.init.constant_(m.bias, 0) 243 | 244 | def forward(self, x): 245 | # output_list是最后一层的所有时间点状态的叠加之后的结果 246 | # output是output_list最后一个时间点[隐层状态, cell状态] 247 | output_list, output = self.conv_lstm(x) 248 | 249 | conv_output_list = [] 250 | for i in range(output_list[0].size(1)): 251 | conv_output_list.append(self.outputCNN(output_list[0][:, i, :, :, :])) 252 | 253 | conv_output = [self.outputCNN(output[0][0]), self.outputCNN(output[0][1])] 254 | conv_output_list_ret = torch.stack(conv_output_list, dim=1) 255 | 256 | # conv_output_list_ret = output_list[0] 257 | # conv_output = output[0] 258 | 259 | return output_list, output, conv_output, conv_output_list_ret 260 | -------------------------------------------------------------------------------- /LSTM/LSTM_Regression.py: -------------------------------------------------------------------------------- 1 | """ 2 | 来源：莫凡Pytorch教学 3 | 作者：EricPengShuai 4 | """ 5 | import torch 6 | import torch.nn as nn 7 | import numpy as np 8 | import matplotlib.pyplot as plt 9 | import time 10 | 11 | class LSTM(nn.Module): 12 | """搭建LSTM网络""" 13 | def __init__(self): 14 | super(LSTM, self).__init__() 15 | self.lstm = nn.LSTM( 16 | input_size=input_size, 17 | hidden_size=hidden_size, 18 | num_layers=num_layers, 19 | batch_first=True,) 20 | self.output_layer = nn.Linear(in_features=hidden_size, out_features=output_size) 21 | 22 | def forward(self, x, hc): 23 | # x (batch, time_step, input_size) 24 | # h_state (n_layers, batch, hidden_size) 25 | # rnn_out (batch, time_step, hidden_size) 26 | rnn_out, hc = self.lstm(x, hc) # h_state是之前的隐层状态 27 | 28 | # 这样将rnn_out直接喂入Linear的效果和分每个时间点喂入的效果差不多 29 | # return self.output_layer(rnn_out.view(-1, hidden_size)).view(1, -1, output_size), hc 30 | 31 | # 将rnn_out分为每个时间点喂入Linear 32 | out = [] 33 | for time in range(rnn_out.size(1)): 34 | every_time_out = rnn_out[:, time, :] # 相当于获取每个时间点上的输出，然后过输出层 35 | temp = self.output_layer(every_time_out) 36 | out.append(temp) 37 | # print(f'Time={time}', rnn_out.shape, every_time_out.shape, temp.shape, len(out)) 38 | return torch.stack(out, dim=1), hc # torch.stack扩成[1, output_size, 1] 39 | 40 | # 设置超参数 41 | input_size = 1 42 | output_size = 1 43 | num_layers = 1 44 | hidden_size = 32 45 | learning_rate = 0.02 46 | train_step = 100 47 | time_step = 10 48 | 49 | # 准备数据 50 | steps = np.linspace(0, 2*np.pi, 100, dtype=np.float32) 51 | x_np = np.sin(steps) 52 | y_np = np.cos(steps) 53 | 54 | # plt.plot(steps, y_np, 'r-', label='target (cos)') 55 | # plt.plot(steps, x_np, 'b-', label='input (sin)') 56 | # plt.legend(loc='best') 57 | # plt.show() 58 | 59 | lstm = LSTM() 60 | print(lstm) 61 | 62 | # 设置优化器和损失函数 63 | # optimizer = torch.optim.Adam(rnn.parameters(), lr=learning_rate) 64 | optimizer = torch.optim.Adam(lstm.parameters(), lr=learning_rate) 65 | loss_function = nn.MSELoss() 66 | startTime = time.time() 67 | 68 | plt.figure(1, figsize=(12, 5)) 69 | plt.ion() 70 | 71 | # 训练 72 | h_state = None # 初始化隐藏层状态 73 | 74 | for step in range(train_step): 75 | start, end = step * np.pi, (step+1) * np.pi 76 | steps = np.linspace(start, end, time_step, dtype=np.float32) 77 | x_np = np.sin(steps) 78 | y_np = np.cos(steps) 79 | 80 | x = torch.from_numpy(x_np[np.newaxis, :, np.newaxis]) 81 | y = torch.from_numpy(y_np[np.newaxis, :, np.newaxis]) 82 | 83 | predict, h_state = lstm(x, h_state) 84 | # 必须要使用detach() https://www.cnblogs.com/catnofishing/p/13287322.html 85 | h_state = (h_state[0].detach(), h_state[1].detach()) 86 | # print(predict.shape) 87 | # exit() 88 | 89 | if step == train_step - 1: 90 | endTime = time.time() 91 | print(f'TimeSum={round(endTime - startTime, 4)}s') 92 | # exit() 93 | loss = loss_function(predict, y) 94 | optimizer.zero_grad() 95 | 96 | loss.backward() 97 | optimizer.step() 98 | 99 | # plotting 100 | plt.plot(steps, y_np.flatten(), 'r-') 101 | plt.plot(steps, predict.detach().numpy().flatten(), 'b-') 102 | plt.draw() 103 | plt.pause(0.05) 104 | 105 | plt.ioff() 106 | plt.show() 107 | 108 | 109 | -------------------------------------------------------------------------------- /LinearNetwork/multiple_dimension_diabetes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "来源： [b站刘二大人--处理多维特征输入](https://www.bilibili.com/video/BV1Y7411d7Ys?p=7)\n", 8 | "# Prepare Dataset" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "metadata": { 15 | "collapsed": true 16 | }, 17 | "outputs": [], 18 | "source": [ 19 | "import numpy as np\n", 20 | "import torch\n", 21 | "import matplotlib.pyplot as plt\n", 22 | " \n", 23 | "xy = np.loadtxt('../dataset/diabetes.csv', delimiter=',', dtype=np.float32)\n", 24 | "x_data = torch.from_numpy(xy[:, :-1]) # 第一个‘：’是指读取所有行，第二个‘：’是指从第一列开始，最后一列不要\n", 25 | "y_data = torch.from_numpy(xy[:, [-1]]) # [-1] 最后得到的是个矩阵\n", 26 | "# print(x_data)\n", 27 | "# print(y_data)" 28 | ] 29 | }, 30 | { 31 | "source": [ 32 | "# Define Model" 33 | ], 34 | "cell_type": "markdown", 35 | "metadata": {} 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 3, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "output_type": "stream", 44 | "name": "stdout", 45 | "text": [ 46 | "Model(\n (linear1): Linear(in_features=8, out_features=6, bias=True)\n (linear2): Linear(in_features=6, out_features=4, bias=True)\n (linear3): Linear(in_features=4, out_features=1, bias=True)\n (relu): ReLU()\n (sigmoid): Sigmoid()\n)\n" 47 | ] 48 | } 49 | ], 50 | "source": [ 51 | "class Model(torch.nn.Module):\n", 52 | " def __init__(self):\n", 53 | " super(Model, self).__init__()\n", 54 | " self.linear1 = torch.nn.Linear(8, 6)\n", 55 | " self.linear2 = torch.nn.Linear(6, 4)\n", 56 | " self.linear3 = torch.nn.Linear(4, 1)\n", 57 | " self.relu = torch.nn.ReLU()\n", 58 | " self.sigmoid = torch.nn.Sigmoid()\n", 59 | " \n", 60 | " def forward(self, x):\n", 61 | " x = self.relu(self.linear1(x))\n", 62 | " x = self.relu(self.linear2(x))\n", 63 | " x = self.sigmoid(self.linear3(x))\n", 64 | " return x\n", 65 | " \n", 66 | "model = Model()\n", 67 | "print(model)" 68 | ] 69 | }, 70 | { 71 | "source": [ 72 | "# Construct Loss and Optimizer\n", 73 | "## 注意在使用二元交叉熵计算损失的维度关系(计算之前还需要激活)，这里的y_pred和y_data维度都是(\\[759, 1\\])" 74 | ], 75 | "cell_type": "markdown", 76 | "metadata": {} 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 5, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "criterion = torch.nn.BCELoss(reduction='mean')\n", 85 | "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n" 86 | ] 87 | }, 88 | { 89 | "source": [ 90 | "# Training Cycle" 91 | ], 92 | "cell_type": "markdown", 93 | "metadata": {} 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 6, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "output_type": "stream", 102 | "name": "stdout", 103 | "text": [ 104 | "100 0.6413201689720154\n", 105 | "200 0.6322777271270752\n", 106 | "300 0.5955487489700317\n", 107 | "400 0.5296488404273987\n", 108 | "500 0.4919593334197998\n", 109 | "600 0.4763002395629883\n", 110 | "700 0.46978849172592163\n", 111 | "800 0.46675390005111694\n", 112 | "900 0.46487873792648315\n", 113 | "1000 0.4639298617839813\n", 114 | "1100 0.46327805519104004\n", 115 | "1200 0.46273529529571533\n", 116 | "1300 0.4622042775154114\n", 117 | "1400 0.4616772532463074\n", 118 | "1500 0.461091011762619\n", 119 | "1600 0.4603952169418335\n", 120 | "1700 0.459516316652298\n", 121 | "1800 0.4584980607032776\n", 122 | "1900 0.45733630657196045\n", 123 | "2000 0.4562112092971802\n" 124 | ] 125 | }, 126 | { 127 | "output_type": "display_data", 128 | "data": { 129 | "text/plain": "
", 130 | "image/svg+xml": "\n\n\n\n \n \n \n \n 2021-03-12T18:38:12.749688\n image/svg+xml\n \n \n Matplotlib v3.3.2, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", 131 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAt7UlEQVR4nO3deZhV9Z3n8fe3NmqjqoCCYl8ENIIJS6G4S8WoaKeDSTCNSYjpxKHpDv0kmaQnZNKTcdLd00nszqYmxhgn2km6RqNGxkaNoUE00QiFICAiiyBrIcVSFFDU9p0/zim8XGq7VXXureXzep773HN+5/c79bm3ivvlLPccc3dEREQ6Ki3VAUREpHdR4RARkYSocIiISEJUOEREJCEqHCIikpCMVAdIhuLiYh8/fnynxp48eZK8vLzuDdQNlCsxypUY5UpMT80FXctWUVFx2N2HnrfA3fv8o7S01Dtr5cqVnR4bJeVKjHIlRrkS01NzuXctG7DWW/hM1a4qERFJSKSFw8zmmtlWM9tuZktb6TPHzNab2WYzeyGmfZeZbQyXrY1pH2xmz5vZtvB5UJSvQUREzhVZ4TCzdOA+4GZgCnC7mU2J61ME/Bj4iLtPBW6LW02Zu09391kxbUuBFe4+GVgRzouISJJEucVxGbDd3Xe6ex1QDsyL6/NJ4Al3fwfA3Q91YL3zgIfD6YeBW7snroiIdIR5RNeqMrP5wFx3vzOcXwjMdvclMX1+AGQCU4GBwA/d/ZFw2dvAUcCBn7r7A2H7MXcvilnHUXc/b3eVmS0CFgGUlJSUlpeXd+p11NTUkJ+f36mxUVKuxChXYpQrMT01F3QtW1lZWUXcHp9AS0fMu+NBsNvpwZj5hcA9cX3uBV4B8oBiYBtwYbhsZPg8DNgAXBvOH4tbx9H2suisquRRrsQoV2KUK3G97ayqvcCYmPnRwP4W+jzr7ifd/TCwGpgG4O77w+dDwJMEu74AKs1sBED43JHdWyIi0k2iLBxrgMlmNsHMsoAFwLK4Pk8B15hZhpnlArOBLWaWZ2YDAcwsD7gR2BSOWQbcEU7fEa4jEuveOcryt+uat2xERIQIC4e7NwBLgOeALcCj7r7ZzBab2eKwzxbgWeB14FWCXVubgBLgJTPbELb/h7s/G67628ANZrYNuCGcj8ST6/bx6NZ6vv7ERk6eaYjqx4iI9CqRXnLE3ZcDy+Pa7o+bvxu4O65tJ+EuqxbWWQVc371JW/a/PjKVI5X7KV+zh9Vvvcud11zAx0tHU5iTmYwfLyLSI+mb421ISzNuuyiLxxZfwYiiHL719BuU/sPzfOrBV/jpCzuo2H2EMw2NqY4pIpJU/eIih1116fjBPP7XV7Jx73H+Y+MBfr+lkn9+5k0AsjLS+MCoQkrHD+LyCUO4dMJg8gfobRWRvkufcAl4/+hC3j+6kKU3v493T5yhYvdRKnYfoWL3UR566W1++sJO0tOM948q5IqJQ/jQxcOYMWYQaWmW6ugiIt1GhaOThg4cwNxLhjP3kuEAnK5rZN07R3l5RxUv76ziZ6t38pNVOygpGMBNU4dz64xRzBhThJmKiIj0bioc3SQnK52rJhVz1aRiAKpr61n55iGe2XiQR9fu4ZGXdzNlRAGfvnwcH5s5iuzM9BQnFhHpHBWOiBRkZzJv+ijmTR9FzZkGfvvaPn75ym7++5Mb+eGKt1hSNolPXDqGARkqICLSu+isqiTIH5DBpy8fxzNfvIZf/5fZjB2cy/94ajM3fn81L207nOp4IiIJUeFIIjPjyonFPPpXV/CLv7yUNDM+/fM/8ZVHN+gLhiLSa6hwpICZMeeiYTzzxWtYUjaJJ1/by7z7/sC2yhOpjiYi0i4VjhTKzkznqzddxC8/P5tjp+r42I//yCs7q1IdS0SkTSocPcCVk4pZtuRqSgqz+cxDr/L7NypTHUlEpFUqHD3EyKIcHvurK7h4RAF/86t1/HGHDpqLSM+kwtGDDMrL4uG/vJTxxbkseqSCN/ZXpzqSiMh5VDh6mKLcLB753GzyB2Sw+JcVHD9Vn+pIIiLnUOHogYYXZnPfp2ay/9hpvvLYet1ISkR6FBWOHqp03CD++y0X8/sth3h07Z5UxxEROUuFowf77JXjufyCwfzj01s4eLw21XFERICIC4eZzTWzrWa23cyWttJnjpmtN7PNZvZC2DbGzFaa2Zaw/Ysx/e8ys33hmPVmdkuUryGV0tKM73z8AzQ0Od96enOq44iIABEWDjNLB+4DbgamALeb2ZS4PkXAj4GPuPtU4LZwUQPwFXe/GLgc+ELc2O+7+/Twcc6tafuacUPyWHzdRJZvPMjaXUdSHUdEJNItjsuA7e6+093rgHJgXlyfTwJPuPs7AO5+KHw+4O7rwukTwBZgVIRZe7T/cu0Ehhdk8w//sYWmJh0oF5HUsqjO2DGz+cBcd78znF8IzHb3JTF9fgBkAlOBgcAP3f2RuPWMB1YDl7h7tZndBXwWqAbWEmyZHG3h5y8CFgGUlJSUlpeXd+p11NTUkJ+f36mx3emlffU8uLGOJdMHMGt4Ro/JFU+5EqNciVGuxHUlW1lZWYW7zzpvgbtH8iDY7fRgzPxC4J64PvcCrwB5QDGwDbgwZnk+UAF8LKatBEgn2Fr6J+Ch9rKUlpZ6Z61cubLTY7tTQ2OTz7l7pf/Zj1Z7U1NTj8kVT7kSo1yJUa7EdSUbsNZb+EyNclfVXmBMzPxoYH8LfZ5195Pufphgy2IagJllAo8Dv3L3J5oHuHuluze6exPwM4JdYn1eeprx19dNZNO+ala99W6q44hIPxZl4VgDTDazCWaWBSwAlsX1eQq4xswyzCwXmA1sseDG3D8Htrj792IHmNmImNmPApsiewU9zK0zRjGqKIf7V+1IdRQR6cciKxzu3gAsAZ4jOLj9qLtvNrPFZrY47LMFeBZ4HXiVYNfWJuAqgl1bH2zhtNvvmtlGM3sdKAO+HNVr6GmyMtJYeMU4/vT2EfadaEp1HBHppyK957gHp8ouj2u7P27+buDuuLaXAGtlnQu7OWavclvpaL73u7f4zz31fCrVYUSkX9I3x3uZIfkDuOX9w/nDvgbdblZEUkKFoxf69OXjqG2E5RsPpDqKiPRDKhy9UOm4QQzNMZ5aH3+SmohI9FQ4eiEz4/KRGfxxx2EOVevihyKSXCocvdQVIzJocvh/r2t3lYgklwpHLzUyP41LRhXw1Pp9qY4iIv2MCkcvNm/aKF7fe5yd79akOoqI9CMqHL3Yh6cFX6J/ZtPBFCcRkf5EhaMXG1GYw7TRhfxuswqHiCSPCkcvd+PU4WzYe5wDx0+nOoqI9BMqHL3cTVOHA/C7zZUpTiIi/YUKRy83aVg+E4fm8Zx2V4lIkqhw9AE3TR3On94+wtGTdamOIiL9gApHH3Dj1OE0Njmrt+kGTyISPRWOPuD9owoZlJvJC7ozoIgkgQpHH5CeZlw9eSir3zpMU5OnOo6I9HEqHH3EdRcO5XDNGbYcrE51FBHp4yItHGY218y2mtl2M1vaSp854a1hN5vZC+2NNbPBZva8mW0LnwdF+Rp6i2snFwOw+q3DKU4iIn1dZIXDzNKB+4CbgSnA7WY2Ja5PEfBj4CPuPhW4rQNjlwIr3H0ysCKc7/eGFWTzvuEDWa3jHCISsSi3OC4Dtrv7TnevA8qBeXF9Pgk84e7vALj7oQ6MnQc8HE4/DNwa3UvoXa67cChrdx/RLWVFJFLmHs3BVDObD8x19zvD+YXAbHdfEtPnB0AmMBUYCPzQ3R9pa6yZHXP3oph1HHX383ZXmdkiYBFASUlJaXl5eadeR01NDfn5+Z0aG6WWcr1R1ch319TypZkDmD4so8fk6gmUKzHKlZiemgu6lq2srKzC3WfFt0f56WIttMVXqQygFLgeyAFeNrNXOji2Te7+APAAwKxZs3zOnDmJDD9r1apVdHZslFrKdXl9Iz9Y9ztO5Y9izpyLe0yunkC5EqNciempuSCabFEWjr3AmJj50UD8TbL3Aofd/SRw0sxWA9PaGVtpZiPc/YCZjQAOIQBkZ6YzfWwRr+ysSnUUEenDojzGsQaYbGYTzCwLWAAsi+vzFHCNmWWYWS4wG9jSzthlwB3h9B3hOiR0+QVD2LTvONW19amOIiJ9VGSFw90bgCXAcwTF4FF332xmi81scdhnC/As8DrwKvCgu29qbWy46m8DN5jZNuCGcF5CV1wwhCaHNW8fSXUUEemjIj2C6u7LgeVxbffHzd8N3N2RsWF7FcExEWnBjLFFZGWk8crOKq6/uCTVcUSkD9I3x/uY7Mx0Zo4t4mUd5xCRiKhw9EGXXzCEzfurOX5axzlEpPupcPRBsycMwR0qdus4h4h0PxWOPmj6mCIy0oyK3UdTHUVE+iAVjj4oJyudqSMLWLtLhUNEup8KRx81c9wgNuw9Rn1jU6qjiEgfo8LRR80aN5ja+ibe2K/7c4hI91Lh6KNKxwXXfVyr4xwi0s1UOPqo4YXZjCrKYZ0Kh4h0MxWOPmzW+EGs3X2EqC6dLyL9kwpHH1Y6bhCV1WfYd+x0qqOISB+iwtGHNR/n0Pc5RKQ7qXD0YReVDCQvK13f5xCRbqXC0YdlpKcxY+wgbXGISLdS4ejjZo4bxJsHq6k505DqKCLSR6hw9HGzxg2iyWH9O8dSHUVE+ggVjj5u+tgizGDdO9pdJSLdI9LCYWZzzWyrmW03s6UtLJ9jZsfNbH34+GbYflFM23ozqzazL4XL7jKzfTHLbonyNfR2BdmZTByaz4Y9x1IdRUT6iMhuHWtm6cB9BPcF3wusMbNl7v5GXNcX3f3DsQ3uvhWYHrOefcCTMV2+7+7/ElX2vmba6CJeeOsQ7o6ZpTqOiPRyUW5xXAZsd/ed7l4HlAPzOrGe64Ed7r67W9P1I9PHFnG4pk5fBBSRbmFRXY7CzOYDc939znB+ITDb3ZfE9JkDPE6wRbIf+Kq7b45bz0PAOne/N5y/C/gsUA2sBb7i7uftwDezRcAigJKSktLy8vJOvY6amhry8/M7NTZKieTadbyRu16u5W+mD+Cy4ZFtZCacK5mUKzHKlZiemgu6lq2srKzC3Wedt8DdI3kAtwEPxswvBO6J61MA5IfTtwDb4pZnAYeBkpi2EiCdYGvpn4CH2stSWlrqnbVy5cpOj41SIrnO1Df65G8s9398enN0gUJ94f1KJuVKjHIlrivZgLXewmdqlLuq9gJjYuZHE2xVnOXu1e5eE04vBzLNrDimy80EWxuVMWMq3b3R3ZuAnxHsEpM2ZGWkMXVkARv2HE91FBHpA6IsHGuAyWY2wcyygAXAstgOZjbcwqO1ZnZZmKcqpsvtwL/HjRkRM/tRYFME2fuc6WOK2LjvOA26I6CIdFFkhcPdG4AlwHPAFuBRd99sZovNbHHYbT6wycw2AD8CFoSbR5hZLsEZWU/Erfq7ZrbRzF4HyoAvR/Ua+pLpY4o4Xd/ItkM1qY4iIr1cpEdKw91Py+Pa7o+Zvhe4t5Wxp4AhLbQv7OaY/cK00UUArN9zjItHFKQ2jIj0avrmeD8xbkguRbmZ+iKgiHSZCkc/YWZMG13EehUOEekiFY5+ZNqYIt6qPMGpOl0pV0Q6T4WjH5k+ppAmh417dVquiHSeCkc/0nyAfMPeYynNISK9mwpHPzIkfwBjBufoOIeIdIkKRz8zbXSRvkEuIl2iwtHPTB9TxL5jp3n3xJlURxGRXkqFo5+ZPqYIQN/nEJFOU+HoZ6aOLCQ9zXSAXEQ6TYWjn8nJSueikoE6QC4inabC0Q9NG1PEhj3HaGqK5iZeItK3qXD0QzPGFFFd28CuqpOpjiIivVCHCoeZfdHMCizwczNbZ2Y3Rh1OojGt+QC5jnOISCd0dIvjc+5eDdwIDAX+Evh2ZKkkUpOG5ZObla7vc4hIp3S0cFj4fAvwf9x9Q0yb9DLpacb7RxXymg6Qi0gndLRwVJjZ7wgKx3NmNhDQPUh7seljitiyv5ozDY2pjiIivUxHC8fngaXApeGd+TIJdle1yczmmtlWM9tuZktbWD7HzI6b2frw8c2YZbvCW8SuN7O1Me2Dzex5M9sWPg/q4GuQGNPHFFHX2MSbB06kOoqI9DIdLRxXAFvd/ZiZfRr4e6DNHeRmlg7cB9wMTAFuN7MpLXR90d2nh49vxS0rC9tnxbQtBVa4+2RgRTgvCdIBchHprI4Wjp8Ap8xsGvDfgN3AI+2MuQzY7u473b0OKAfmdTrpe+YBD4fTDwO3dsM6+50RhdkMHTiA9e8cS3UUEellzL39L4GZ2Tp3nxnuStrn7j9vbmtjzHxgrrvfGc4vBGa7+5KYPnOAx4G9wH7gq+6+OVz2NnAUcOCn7v5A2H7M3Yti1nHU3c/bXWVmi4BFACUlJaXl5eXtvs6W1NTUkJ+f36mxUeqOXD9cV8uBk018+5rcbkrVt9+vKChXYpQrcV3JVlZWVhG3xyfg7u0+gBeArwPbgOFAOrCxnTG3AQ/GzC8E7onrUwDkh9O3ANtilo0Mn4cBG4Brw/ljces42l7+0tJS76yVK1d2emyUuiPXPSve8nFfe9qPnarreqBQX36/oqBciVGuxHUlG7DWW/hM7eiuqr8AzhB8n+MgMAq4u50xe4ExMfOjCbYqYotWtbvXhNPLgUwzKw7n94fPh4AnCXZ9AVSa2QiA8PlQB1+DxJk+JthQ061kRSQRHSocYbH4FVBoZh8Gat29vWMca4DJZjbBzLKABcCy2A5mNtzMLJy+LMxTZWZ54Sm/mFkewRcPN4XDlgF3hNN3AE915DXI+d4/uhCA9XuOpjiJiPQmGR3pZGafINjCWEXwxb97zOzv3P03rY1x9wYzWwI8R7Br6yF332xmi8Pl9wPzgb82swbgNLDA3d3MSoAnw5qSAfza3Z8NV/1t4FEz+zzwDsEuMemEwpxMLhiapyvlikhCOlQ4gG8QfIfjEICZDQV+D7RaOODs7qflcW33x0zfC9zbwridwLRW1lkFXN/B3NKOmWMH8Z9vHsLdCQu1iEibOnqMI625aISqEhgrPdiscYM4crKOtw/rSrki0jEd3eJ41syeA/49nP8L4rYkpHcqHRccIF+7+ygXDO2ZpxOKSM/S0YPjfwc8AHyAYBfSA+7+tSiDSXJMHJpPYU4m63brALmIdExHtzhw98cJvqwnfUhamjFzbBFrVThEpIPa3OIwsxNmVt3C44SZVScrpERr1vjBbD9Uw7FTdamOIiK9QJuFw90HuntBC4+B7l6QrJASrZljg+Mc697RVoeItE9nRgnTxxSRnmZUaHeViHSACoeQk5XO1JEFrN2lwiEi7VPhECA4LXfD3mPUN+rGjiLSNhUOAYLCUVvfxBv7dc6DiLRNhUOA974IqOMcItIeFQ4BYERhDqOKclQ4RKRdKhxy1mUTBvPKziqamtq/K6SI9F8qHHLWlROHUHWyjq2VJ1IdRUR6MBUOOeuqScUA/GH74RQnEZGeTIVDzhpZlMMFxXn8cUdVqqOISA+mwiHnuHLSEP60s0rf5xCRVkVaOMxsrpltNbPtZra0heVzzOy4ma0PH98M28eY2Uoz22Jmm83sizFj7jKzfTFjbonyNfQ3V00s5mRdIxt0O1kRaUWHL6ueKDNLB+4DbgD2AmvMbJm7vxHX9UV3/3BcWwPwFXdfZ2YDgQozez5m7Pfd/V+iyt6fXTFxCGbwh+1VzBo/ONVxRKQHinKL4zJgu7vvdPc6oByY15GB7n7A3deF0yeALcCoyJLKWUW5WVwyspAXt72b6igi0kOZezTn7JvZfGCuu98Zzi8EZrv7kpg+cwhuDrUX2A981d03x61nPLAauMTdq83sLuCzQDWwlmDL5LxvrZnZImARQElJSWl5eXmnXkdNTQ35+T3vlqpR5np8Wx1P76jnng/mkp9lPSZXVyhXYpQrMT01F3QtW1lZWYW7zzpvgbtH8gBuAx6MmV8I3BPXpwDID6dvAbbFLc8HKoCPxbSVAOkEW0v/BDzUXpbS0lLvrJUrV3Z6bJSizFWx+4iP+9rT/tT6fQmP7Y/vV1coV2KUK3FdyQas9RY+U6PcVbUXGBMzP5pgq+Isd69295pwejmQaWbFAGaWSbA18it3fyJmTKW7N7p7E/Azgl1i0o2mjS5icF4WK988lOooItIDRVk41gCTzWyCmWUBC4BlsR3MbLiZWTh9WZinKmz7ObDF3b8XN2ZEzOxHgU0RvoZ+KT3NuO7CoazaeohGXX5EROJEVjjcvQFYAjxHcHD7UXffbGaLzWxx2G0+sMnMNgA/AhaEm0dXEeza+mALp91+18w2mtnrQBnw5aheQ39W9r5hHD1Vz3qdlisicSI7HRfO7n5aHtd2f8z0vcC9LYx7CWjxqKy7L+zmmNKCaycXk2awauuhs5dcFxEBfXNcWlGUm8WscYN5/o3KVEcRkR5GhUNaNfeS4bx58AQ7361JdRQR6UFUOKRVcy8ZDsAzmw6mOImI9CQqHNKqkUU5zBhbxDObDqQ6ioj0ICoc0qZbLhnBpn3VvFN1KtVRRKSHUOGQNr23u0pbHSISUOGQNo0ZnMsHRheybMP+9juLSL+gwiHt+uiMUWzeX83Wg7oXuYiocEgH/Pm0kWSkGU+8tjfVUUSkB1DhkHYV5w/guguH8tvX9unaVSKiwiEd87GZo6msPsMfdxxOdRQRSTEVDumQ6y8exsDsDB6v0O4qkf5OhUM6JDsznY9MG8kzmw5y7FRdquOISAqpcEiHffrycZxpaOKxtdrqEOnPVDikwy4eUcCl4wfxyz/tpkkHyUX6LRUOScinLx/H7qpTvLhdB8lF+isVDknI3EuGU5yfxb+9vCvVUUQkRSItHGY218y2mtl2M1vawvI5ZnY85vaw32xvrJkNNrPnzWxb+Kzb0yXRgIx0br9sLCvePMTbh0+mOo6IpEBkhcPM0oH7gJuBKcDtZjalha4vuvv08PGtDoxdCqxw98nAinBekmjhFePITE/jZy/uTHUUEUmBKLc4LgO2u/tOd68DyoF53TB2HvBwOP0wcGv3RZaOGDYwm4/PHM1vKvby7okzqY4jIklm7tGcHWNm84G57n5nOL8QmO3uS2L6zAEeB/YC+4Gvuvvmtsaa2TF3L4pZx1F3P293lZktAhYBlJSUlJaXl3fqddTU1JCfn9+psVFKda6DJ5v4+oun+fAFmXz8wqwek6s1ypUY5UpMT80FXctWVlZW4e6z4tszupyqddZCW3yVWgeMc/caM7sF+C0wuYNj2+TuDwAPAMyaNcvnzJmTyPCzVq1aRWfHRqkn5Fp1pIIXdhzmHxdeRWFOZo/J1RLlSoxyJaan5oJoskW5q2ovMCZmfjTBVsVZ7l7t7jXh9HIg08yK2xlbaWYjAMLnQ9HEl/b87fWTqK5t4EEd6xDpV6IsHGuAyWY2wcyygAXAstgOZjbczCycvizMU9XO2GXAHeH0HcBTEb4GacPUkYX82QdG8NBLb1NVo2MdIv1FZIXD3RuAJcBzwBbg0fD4xWIzWxx2mw9sMrMNwI+ABR5ocWw45tvADWa2DbghnJcU+fKHLuR0fSM/WbUj1VFEJEmiPMbRvPtpeVzb/THT9wL3dnRs2F4FXN+9SaWzJg3L5+MzR/PIy7tZeMW4VMcRkSTQN8ely75600Vkphv/8PQbqY4iIkmgwiFdVlKQzd9eP5nfbznEhncbUh1HRCKmwiHd4nNXTeCC4jx+vaWO2vrGVMcRkQipcEi3yMpI41vzLqHylPP9599KdRwRiZAKh3SbqycXc93oDH724k7WvXM01XFEJCIqHNKtFrwvi+EF2fzdYxu0y0qkj1LhkG6Vk2F8Z/4H2PHuSe5atrn9ASLS66hwSLe7ZvJQvlA2kfI1e3i8QvcnF+lrVDgkEl/+0IXMnjCYv//tJt6qPJHqOCLSjVQ4JBIZ6Wn86PYZ5A1I586H1+paViJ9iAqHRKakIJsHPjOLyupa7nxkrQ6Wi/QRKhwSqZljB/HDBdNZv+cYXypfT0NjU6ojiUgXqXBI5OZeMoK//7MpPLv5IF9+dAP1Kh4ivVqkV8cVafb5qyfQ0NjEPz/zJvUNTfzo9hlkZej/LSK9kf7lStL81XUT+eaHgy2Pz/6fVzl+uj7VkUSkE1Q4JKk+d/UEvveJaazZdYSP/+SP7DlyKtWRRCRBKhySdB+bOZpHPjebQ9W13HrfH3hx27upjiQiCYi0cJjZXDPbambbzWxpG/0uNbNGM5sfzl9kZutjHtVm9qVw2V1mti9m2S1RvgaJxhUTh/DkF65icF4Wn3noVf7lua0640qkl4iscJhZOnAfcDMwBbjdzKa00u87BPcXB8Ddt7r7dHefDpQCp4AnY4Z9v3l5eItZ6YUmDs3nqSVXMX/maO5duZ1bf/wH3thfnepYItKOKLc4LgO2u/tOd68DyoF5LfT7W+Bx4FAr67ke2OHuu6OJKamUm5XB3bdN475PzuTg8Vo+cu9LfOfZNzlRqwPnIj2VuXs0Kw52O8119zvD+YXAbHdfEtNnFPBr4IPAz4Gn3f03cet5CFjn7veG83cBnwWqgbXAV9z9vJs/mNkiYBFASUlJaXl5eadeR01NDfn5+Z0aG6W+mKumzinfWsdL+xoYmAW3TsriutEZZKRZSnNFSbkSo1yJ60q2srKyCnefdd4Cd4/kAdwGPBgzvxC4J67PY8Dl4fQvgPlxy7OAw0BJTFsJkE6wtfRPwEPtZSktLfXOWrlyZafHRqkv59qw56h/4v4/+rivPe3Xffc//dd/2u219Q0pzxUF5UqMciWuK9mAtd7CZ2qUu6r2AmNi5kcD++P6zALKzWwXMB/4sZndGrP8ZoKtjcrmBnevdPdGd28CfkawS0z6kA+MLqJ80eU8+JlZDMzO5OtPbOTa767kgdU7OHqyLtXxRPq9KL85vgaYbGYTgH3AAuCTsR3cfULztJn9gmBX1W9jutwO/HvsGDMb4e4HwtmPApu6PbmknJnxoSklXH/xMP6wvYofr9rO/17+Jv/y3FvcMKWE+bNGc/WkYjLTdUa5SLJFVjjcvcHMlhCcLZVOsEtps5ktDpff39Z4M8sFbgD+Km7Rd81sOuDArhaWSx9iZlw9uZirJxez5UA1j63dy5Ov7eU/Nh6gMCeT6983jBunlnDthUPJzdIVdESSIdJ/aR6cKrs8rq3FguHun42bPwUMaaHfwm6MKL3IxSMK+OafT2Hpze9j5dZDPLf5ICu2HOKJ1/aRmW7MGDuIKycO4apJxUwbXaRrYYlERP9Fk14nKyONm6YO56apw6lvbOLVt4+w+q13+cOOw/xwxTZ+8Ptt5GSmM2NsEdPGFDFtdBGna/XlQpHuosIhvVpmehpXTSrmqknFABw7VccrO4/w8o7DvLbnGA++uJP6xuCU83+u+D0XDR/IRSUDubBkIBcOH8jkYfnkDdA/A5FE6F+M9ClFuVnMvWQ4cy8ZDkBtfSNbDlTzm5VrOZ1TzFuVJ/i3V3ZzpuG9LZDRg3K4sGQg44fkMaE4l/HFeYwfksfIohzSu+E7JCJ9jQqH9GnZmenMGDuI4+MymTNnOgCNTc6eI6fYWnmCbZUn2FpZw7bKE/xxx2Fq698rKFnpaYwdkntOQRkzKJfi/AEUD8xicG4WGTqrS/ohFQ7pd9LTLNiqKM7jpqnDz7Y3NTmVJ2p5+/BJdh0+xa6qk+w6fJJdVSdZve1d6hrOPU5iBkU5mRTnD2BIfhZD8gdQkJ1JQU5G8JydQUFO5rltOZkMzM4gJzMdM23NSO+kwiESSkszRhTmMKIwhysnnrusqck5UF3LvqOnqao5w+GaMxyuqaPq5BkOnwiet+yvprq2gerT9dS1c6XfjDSjICeTTK9nyIYXyc1KJycrnZzM9HA6I2Y6vYXpDHKy0sjJzHivPSud3Mx0bQVJ5FQ4RDogLc0YVZTDqKKcDvWvrW+kuraeE2EhqT77XE/16QZO1AbT23btI78wm9P1jZyobeBQ9RlO1zdyqq6R03UNnKpvJNHLyWWmW0xxCQpN83P22em0s/PZMctzMtMZkJnGjsoG0re923qfjDTSdPyn31LhEIlA84ftsIFt91u1qoo5cy5tdbm7c6ahidN1jWcLSm34fKquIWa6sYXpBk7XNwUFKOxfdbKO2vpg+en6Rk7XNZ5zosA5Xnu1zewDMtLOKSbNxSU7pijlnG2Lmc9MO7/tbGFLO3dcRroKVA+kwiHSg5nZ2Q/YQRH9jKYmp7ahkdr6prPF5KWX/8TUaTOCAhMWrebp2rCQxRaf+LZjp+qDMeHYYHznvkuTlZF2tjh5wxkGb3iRnJgCk31O8Qr65mdnkD8gOJ6Un51BQXYGA7MzyR+QwcDsDPKyMlSQukCFQ6SfS0szcrMyyM16r21vYTqXjh/crT8nfuupuRgFBamphbb3+p2pD8bt3refgYNyzvaprq0PC1bTe4WsA7v3zCA/KygiA7Mzyc8OpvMHBCc0FLbxKMjJZOCA/l14VDhEJCm6Y+tp1aojzJlz/u0hYrk7p+qCY0Ynaus5caaBE7UN1DTP1zaEbfXvtZ+p58jJOnZXnaL6dD3HT9fT0NR69UkzGJj9XjFpqj3NY/vWtVh0inIzGZSbxeC8LIpyM8nOTO/kq+85VDhEpE8xM/IGZJA3IIPhhdmdWkdz8TkeFpHYR3ULbXtqYMvB6rPLmq9W0JK8rHQG5QWFpLmgBM+ZQXtu1jnLi3Ize9xVoFU4RETixBafkR04k27VqlXMmTMHCIrO6fr3is6xU/UcO1XHkZP1HD1Vx5GTdRw9WceRU8HzzsM1HD1ZT82ZhlbXX5CdERSSsJi0VWgG52VRmJMZ6VUPVDhERLqRWfMxowxGFHbs9G2AMw3BSQXxheWcgnOqjsrqWt48UE3VybpWz4hr/nLqoLws/mJCI3O66bU1U+EQEekBBmSkU1KQTklBx3evna5rjCkwdTEFpp4jJ89w9GQ9uZnHuj2rCoeISC+Vk5XOqKy2v5i6atWqbv+5PeuIi4iI9HiRFg4zm2tmW81su5ktbaPfpWbWaGbzY9p2mdlGM1tvZmtj2geb2fNmti18jup7USIi0oLICoeZpQP3ATcDU4DbzWxKK/2+Q3Bv8nhl7j7d3WNP3F4KrHD3ycCKcF5ERJIkyi2Oy4Dt7r7T3euAcmBeC/3+FngcONTB9c4DHg6nHwZu7WJOERFJgHmil97s6IqD3U5z3f3OcH4hMNvdl8T0GQX8Gvgg8HPgaXf/TbjsbeAo4MBP3f2BsP2YuxfFrOOou5+3u8rMFgGLAEpKSkrLy8s79TpqamrIz8/v1NgoKVdilCsxypWYnpoLupatrKysIm6PT8DdI3kAtwEPxswvBO6J6/MYcHk4/QtgfsyykeHzMGADcG04fyxuHUfby1JaWuqdtXLlyk6PjZJyJUa5EqNciempudy7lg1Y6y18pkZ5Ou5eYEzM/Ghgf1yfWUB5eCe0YuAWM2tw99+6+34Adz9kZk8S7PpaDVSa2Qh3P2BmI+j4Li4REekGUR7jWANMNrMJZpYFLACWxXZw9wnuPt7dxwO/Af7G3X9rZnlmNhDAzPKAG4FN4bBlwB3h9B3AUxG+BhERiRPZFoe7N5jZEoKzpdKBh9x9s5ktDpff38bwEuDJcEskA/i1uz8bLvs28KiZfR54h2CXWJsqKioOm9nuTr6UYuBwJ8dGSbkSo1yJUa7E9NRc0LVs41pqjOzgeF9hZmu9pYNDKaZciVGuxChXYnpqLogmm745LiIiCVHhEBGRhKhwtO+BVAdohXIlRrkSo1yJ6am5IIJsOsYhIiIJ0RaHiIgkRIVDREQSosLRio5eEj6inz3GzFaa2RYz22xmXwzb7zKzfeGl5teb2S0xY74eZt1qZjdFmO28y923dan7ZOQys4ti3pP1ZlZtZl9K1ftlZg+Z2SEz2xTTlvB7ZGal4Xu93cx+ZOEXm7o5191m9qaZvW5mT5pZUdg+3sxOx7x398eMSUauhH93Scr1f2My7TKz9WF7Mt+v1j4fkvc31tJ1SPr7g+ALizuAC4AsgmtlTUnizx8BzAynBwJvEVya/i7gqy30nxJmHABMCLOnR5RtF1Ac1/ZdYGk4vRT4TrJzxf3uDhJ8cSkl7xdwLTAT2NSV9wh4FbgCMOAZ4OYIct0IZITT34nJNT62X9x6kpEr4d9dMnLFLf9X4JspeL9a+3xI2t+Ytjha1tFLwkfC3Q+4+7pw+gSwBRjVxpB5QLm7n3H3t4HtBK8hWVq71H0qcl0P7HD3tq4UEGkud18NHGnhZ3b4PbLgOmwF7v6yB//CH6GLtxBoKZe7/87dG8LZVwiuKdeqZOVqQ0rfr2bh/8w/Afx7W+uIKFdrnw9J+xtT4WjZKGBPzPxe2v7gjoyZjQdmAH8Km5aEuxUeitkUTWZeB35nZhUWXLoeoMTdD0DwR01wReNk52q2gHP/Maf6/WqW6Hs0KpxOZsbPEfyvs9kEM3vNzF4ws2vCtmTmSuR3l+z36xqg0t23xbQl/f2K+3xI2t+YCkfLWtrPl/Tzls0sn+AmV19y92rgJ8BEYDpwgGBTGZKb9yp3n0lwZ8cvmNm1bfRN6vtowcU0P0JwuX7oGe9Xe1rLkuz37htAA/CrsOkAMNbdZwD/Ffi1mRUkMVeiv7tk/05v59z/oCT9/Wrh86HVrq1k6HQ2FY6WdeSS8JEys0yCP4pfufsTAO5e6e6N7t4E/Iz3dq8kLa/HXO4eaL7cfWW42du8ad58qftkv483A+vcvTLMmPL3K0ai79Fezt1tFFlGM7sD+DDwqXCXBeFujapwuoJgv/iFycrVid9dMt+vDOBjwP+NyZvU96ulzweS+DemwtGydi8JH6Vw/+nPgS3u/r2Y9hEx3T7KuZeaX2BmA8xsAjCZ4KBXd+dq7XL3rV3qPim5Ypzzv8BUv19xEnqPwl0NJ8zs8vDv4TNEcAsBM5sLfA34iLufimkfambp4fQFYa6dScyV0O8uWblCHwLedPezu3mS+X619vlAMv/GunJ0vy8/gFsIzlbYAXwjyT/7aoJNxteB9eHjFuDfgI1h+zJgRMyYb4RZt9LFszbayHUBwdkZG4DNze8LMARYAWwLnwcnM1f4c3KBKqAwpi0l7xdB8ToA1BP8r+7znXmPCG50tilcdi/hlR66Odd2gv3fzX9n94d9Px7+jjcA64A/T3KuhH93ycgVtv8CWBzXN5nvV2ufD0n7G9MlR0REJCHaVSUiIglR4RARkYSocIiISEJUOEREJCEqHCIikhAVDpEezszmmNnTqc4h0kyFQ0REEqLCIdJNzOzTZvZqeD+Gn5pZupnVmNm/mtk6M1thZkPDvtPN7BV77z4Yg8L2SWb2ezPbEI6ZGK4+38x+Y8G9M37V4fsmiERAhUOkG5jZxcBfEFwEcjrQCHwKyCO4ftZM4AXgf4ZDHgG+5u4fIPiGdHP7r4D73H0acCXBN5chuALqlwjurXABcFXEL0mkVRmpDiDSR1wPlAJrwo2BHIKLzDXx3sXwfgk8YWaFQJG7vxC2Pww8Fl4HbJS7Pwng7rUA4fpe9fDaSBbcdW488FLkr0qkBSocIt3DgIfd/evnNJr9j7h+bV3jp63dT2diphvRv11JIe2qEukeK4D5ZjYMzt7/eRzBv7H5YZ9PAi+5+3HgaMzNfhYCL3hwT4W9ZnZruI4BZpabzBch0hH6X4tIN3D3N8zs7wnujphGcEXVLwAngalmVgEcJzgOAsFlr+8PC8NO4C/D9oXAT83sW+E6bkviyxDpEF0dVyRCZlbj7vmpziHSnbSrSkREEqItDhERSYi2OEREJCEqHCIikhAVDhERSYgKh4iIJESFQ0REEvL/AcRaYES9HCOTAAAAAElFTkSuQmCC\n" 132 | }, 133 | "metadata": { 134 | "needs_background": "light" 135 | } 136 | } 137 | ], 138 | "source": [ 139 | "y_loss = [] \n", 140 | "max_epoch = 2001\n", 141 | "for epoch in range(1, max_epoch):\n", 142 | " # forward\n", 143 | " y_pred = model(x_data)\n", 144 | " # 注意在使用二元交叉熵计算损失的维度关系(计算之前还需要激活)，这里的y_pred和y_data维度都是([759, 1])\n", 145 | " loss = criterion(y_pred, y_data)\n", 146 | " if epoch % 100 == 0:\n", 147 | " print(epoch, loss.item())\n", 148 | " y_loss.append(loss.item())\n", 149 | " # backward\n", 150 | " optimizer.zero_grad()\n", 151 | " loss.backward()\n", 152 | " # update\n", 153 | " optimizer.step()\n", 154 | "\n", 155 | "\n", 156 | "# 展示epoch-loss变化图像\n", 157 | "x_epoch = np.arange(1, max_epoch)\n", 158 | "plt.plot(x_epoch, y_loss)\n", 159 | "plt.xlabel('epoch')\n", 160 | "plt.ylabel('loss')\n", 161 | "plt.grid()\n", 162 | "plt.show()\n", 163 | "\n" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 7, 169 | "metadata": {}, 170 | "outputs": [ 171 | { 172 | "output_type": "stream", 173 | "name": "stdout", 174 | "text": [ 175 | "torch.Size([759, 1])\ntorch.Size([759, 1])\n" 176 | ] 177 | } 178 | ], 179 | "source": [ 180 | "print(y_data.shape)\n", 181 | "print(y_pred.shape)\n" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [] 190 | } 191 | ], 192 | "metadata": { 193 | "kernelspec": { 194 | "name": "python3", 195 | "display_name": "Python 3.6.12 64-bit ('pytorch_dl': conda)", 196 | "metadata": { 197 | "interpreter": { 198 | "hash": "6b962ed846ab7a2f9c4286c5c9f447c6ba721d3b496bd62adf19906831909870" 199 | } 200 | } 201 | }, 202 | "language_info": { 203 | "codemirror_mode": { 204 | "name": "ipython", 205 | "version": 3 206 | }, 207 | "file_extension": ".py", 208 | "mimetype": "text/x-python", 209 | "name": "python", 210 | "nbconvert_exporter": "python", 211 | "pygments_lexer": "ipython3", 212 | "version": "3.6.12-final" 213 | } 214 | }, 215 | "nbformat": 4, 216 | "nbformat_minor": 2 217 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pytorch-Learning 2 | Pytorch Framework learning for deeplearning 3 | 4 | ## CNN 5 | ### 基础知识 6 | - 参考 [CNN](./CNN/CNN.md) 7 | 8 | ### 相关应用 9 | #### MINST Classification 10 | - [CNN_MINST.ipynb](./CNN/CNN_MINST.ipynb): 使用 [torch.nn.RNN](https://pytorch.org/docs/stable/generated/torch.nn.RNN.html) 做手写数字书识别 11 | - 其中使用的是`./dataset/MINST`文件夹的`names_test.csv.gz`和`names_train.csv.gz`数据集 12 | 13 | ## RNN 14 | ### 基础知识 15 | - 需要理解输入维度和隐层维度： 16 | 17 | ```python 18 | # RNN需要指明输入大小、隐层大小以及层数（默认为1） 19 | cell = torch.nn.RNN(input_size, hidden_size, num_layers) 20 | 21 | # input: (seq_len, batch_size, input_size) 所有时间点的第一层状态 22 | # hidden: (num_layers, batch_size, hidden_size) 第一个时刻所有层的状态 23 | 24 | # out: (seq_len, batch_size, hidden_size) 所有时间点的最后一层状态 25 | # hidden: (num_layers, batch_size, hidden_size) 最后时刻的所以层状态 26 | out, hidden = cell(inputs, hidden) 27 | ``` 28 | 29 | - 自然语言的输入处理需要学会word embedding 30 | 31 | - 另外可以参考 [RNN](./RNN/RNN.md) 32 | 33 | ### 相关应用 34 | 35 | [RNNcell.ipynb](./RNN/RNNcell.ipynb): 学习使用 [torch.nn.RNNcell](https://pytorch.org/docs/stable/generated/torch.nn.RNNCell.html?highlight=rnncell#torch.nn.RNNCell), 用于`hello --> ohlol` 36 | - 其实也是一个分类问题 37 | 38 | #### RNN for Regression 39 | - [RNN_Regression.py](./RNN/RNN_Regression.py): 使用 [torch.nn.RNN](https://pytorch.org/docs/stable/generated/torch.nn.RNN.html) 模拟`sin(x)`逼近`cos(x)` 40 | - 效果图： 41 | 42 | ![RNN_Regression](https://i.loli.net/2021/03/12/4ozBxbLsX1c6f3J.gif) 43 | 44 | 45 | #### GRU for Classification 46 | 47 | - [GRU_Classifier.ipynb](./RNN/GRU_Classifier.ipynb): 使用 [torch.nn.RNN](https://pytorch.org/docs/stable/generated/torch.nn.GRU.html) 训练名字到国家的分类，即输入名字输出其属于哪个国家的 48 | - 其中使用的数据集在`./dataset`文件夹中 49 | - 对应的Python脚本文件可以参考: [GRU_Classifier.py](./RNN/GRU_Classifier.py) 50 | 51 | #### LSTM for Regression and Prediction 52 | - [LSTM_Regression.py](./LSTM/LSTM_Regression.py): 使用 [torch.nn.LSTM](https://pytorch.org/docs/stable/generated/torch.nn.LSTM.html) 模拟`sin(x)`逼近`cos(x)` 53 | - 效果图: 54 | 55 | ![LSTM_Regression](https://i.loli.net/2021/03/12/7OJvI1sP26HuzAF.gif) 56 | 57 | - [LSTM_airplane_forcast.ipynb](./LSTM/LSTM_airplane_forcast.ipynb): 根据前9年的数据预测后3年的客流, 这个为了训练过程简单使用的数据是: `./dataset/airplane_data.csv`，只有144条数据，所以训练效果不是那么好，只是为了简单理解LSTM做回归分析的方法 58 | 59 | **- 这个例子好像是有点问题的：根据[简书](https://www.jianshu.com/p/18f397d908be)的评论** 60 | 61 | > 1. Data leakage的问题：数据的预处理放在了数据集分割之前，测试集的信息泄露给了训练集 62 | > 2. 下面讨论最多的input_size和seq_len的问题：若本例目的是"以t-2,t-1的数据预测t的数据"的话 63 | > 根据Pytorch的DOC定义“input of shape (seq_len, batch, input_size)” 64 | > 而本例的输入维度从这可以看出来了train_X = train_X.reshape(-1, 1, 2) 65 | > 说明input_size=2 batch=1 seq_len=？（我没算），不过这似大概没能用到LSTM的特性，或者说没法用"以t-2,t-1的数据预测t的数据"来解释本结构的目的。 66 | > 我比较同意上面讨论的人的看法，即特征数（input_size）为1，seq_len为2，应该是比较合理的 67 | 68 | 69 | 70 | 71 | 72 | ## Convolutional LSTM 73 | 74 | 一般来说CNN可以提取图片的空间特征，LSTM可以提取时间特征，如果有时间序列的图片场景，我们可以使用 **Convolutional LSTM (ConvLSTM)** 提取**时空特征**（Spatio-temporal features），该网络由香港科技大学的 Shi Xingjian 等人提出，具体的论文可以参考：[Convolutional LSTM Network: A Machine Learning Approach for Precipitation Nowcasting](https://arxiv.org/abs/1506.04214) 75 | 76 | ![ConvLSTM](https://s2.loli.net/2022/01/11/n7FSVUbqstr28dj.png) 77 | 78 | > 代码参考：[ConvLSTM.py](./ConvLSTM.py)，其中主要有四个类： 79 | > 80 | > - `ConvLSTMCell`是卷积LSTM的细胞单元：通过卷积操作计算之后返回LSTM中隐层状态和细胞状态； 81 | > - `ConvLSTM`是实现卷积LSTM提取时空特征的一个模型：输入格式为 $(B, T, C, H, W)$ 或者 $(T, B, C, H, W)$，B是batchSize，T是timeLength或seqLen，CHW分别是图片的channel、height和width 82 | > - PyTorch中CNN的输入形状为：$(B,C_{in},H,W)$ ，[参考链接](https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html?highlight=conv#torch.nn.Conv2d) 83 | > - PyTorch中LSTM的输入形状为：$(B,T,Hidden_{in})$ 或者 $(T,B,Hidden_{in})$ ，[参考链接](https://pytorch.org/docs/stable/generated/torch.nn.LSTM.html?highlight=lstm#torch.nn.LSTM) 84 | > - `outputCNN`：一个简单的CNN网络处理ConvLSTM的输出 85 | > - 一般LSTM的输出后面都会接一个Linear层处理，这里ConvLSTM的输出就是使用CNN处理 86 | > - `ConvLSTM_model`：使用ConvLSTM以及outputCNN的混合网络 87 | 88 | 89 | 90 | ## Transformer 91 | 92 | - 参考 [transformer.md](./Transformer.md) 93 | 94 | ## 其他 95 | 1. [multiple_dimension_diabetes.ipynb](./LinearNetwork/multiple_dimension_diabetes.ipynb): 学习处理多维特征输入，使用**二分类交叉熵** [torch.nn.BCELoss](https://pytorch.org/docs/stable/generated/torch.nn.BCELoss.html?highlight=bce#torch.nn.BCELoss) 96 | - 使用的`./dataset`文件夹中的`diabetes.csv.gz`数据集 97 | 98 | 2. [softmax_classifier.ipynb](./softmax_classifier.ipynb): 学习处理多维特征分类，使用**交叉熵** [torch.nn.CrossEntropyLoss](https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html?highlight=crossentropy#torch.nn.CrossEntropyLoss) 99 | - 使用的是`./dataset/MINST`数据集 100 | 101 | -------------------------------------------------------------------------------- /RNN/GRU_Classifier.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import torch 5 | from torch.utils.data import Dataset 6 | from torch.utils.data import DataLoader 7 | import gzip 8 | import csv 9 | 10 | class NameDataset(Dataset): 11 | def __init__(self, is_train_set): 12 | filename = './dataset/names_train.csv.gz' if is_train_set else './dataset//names_test.csv.gz' 13 | with gzip.open(filename, 'rt') as f: # r表示只读，从文件头开始 t表示文本模式 14 | reader = csv.reader(f) 15 | rows = list(reader) 16 | self.names = [row[0] for row in rows] 17 | self.len = len(self.names) 18 | self.countries = [row[1] for row in rows] 19 | 20 | self.country_list = list(sorted(set(self.countries))) 21 | self.country_dict = self.getCountryDict() 22 | self.country_num = len(self.country_list) 23 | 24 | def __getitem__(self, index): # 根据索引拿到的是 名字，国家的索引 25 | return self.names[index], self.country_dict[self.countries[index]] 26 | 27 | def __len__(self): 28 | return self.len 29 | 30 | def getCountryDict(self): 31 | country_dict = dict() 32 | for idx, country_name in enumerate(self.country_list, 0): 33 | country_dict[country_name] = idx 34 | return country_dict 35 | 36 | def idx2country(self, index): 37 | return self.country_list[index] 38 | 39 | def getCountriesNum(self): 40 | return self.country_num 41 | 42 | HIDDEN_SIZE = 100 43 | BATCH_SIZE = 256 44 | N_LAYER = 2 45 | N_EPOCHS = 50 46 | N_CHARS = 128 # 这个是为了构造嵌入层 47 | 48 | trainSet = NameDataset(is_train_set=True) 49 | trainLoader = DataLoader(trainSet, batch_size=BATCH_SIZE, shuffle=True) 50 | 51 | testSet = NameDataset(is_train_set=False) 52 | testLoader = DataLoader(testSet, batch_size=BATCH_SIZE, shuffle=False) 53 | 54 | N_COUNTRY = trainSet.getCountriesNum() 55 | 56 | 57 | 58 | for idx, (names, countries) in enumerate(trainLoader): 59 | print(names.__len__(), type(countries), countries.shape) 60 | if idx == 0: 61 | break 62 | 63 | 64 | class RNNClassifier(torch.nn.Module): 65 | def __init__(self, input_size, hidden_size, output_size, n_layers=1, bidirectional=True): 66 | super(RNNClassifier, self).__init__() 67 | self.hidden_size = hidden_size 68 | self.n_layers = n_layers 69 | self.n_directions = 2 if bidirectional else 1 # 使用双向的GRU 70 | 71 | # 嵌入层（𝑠𝑒𝑞𝐿𝑒𝑛, 𝑏𝑎𝑡𝑐ℎ𝑆𝑖𝑧𝑒） --> (𝑠𝑒𝑞𝐿𝑒𝑛, 𝑏𝑎𝑡𝑐ℎ𝑆𝑖𝑧𝑒, hidden_size) 72 | self.embedding = torch.nn.Embedding(input_size, hidden_size) 73 | self.gru = torch.nn.GRU(hidden_size, hidden_size, n_layers, bidirectional=bidirectional) 74 | self.fc = torch.nn.Linear(hidden_size * self.n_directions, output_size) 75 | 76 | def _init_hidden(self, batch_size): 77 | hidden = torch.zeros(self.n_layers * self.n_directions, batch_size, self.hidden_size) 78 | return hidden 79 | 80 | def forward(self, x_input, seq_lengths): 81 | # input shape : B x S -> S x B 82 | x_input = x_input.t() 83 | batch_size = x_input.size(1) 84 | hidden = self._init_hidden(batch_size) 85 | embedding = self.embedding(x_input) 86 | 87 | # pack them up 88 | gru_input = torch.nn.utils.rnn.pack_padded_sequence(embedding, seq_lengths) 89 | output, hidden = self.gru(gru_input, hidden) 90 | if self.n_directions == 2: 91 | hidden_cat = torch.cat([hidden[-1], hidden[-2]], dim=1) 92 | else: 93 | hidden_cat = hidden[-1] 94 | fc_output = self.fc(hidden_cat) 95 | return fc_output 96 | 97 | 98 | def name2list(name): 99 | arr = [ord(c) for c in name] 100 | return arr, len(arr) 101 | 102 | def make_tensors(names, countries): 103 | sequences_and_lengths = [name2list(name) for name in names] 104 | name_sequences = [s1[0] for s1 in sequences_and_lengths] 105 | seq_lengths = torch.LongTensor([s1[1] for s1 in sequences_and_lengths]) 106 | countries = countries.long() 107 | 108 | # make tensor of name, BatchSize * seqLen 109 | # 他这里补零的方式先将所有的0 Tensor给初始化出来，然后在每行前面填充每个名字 110 | seq_tensor = torch.zeros(len(name_sequences), seq_lengths.max()).long() 111 | # print("seq_lengths.max:", seq_lengths.max()) 112 | for idx, (seq, seq_len) in enumerate(zip(name_sequences, seq_lengths), 0): 113 | seq_tensor[idx, :seq_len] = torch.LongTensor(seq) 114 | 115 | # sort by length to use pack_padded_sequence 116 | # 将名字长度降序排列，并且返回降序之后的长度在原tensor中的小标perm_idx 117 | seq_lengths, perm_idx = seq_lengths.sort(dim=0, descending=True) 118 | # 这个Tensor中的类似于列表中切片的方法神奇啊，直接返回下标对应的元素，相等于排序了 119 | seq_tensor = seq_tensor[perm_idx] 120 | countries = countries[perm_idx] 121 | 122 | # 返回排序之后名字Tensor，排序之后的名字长度Tensor，排序之后的国家名字Tensor 123 | return seq_tensor, seq_lengths, countries 124 | 125 | 126 | # ### 训练数据 127 | 128 | # In[5]: 129 | 130 | 131 | classifier = RNNClassifier(N_CHARS, HIDDEN_SIZE, N_COUNTRY, N_LAYER) 132 | # if torch.cuda.is_available(): 133 | # classifier = classifier.cuda() 134 | 135 | criterion = torch.nn.CrossEntropyLoss() 136 | optimizer = torch.optim.Adam(classifier.parameters(), lr=0.001) 137 | 138 | import time 139 | import math 140 | 141 | def trainModel(): 142 | def time_since(since): 143 | s = time.time() - since 144 | m = math.floor(s / 60) 145 | s -= m * 60 146 | return '%dm %ds' % (m, s) 147 | 148 | total_loss = 0 149 | for i, (names, countries) in enumerate(trainLoader, 1): 150 | # print(type(names), type(countries)) 151 | # print(len(names), countries.shape) 152 | inputs, seq_lengths, target = make_tensors(names, countries) 153 | # if torch.cuda.is_available(): 154 | # inputs = inputs.cuda() 155 | # seq_lengths = seq_lengths.cuda() 156 | # target = target.cuda() 157 | 158 | output = classifier(inputs, seq_lengths) 159 | # print("Shape:", output.shape, target.shape) 160 | # 注意输出和目标的维度：Shape: torch.Size([256, 18]) torch.Size([256]) 161 | loss = criterion(output, target) 162 | optimizer.zero_grad() 163 | loss.backward() 164 | optimizer.step() 165 | 166 | total_loss += loss.item() 167 | if i % 10 == 0: 168 | print(f'[{time_since(start)}] Epoch {epoch} ', end='') 169 | print(f'[{i * len(inputs)}/{len(trainSet)}] ', end='') 170 | print(f'loss={total_loss / (i * len(inputs))}') 171 | return total_loss 172 | 173 | def testModel(): 174 | correct = 0 175 | total = len(testSet) 176 | print("evaluating trained model ... ") 177 | with torch.no_grad(): 178 | for i, (names, countries) in enumerate(testLoader): 179 | inputs, seq_lengths, target = make_tensors(names, countries) 180 | output = classifier(inputs, seq_lengths) 181 | # 注意这个keepdim的使用，为了直接和target计算loss 182 | pred = output.max(dim=1, keepdim=True)[1] 183 | # 注意这个view_as 和 eq 184 | correct += pred.eq(target.view_as(pred)).sum().item() 185 | 186 | percent = '%.2f' % (100 * correct / total) 187 | print(f'Test set: Accuracy {correct}/{total} {percent}%') 188 | 189 | return correct / total 190 | 191 | 192 | N_EPOCHS = 50 193 | start = time.time() 194 | print("Training for %d epochs..." % N_EPOCHS) 195 | acc_list = [] 196 | for epoch in range(1, N_EPOCHS + 1): 197 | # Train cycle 198 | trainModel() 199 | acc = testModel() 200 | acc_list.append(acc) 201 | 202 | 203 | 204 | import matplotlib.pyplot as plt 205 | import numpy as np 206 | 207 | epoch = np.arange(1, len(acc_list) + 1) 208 | acc_list = np.array(acc_list) 209 | plt.plot(epoch, acc_list) 210 | plt.xlabel('Epoch') 211 | plt.ylabel('Accuracy') 212 | plt.grid() 213 | plt.show() 214 | 215 | -------------------------------------------------------------------------------- /RNN/RNN.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # 循环神经网络 4 | 5 | ## 先看RNN 6 | 7 | ### 基本结构 8 | 9 | LSTM源于**Recurrent Nerual Network**（RNN），RNN和全连接神经网络的原理并无区别，都是基于梯度下降的原则，只是在结构上有点区别，首先看一下RNN结构： 10 | 11 | RNN 12 | 13 | RNN有两个输入： 14 | 15 | 1. **当前时刻输入xt，用于实时更新状态** 16 | 2. **上一时刻隐藏层的状态ht-1，用于记忆状态，而不同时刻的网络共用的是同一套参数$（U，W，b）$** 17 | 18 | 19 | 20 | ### 优化方式 21 | 22 | 1. 全连接神经网络一般通过 **反向传播（BackPropagation）** 优化 23 | 2. RNN通过**BackPropagation Through Time**（BPTT）优化，原理一样，只是这个需要考虑时间序列 24 | 25 | 26 | 27 | ### 结构和优势 28 | 29 | - **one to many**：输入图像，输出描述图像的文本 30 | - **many to many**：输入输出等长，在线作诗机器人 31 | - **many to many**：输入输出不等长，**Seq2Seq模型**，在线翻译；另外还有**Attention Mechanism**，突出重点 32 | - 经典模型：LSTM、GRU 33 | 34 | RNN输入是有序的，可以**模拟人类阅读的顺序**去读取文本或者别的序列化数据，且通过隐藏层神经元的编码，**上一个隐藏层神经元的信息可以传递到下一个隐藏层神经元，所以形成了一定的记忆能力**，可以更好地理解序列化数据 35 | 36 | 37 | 38 | ### 存在问题 39 | 40 | RNN存在 **梯度爆炸（Gradient Explode）和梯度消失（Gradient Vanish）** 的问题。而且，在前向过程中，开始时刻的输入对后面时刻的影响越来越小，这就是长距离依赖问题。这样一来，就失去了 **“记忆”** 的能力，要知道生物的神经元拥有对过去时序状态很强的记忆能力。 41 | 42 |
43 | 梯度爆炸和梯度消失 44 | 梯度爆炸和梯度消失 45 |
46 | 47 | 48 | 49 | 50 | 51 | > 为了解决RNN的梯度消失的问题，提出了LSTM 52 | 53 | 54 | 55 | ## 再看LSTM 56 | 57 | - 三个门控信号：**输入门、遗忘门、输出门** 58 | - 和RNN不同在于：LSTM有四个输入**（$Z，Z_i，Z_f，Z_o$）** 59 | 60 |
61 | LSTM结构 62 | LSTM结构 63 |
64 | 65 | 66 | 67 | 68 | 69 | > 说明： 70 | > 71 | > 1. 作用于$Z_i, Z_f, Z_o$的函数：通常是Sigmoid函数，即$\sigma(z) = \frac{1}{1+e^{-z}}$，让它们的输出在0\~1之间 72 | > 2. Forget Gate：当$f(Z_f) = 1$时表示记住这个输入，反之表示遗忘 73 | > 3. 上右图中：$c'=g(Z)f(Z_i)+cf(Z_f)$ 74 | > 4. 和四个输入相关参数是不一样的 75 | 76 | **为什么LSTM可以处理梯度消失呢？** 77 | 78 | - RNN中每个时间点memory里面的信息都会被”冲掉“，因为每个时间点neuron的output都会被放到memory里面去 79 | - LSTM中memory里面的信息是和input相加的，如果memory受到的影响会直接存在，除非$f(Z_f) = 0$ 。所以一般确保给到Forget Gate给到的参数比较大，这样就可以保证到forget gate经常处于开放状态 80 | - 现在还有一种网络GRU（**Gate Recurrent Unit**），只有两个Gate，所以参数相对于LSTM较少，可以防止**overfitting**。具体来说它是将**Input Gate**和**Forget Gate**联动起来，当**Input Gate**被打开时，**Forget Gate**自动关闭忘记memory中的信息；当**Forget Gate**打开时，**Input Gate**关闭，只要memory里面的值，也就是说只有把memory里面的值”清洗“掉，新的值才会被放进来。 81 | - 还有很多其他的方法避免gradient vanish：比如**Clockwise RNN、Structurally Constrained Recurrent Network（SCRN）**等等 82 | 83 | 84 | 85 | ## 参考文献 86 | 87 | 1. b站 李宏毅 https://www.bilibili.com/video/BV1JE411g7XF?t=4028&p=20 88 | 2. b站 [阿力阿哩哩](https://space.bilibili.com/299585150) [RNN](https://www.bilibili.com/video/BV177411f7RM?t=587) https://www.bilibili.com/video/BV177411f7ad?t=515 89 | 3. 简书 [如何简单的理解LSTM——其实没有那么复杂](https://www.jianshu.com/p/4b4701beba92) 90 | 4. https://colah.github.io/posts/2015-08-Understanding-LSTMs/ -------------------------------------------------------------------------------- /RNN/RNN_Regression.py: -------------------------------------------------------------------------------- 1 | """ 2 | 来源：莫凡Pytorch教学 3 | 源作者：Troublemaker 4 | 日期：2020/4/11 10:59 5 | 脚本：rnn_regression.py 6 | 修改者：EricPengShuai 7 | 模拟sin函数逼近cos函数 8 | """ 9 | import torch 10 | import torch.nn as nn 11 | import numpy as np 12 | import matplotlib.pyplot as plt 13 | import time 14 | 15 | class RNN(nn.Module): 16 | """搭建rnn网络""" 17 | def __init__(self): 18 | super(RNN, self).__init__() 19 | self.rnn = nn.RNN( 20 | input_size=input_size, 21 | hidden_size=hidden_size, 22 | num_layers=num_layers, 23 | batch_first=True,) 24 | self.output_layer = nn.Linear(in_features=hidden_size, out_features=output_size) 25 | 26 | def forward(self, x, hc): 27 | # x (batch, time_step, input_size) 28 | # h_state (n_layers, batch, hidden_size) 29 | # rnn_out (batch, time_step, hidden_size) 30 | rnn_out, hc = self.rnn(x, hc) # h_state是之前的隐层状态 31 | 32 | out = [] 33 | for time in range(rnn_out.size(1)): 34 | every_time_out = rnn_out[:, time, :] # 相当于获取每个时间点上的输出，然后过输出层 35 | temp = self.output_layer(every_time_out) 36 | out.append(temp) 37 | # print(f'Time={time}', rnn_out.shape, every_time_out.shape, temp.shape, len(out)) 38 | return torch.stack(out, dim=1), hc # torch.stack扩成[1, output_size, 1] 39 | 40 | # 设置超参数 41 | input_size = 1 42 | output_size = 1 43 | num_layers = 1 44 | hidden_size = 32 45 | learning_rate = 0.02 46 | train_step = 100 47 | time_step = 10 48 | 49 | # 准备数据 50 | steps = np.linspace(0, 2*np.pi, 100, dtype=np.float32) 51 | x_np = np.sin(steps) 52 | y_np = np.cos(steps) 53 | 54 | # plt.plot(steps, y_np, 'r-', label='target (cos)') 55 | # plt.plot(steps, x_np, 'b-', label='input (sin)') 56 | # plt.legend(loc='best') 57 | # plt.show() 58 | 59 | rnn = RNN() 60 | print(rnn) 61 | 62 | # 设置优化器和损失函数 63 | optimizer = torch.optim.Adam(rnn.parameters(), lr=learning_rate) 64 | loss_function = nn.MSELoss() 65 | startTime = time.time() 66 | 67 | plt.figure(1, figsize=(12, 5)) 68 | plt.ion() 69 | 70 | # 训练 71 | h_state = None # 初始化隐藏层状态 72 | 73 | for step in range(train_step): 74 | start, end = step * np.pi, (step+1) * np.pi 75 | steps = np.linspace(start, end, time_step, dtype=np.float32) 76 | x_np = np.sin(steps) 77 | y_np = np.cos(steps) 78 | 79 | x = torch.from_numpy(x_np[np.newaxis, :, np.newaxis]) 80 | y = torch.from_numpy(y_np[np.newaxis, :, np.newaxis]) 81 | # print('x y shape:', x.shape, y.shape) 82 | 83 | pridect, h_state = rnn(x, h_state) 84 | h_state = h_state.detach() # 重要！！！ 需要将该时刻隐藏层的状态作为下一时刻rnn的输入 85 | 86 | if step == train_step - 1: 87 | endTime = time.time() 88 | print(f'TimeSum={round(endTime - startTime, 4)}s') 89 | # exit() 90 | loss = loss_function(pridect, y) 91 | optimizer.zero_grad() 92 | 93 | loss.backward() 94 | optimizer.step() 95 | 96 | # plotting 97 | plt.plot(steps, y_np.flatten(), 'r-') 98 | plt.plot(steps, pridect.detach().numpy().flatten(), 'b-') 99 | plt.draw() 100 | plt.pause(0.05) 101 | 102 | plt.ioff() 103 | plt.show() 104 | 105 | -------------------------------------------------------------------------------- /RNN/RNNcell.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.6.12-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3.6.12 64-bit ('pytorch_dl': conda)", 19 | "metadata": { 20 | "interpreter": { 21 | "hash": "6b962ed846ab7a2f9c4286c5c9f447c6ba721d3b496bd62adf19906831909870" 22 | } 23 | } 24 | } 25 | }, 26 | "nbformat": 4, 27 | "nbformat_minor": 2, 28 | "cells": [ 29 | { 30 | "source": [ 31 | "来源：[b站刘二大人--RNN基础篇](https://www.bilibili.com/video/BV1Y7411d7Ys?p=12)\n", 32 | "# How to use RNNCell\n", 33 | "## 注意几个参数\n", 34 | "1. 输入和隐层（输出）维度\n", 35 | "2. 序列长度\n", 36 | "3. 批处理大小\n", 37 | "\n", 38 | "- **注 调用RNNCell这个需要循环，循环长度就是序列长度**" 39 | ], 40 | "cell_type": "markdown", 41 | "metadata": {} 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 22, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "output_type": "stream", 50 | "name": "stdout", 51 | "text": [ 52 | "==================== 0 ====================\nInput size: torch.Size([1, 4]) tensor([[ 1.9129, -0.7440, 0.2329, 1.3065]])\nhidden size: torch.Size([1, 2]) tensor([[-0.0790, -0.8957]], grad_fn=)\ntensor([[-0.0790, -0.8957]], grad_fn=)\n==================== 1 ====================\nInput size: torch.Size([1, 4]) tensor([[-0.6290, -0.2338, -0.2949, 0.3956]])\nhidden size: torch.Size([1, 2]) tensor([[ 0.0170, -0.0005]], grad_fn=)\ntensor([[ 0.0170, -0.0005]], grad_fn=)\n==================== 2 ====================\nInput size: torch.Size([1, 4]) tensor([[-0.6959, 1.0590, -0.6798, 0.6989]])\nhidden size: torch.Size([1, 2]) tensor([[0.4216, 0.6813]], grad_fn=)\ntensor([[0.4216, 0.6813]], grad_fn=)\n" 53 | ] 54 | } 55 | ], 56 | "source": [ 57 | "import torch\n", 58 | "\n", 59 | "batch_size = 1 # 批处理大小\n", 60 | "seq_len = 3 # 序列长度\n", 61 | "input_size = 4 # 输入维度\n", 62 | "hidden_size = 2 # 隐层维度\n", 63 | "\n", 64 | "cell = torch.nn.RNNCell(input_size=input_size, hidden_size=hidden_size)\n", 65 | "\n", 66 | "# (seq, batch, features)\n", 67 | "dataset = torch.randn(seq_len, batch_size, input_size)\n", 68 | "hidden = torch.zeros(batch_size, hidden_size)\n", 69 | "\n", 70 | "# 这个循环就是处理seq_len长度的数据\n", 71 | "for idx, data in enumerate(dataset):\n", 72 | " print('=' * 20, idx, '=' * 20)\n", 73 | " print('Input size:', data.shape, data)\n", 74 | "\n", 75 | " hidden = cell(data, hidden)\n", 76 | "\n", 77 | " print('hidden size:', hidden.shape, hidden)\n", 78 | " print(hidden)" 79 | ] 80 | }, 81 | { 82 | "source": [ 83 | "# How to use RNN\n", 84 | "## 确定几个参数\n", 85 | "1. input_size和hidden_size: 输入维度和隐层维度\n", 86 | "2. batch_size: 批处理大小\n", 87 | "3. seq_len: 序列长度\n", 88 | "4. num_layers: 隐层数目\n", 89 | "\n", 90 | "- **注 直接调用RNN这个不用循环**\n", 91 | "- **注：如果使用batch_first: if True, the input and output tensors are provided as:(batch_size, seq_len, input_size)**" 92 | ], 93 | "cell_type": "markdown", 94 | "metadata": {} 95 | }, 96 | { 97 | "source": [ 98 | "import torch\n", 99 | "\n", 100 | "batch_size = 1\n", 101 | "seq_len = 3\n", 102 | "input_size = 4\n", 103 | "hidden_size = 2\n", 104 | "num_layers = 1\n", 105 | "\n", 106 | "cell = torch.nn.RNN(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers)\n", 107 | "\n", 108 | "# (seqLen, batchSize, inputSize)\n", 109 | "inputs = torch.randn(seq_len, batch_size, input_size)\n", 110 | "hidden = torch.zeros(num_layers, batch_size, hidden_size)\n", 111 | "\n", 112 | "out, hidden = cell(inputs, hidden)\n", 113 | "\n", 114 | "print('Output size:', out.shape) # (seq_len, batch_size, hidden_size)\n", 115 | "print('Output:', out)\n", 116 | "print('Hidden size:', hidden.shape) # (num_layers, batch_size, hidden_size)\n", 117 | "print('Hidden:', hidden)" 118 | ], 119 | "cell_type": "code", 120 | "metadata": {}, 121 | "execution_count": 23, 122 | "outputs": [ 123 | { 124 | "output_type": "stream", 125 | "name": "stdout", 126 | "text": [ 127 | "Output size: torch.Size([3, 1, 2])\nOutput: tensor([[[ 0.3689, 0.5982]],\n\n [[ 0.1233, 0.2617]],\n\n [[-0.3517, -0.7246]]], grad_fn=)\nHidden size: torch.Size([1, 1, 2])\nHidden: tensor([[[-0.3517, -0.7246]]], grad_fn=)\n" 128 | ] 129 | } 130 | ] 131 | }, 132 | { 133 | "source": [ 134 | "# Example: Using RNNCell\n", 135 | "## Hello --> ohlol\n", 136 | "1. 首先需要将输入的单词转成向量`one-hot vector`\n", 137 | "2. 注意input_size，如下图\n", 138 | "\n", 139 | "![转化成向量](https://s2.loli.net/2022/01/11/DwGu2hUaopHljtY.png)\n", 140 | "\n", 141 | "## 注意交叉熵在计算loss的时候维度关系，这里的hidden是`([1, 4])`, label是 `([1])`" 142 | ], 143 | "cell_type": "markdown", 144 | "metadata": {} 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 26, 149 | "metadata": {}, 150 | "outputs": [ 151 | { 152 | "output_type": "stream", 153 | "name": "stdout", 154 | "text": [ 155 | "torch.Size([5, 1, 4]) torch.Size([5, 1])\n" 156 | ] 157 | } 158 | ], 159 | "source": [ 160 | "import torch\n", 161 | "input_size = 4\n", 162 | "hidden_size = 4\n", 163 | "batch_size = 1\n", 164 | "\n", 165 | "idx2char = ['e', 'h', 'l', 'o']\n", 166 | "x_data = [1, 0, 2, 3, 3] # hello中各个字符的下标\n", 167 | "y_data = [3, 1, 2, 3, 2] # ohlol中各个字符的下标\n", 168 | "\n", 169 | "one_hot_lookup = [[1, 0, 0, 0],\n", 170 | " [0, 1, 0, 0],\n", 171 | " [0, 0, 1, 0],\n", 172 | " [0, 0, 0, 1]]\n", 173 | "x_one_hot = [one_hot_lookup[x] for x in x_data] # (seqLen, inputSize)\n", 174 | "\n", 175 | "inputs = torch.Tensor(x_one_hot).view(-1, batch_size, input_size)\n", 176 | "labels = torch.LongTensor(y_data).view(-1, 1) # torch.Tensor默认是torch.FloatTensor是32位浮点类型数据，torch.LongTensor是64位整型\n", 177 | "print(inputs.shape, labels.shape)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 27, 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "import torch.nn as nn\n", 187 | "\n", 188 | "class Model(nn.Module):\n", 189 | " def __init__(self, input_size, hidden_size, batch_size):\n", 190 | " super(Model, self).__init__()\n", 191 | " self.batch_size = batch_size\n", 192 | " self.input_size = input_size\n", 193 | " self.hidden_size = hidden_size\n", 194 | " self.rnncell = nn.RNNCell(input_size=self.input_size, hidden_size=self.hidden_size)\n", 195 | "\n", 196 | " def forward(self, inputs, hidden):\n", 197 | " hidden = self.rnncell(inputs, hidden) # (batch_size, hidden_size)\n", 198 | " return hidden\n", 199 | "\n", 200 | " def init_hidden(self):\n", 201 | " return torch.zeros(self.batch_size, self.hidden_size)\n", 202 | "\n", 203 | "net = Model(input_size, hidden_size, batch_size)\n", 204 | "\n", 205 | "criterion = torch.nn.CrossEntropyLoss()\n", 206 | "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 28, 212 | "metadata": {}, 213 | "outputs": [ 214 | { 215 | "output_type": "stream", 216 | "name": "stdout", 217 | "text": [ 218 | "Predicted string:lhlhh, Epoch [1/15] loss=6.8407\nPredicted string:lllll, Epoch [2/15] loss=5.2957\nPredicted string:lllol, Epoch [3/15] loss=4.9344\nPredicted string:lllol, Epoch [4/15] loss=4.7035\nPredicted string:oolol, Epoch [5/15] loss=4.4781\nPredicted string:oolol, Epoch [6/15] loss=4.2419\nPredicted string:ohlol, Epoch [7/15] loss=3.9733\nPredicted string:ohlol, Epoch [8/15] loss=3.6942\nPredicted string:ohlol, Epoch [9/15] loss=3.4917\nPredicted string:ohloo, Epoch [10/15] loss=3.3837\nPredicted string:ohloo, Epoch [11/15] loss=3.2953\nPredicted string:ohlol, Epoch [12/15] loss=3.1331\nPredicted string:ohlol, Epoch [13/15] loss=2.9294\nPredicted string:ohlol, Epoch [14/15] loss=2.7344\nPredicted string:ohlol, Epoch [15/15] loss=2.5680\n" 219 | ] 220 | } 221 | ], 222 | "source": [ 223 | "epochs = 15\n", 224 | "\n", 225 | "for epoch in range(epochs):\n", 226 | " loss = 0\n", 227 | " optimizer.zero_grad()\n", 228 | " hidden = net.init_hidden()\n", 229 | " print('Predicted string:', end='')\n", 230 | " for input, label in zip(inputs, labels):\n", 231 | " hidden = net(input, hidden)\n", 232 | " # 注意交叉熵在计算loss的时候维度关系，这里的hidden是([1, 4]), label是 ([1])\n", 233 | " loss += criterion(hidden, label)\n", 234 | " _, idx = hidden.max(dim = 1)\n", 235 | " print(idx2char[idx.item()], end='')\n", 236 | " loss.backward()\n", 237 | " optimizer.step()\n", 238 | " print(', Epoch [%d/15] loss=%.4f' % (epoch+1, loss.item()))" 239 | ] 240 | }, 241 | { 242 | "source": [ 243 | "# Example: Using RNN\n", 244 | "## 注意`inputs`和`labels`的维度\n", 245 | "- `inputs`维度是: (seqLen, batch_size, input_size)\n", 246 | "- `labels`维度是: (seqLen * batch_size)\n", 247 | "\n", 248 | "## 注意`outputs`维度，对应和`labels`做交叉熵的维度\n", 249 | "- `outputs`维度是: (seqLen, batch_size, hidden_size)\n", 250 | "- 为了能和labels做交叉熵，需要reshape一下: outputs.view(-1, hidden_size)" 251 | ], 252 | "cell_type": "markdown", 253 | "metadata": {} 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 29, 258 | "metadata": {}, 259 | "outputs": [ 260 | { 261 | "output_type": "stream", 262 | "name": "stdout", 263 | "text": [ 264 | "torch.Size([5, 1, 4]) torch.Size([5])\n" 265 | ] 266 | } 267 | ], 268 | "source": [ 269 | "import torch\n", 270 | "input_size = 4\n", 271 | "hidden_size = 4\n", 272 | "batch_size = 1\n", 273 | "seq_len = 5\n", 274 | "num_layers = 1\n", 275 | "\n", 276 | "idx2char = ['e', 'h', 'l', 'o']\n", 277 | "x_data = [1, 0, 2, 3, 3] # hello中各个字符的下标\n", 278 | "y_data = [3, 1, 2, 3, 2] # ohlol中各个字符的下标\n", 279 | "\n", 280 | "one_hot_lookup = [[1, 0, 0, 0],\n", 281 | " [0, 1, 0, 0],\n", 282 | " [0, 0, 1, 0],\n", 283 | " [0, 0, 0, 1]]\n", 284 | "x_one_hot = [one_hot_lookup[x] for x in x_data] # (seqLen, inputSize)\n", 285 | "\n", 286 | "inputs = torch.Tensor(x_one_hot).view(seq_len, batch_size, input_size)\n", 287 | "labels = torch.LongTensor(y_data) \n", 288 | "print(inputs.shape, labels.shape)" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 31, 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "import torch.nn as nn\n", 298 | "\n", 299 | "class Model(nn.Module):\n", 300 | " def __init__(self, input_size, hidden_size, batch_size, num_layers=1):\n", 301 | " super(Model, self).__init__()\n", 302 | " self.num_layers = num_layers\n", 303 | " self.batch_size = batch_size\n", 304 | " self.input_size = input_size\n", 305 | " self.hidden_size = hidden_size\n", 306 | " self.rnn = nn.RNN(input_size=self.input_size, hidden_size=self.hidden_size, )\n", 307 | "\n", 308 | " def forward(self, inputs):\n", 309 | " hidden = torch.zeros(self.num_layers, self.batch_size, self.hidden_size)\n", 310 | " out, _ = self.rnn(inputs, hidden) # 注意维度是(seqLen, batch_size, hidden_size)\n", 311 | " return out.view(-1, self.hidden_size) # 为了容易计算交叉熵这里调整维度为(seqLen * batch_size, hidden_size)\n", 312 | "\n", 313 | "net = Model(input_size, hidden_size, batch_size)\n", 314 | "\n", 315 | "criterion = torch.nn.CrossEntropyLoss()\n", 316 | "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": 32, 322 | "metadata": {}, 323 | "outputs": [ 324 | { 325 | "output_type": "stream", 326 | "name": "stdout", 327 | "text": [ 328 | "Predicted: ololl, Epoch [1/15] loss = 1.189\nPredicted: ollll, Epoch [2/15] loss = 1.070\nPredicted: ollll, Epoch [3/15] loss = 0.976\nPredicted: ohlll, Epoch [4/15] loss = 0.883\nPredicted: ohlol, Epoch [5/15] loss = 0.788\nPredicted: ohlol, Epoch [6/15] loss = 0.715\nPredicted: ohlol, Epoch [7/15] loss = 0.652\nPredicted: ohlol, Epoch [8/15] loss = 0.603\nPredicted: ohlol, Epoch [9/15] loss = 0.570\nPredicted: ohlol, Epoch [10/15] loss = 0.548\nPredicted: ohlol, Epoch [11/15] loss = 0.530\nPredicted: ohlol, Epoch [12/15] loss = 0.511\nPredicted: ohlol, Epoch [13/15] loss = 0.488\nPredicted: ohlol, Epoch [14/15] loss = 0.462\nPredicted: ohlol, Epoch [15/15] loss = 0.439\n" 329 | ] 330 | } 331 | ], 332 | "source": [ 333 | "epochs = 15\n", 334 | "\n", 335 | "for epoch in range(epochs):\n", 336 | " optimizer.zero_grad()\n", 337 | " outputs = net(inputs) \n", 338 | " # print(outputs.shape, labels.shape)\n", 339 | " # 这里的outputs维度是([seqLen * batch_size, hidden]), labels维度是([seqLen])\n", 340 | " loss = criterion(outputs, labels) \n", 341 | " loss.backward() \n", 342 | " optimizer.step()\n", 343 | "\n", 344 | " _, idx = outputs.max(dim=1) \n", 345 | " idx = idx.data.numpy() \n", 346 | " print('Predicted: ', ''.join([idx2char[x] for x in idx]), end='') \n", 347 | " print(', Epoch [%d/15] loss = %.3f' % (epoch + 1, loss.item()))" 348 | ] 349 | }, 350 | { 351 | "source": [ 352 | "# 将一个单词变成vector\n", 353 | "## One-hot encoding of words and characters\n", 354 | "- one-hot vectors high-dimension --> lower-dimension\n", 355 | "- one-hot vectors sparse --> dense\n", 356 | "- one-hot vectors hardcoded --> learn from data\n", 357 | "\n", 358 | "## Embedding\n", 359 | "![ont_hot_vector VS embedding](https://ericpengshuai.github.io/shen-du-xue-xi/ac81e297adc0/ont_hot_VS_embedding.png)\n", 360 | "![embedding](https://ericpengshuai.github.io/shen-du-xue-xi/ac81e297adc0/embedding.png)" 361 | ], 362 | "cell_type": "markdown", 363 | "metadata": {} 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": 33, 368 | "metadata": {}, 369 | "outputs": [], 370 | "source": [ 371 | "import torch.nn as nn\n", 372 | "\n", 373 | "# parameters\n", 374 | "num_class = 4 \n", 375 | "input_size = 4 \n", 376 | "hidden_size = 8 \n", 377 | "embedding_size = 10 \n", 378 | "num_layers = 2 \n", 379 | "batch_size = 1 \n", 380 | "seq_len = 5\n", 381 | "\n", 382 | "class Model(nn.Module):\n", 383 | " def __init__(self):\n", 384 | " super(Model, self).__init__()\n", 385 | " self.emb = torch.nn.Embedding(input_size, embedding_size)\n", 386 | " self.rnn = nn.RNN(input_size=embedding_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=True)\n", 387 | " self.fc = nn.Linear(hidden_size, num_class)\n", 388 | "\n", 389 | " def forward(self, x):\n", 390 | " hidden = torch.zeros(num_layers, x.size(0), hidden_size)\n", 391 | " x = self.emb(x) # (batch, seqLen, embeddingSize) \n", 392 | " x, _ = self.rnn(x, hidden) # 输出(𝒃𝒂𝒕𝒄𝒉𝑺𝒊𝒛𝒆, 𝒔𝒆𝒒𝑳𝒆𝒏, hidden_size)\n", 393 | " x = self.fc(x) # 输出(𝒃𝒂𝒕𝒄𝒉𝑺𝒊𝒛𝒆, 𝒔𝒆𝒒𝑳𝒆𝒏, 𝒏𝒖𝒎𝑪𝒍𝒂𝒔𝒔)\n", 394 | " return x.view(-1, num_class) # reshape to use Cross Entropy: (𝒃𝒂𝒕𝒄𝒉𝑺𝒊𝒛𝒆×𝒔𝒆𝒒𝑳𝒆𝒏, 𝒏𝒖𝒎𝑪𝒍𝒂𝒔𝒔)\n", 395 | " \n", 396 | "net = Model()\n", 397 | "\n", 398 | "criterion = torch.nn.CrossEntropyLoss()\n", 399 | "optimizer = torch.optim.Adam(net.parameters(), lr=0.05)" 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": 34, 405 | "metadata": {}, 406 | "outputs": [ 407 | { 408 | "output_type": "stream", 409 | "name": "stdout", 410 | "text": [ 411 | "Predicted: ollll, Epoch [1/15] loss = 1.290\nPredicted: olooo, Epoch [2/15] loss = 1.071\nPredicted: ollol, Epoch [3/15] loss = 0.913\nPredicted: ollol, Epoch [4/15] loss = 0.785\nPredicted: ollol, Epoch [5/15] loss = 0.660\nPredicted: ohlol, Epoch [6/15] loss = 0.541\nPredicted: ohlol, Epoch [7/15] loss = 0.435\nPredicted: ohlol, Epoch [8/15] loss = 0.343\nPredicted: ohlol, Epoch [9/15] loss = 0.251\nPredicted: ohlol, Epoch [10/15] loss = 0.171\nPredicted: ohlol, Epoch [11/15] loss = 0.121\nPredicted: ohlol, Epoch [12/15] loss = 0.081\nPredicted: ohlol, Epoch [13/15] loss = 0.052\nPredicted: ohlol, Epoch [14/15] loss = 0.036\nPredicted: ohlol, Epoch [15/15] loss = 0.025\n" 412 | ] 413 | } 414 | ], 415 | "source": [ 416 | "idx2char = ['e', 'h', 'l', 'o'] \n", 417 | "x_data = [[1, 0, 2, 2, 3]] # (batch, seq_len) \n", 418 | "y_data = [3, 1, 2, 3, 2] # (batch * seq_len)\n", 419 | "\n", 420 | "inputs = torch.LongTensor(x_data) # Input should be LongTensor: (batchSize, seqLen)\n", 421 | "labels = torch.LongTensor(y_data) # Target should be LongTensor: (batchSize * seqLen)\n", 422 | "\n", 423 | "epochs = 15\n", 424 | "\n", 425 | "for epoch in range(epochs):\n", 426 | " optimizer.zero_grad()\n", 427 | " outputs = net(inputs) \n", 428 | " loss = criterion(outputs, labels) \n", 429 | " loss.backward() \n", 430 | " optimizer.step()\n", 431 | "\n", 432 | " _, idx = outputs.max(dim=1) \n", 433 | " idx = idx.data.numpy() \n", 434 | " print('Predicted: ', ''.join([idx2char[x] for x in idx]), end='') \n", 435 | " print(', Epoch [%d/15] loss = %.3f' % (epoch + 1, loss.item()))" 436 | ] 437 | } 438 | ] 439 | } -------------------------------------------------------------------------------- /Transformer.md: -------------------------------------------------------------------------------- 1 | # Transformer 2 | 3 | transformer框架图 4 | 5 | ## Encoder 6 | 7 | 上图左半部分，6层结构，每层结构包含两个子层 8 | 9 | ### 1. 输入：字向量与位置编码 10 | 11 | $$ 12 | X_{embedding} = X_{word-embedding} + X_{pos-embedding} 13 | $$ 14 | 15 | 16 | 17 | ### 2. 多头注意力层 18 | 19 | multi-head attention：**(Q、K、V)** 20 | $$ 21 | Q = \text{Linear}_q(X) = XW_{Q}\\ 22 | K = \text{Linear}_k(X) = XW_{K}\\ 23 | V = \text{Linear}_v(X) = XW_{V}\\ 24 | X_{attention} = \text{Self-Attention}(Q,K,V) 25 | $$ 26 | 注意：Ecoder的Q是embedding来的，是已知的，而Decoder输出的Q是预测的，也就是结果预测的词 27 | 28 | ### 3. self_attention残差连接和Layer Normalization 29 | 30 | 注意这里的Norm是Layer Norm，而不是BN（效果不太好） 31 | $$ 32 | X_{attention} = X + X_{attention}\\ 33 | X_{attention} = \text{LayerNorm}(X_{attention}) 34 | $$ 35 | 36 | 37 | ### 4. 前馈连接层 38 | 39 | feed forward：就是全连接。其实就是两层线性映射并用激活函数激活，比如说 ReLU 40 | $$ 41 | X_{hidden} = \text{Linear}(\text{ReLU}(\text{Linear}(X_{attention}))) 42 | $$ 43 | 44 | ### 5. feed_forward残差连接和Layer Normalization 45 | 46 | $$ 47 | X_{hidden} = X_{attention} + X_{hidden}\\ 48 | X_{hidden} = \text{LayerNorm}(X_{hidden}) 49 | $$ 50 | 51 | 其中 $X_{hidden} \in \mathbb{R}^{batch\_size \ * \ seq\_len. \ * \ embed\_dim}$ 52 | 53 | Encoder结构 54 | 55 | ## Decoder 56 | 57 | 上图右半部分，6层结构，每层结构包含三个子层 58 | 59 | 60 | 61 | ### **遮掩多头注意力层** 62 | 63 | - mask防止训练过程使用未来输出的单词，**保证预测位置i的信息只能基于比i小的输出** 64 | - encoder层可以并行计算，decoder层像RNN一样一个一个训练，需要使用上一个位置的输入当做attention的query 65 | 66 | 67 | 68 | 69 | 70 | ### **多头注意力结构** 71 | 72 | - 输入来源1：第一个子层的输出 73 | - 输入来源2：Encoder层的输出 74 | 75 | 这是和encoder层区别的地方以及这里的信息交换是权值共享 76 | 77 | 78 | 79 | ### **前馈连接层** 80 | 81 | **残差连接**：Output Embedding --> Add & Norm， Add & Norm --> Add & Norm， Add & Norm --> Add & Norm 82 | 83 | - 残差结构可以解决梯度消失问题，增加模型的复杂性 84 | 85 | - Norm层为了对attention层的输出进行分布归一化（对一层进行一次归一化），区别于cv中的batchNorm（对一个batchSize中的样本进行归一化） 86 | 87 | 88 | 89 | 90 | ## 几个术语 91 | 92 | - **self-attention**：是transformer用来找到重点关注与当前单词相关词语的一种方法 93 | 94 | - **Multi-Headed Attention**：多头注意力机制是指有多组Q,K,V矩阵，一组Q,K,V矩阵代表一次注意力机制的运算，transformer使用了8组，所以最终得到了8个矩阵，将这8个矩阵拼接起来后再乘以一个参数矩阵WO,即可得出最终的多注意力层的输出。 95 | 96 | - **Positional Encoding**：为了解释输入序列中单词顺序而存在，维度和embedding的维度一致。这个向量决定了当前词的位置，或者说是在一个句子中不同的词之间的距离。 97 | 98 | - **layer normalization**：在transformer中，每一个子层（自注意力层，全连接层）后都会有一个Layer normalization层 99 | 100 | 101 | 102 | ## 参考 103 | 104 | 1. [b站视频讲解](https://urlify.cn/yiuMNv)-[Transformer](https://wmathor.com/index.php/archives/1438/)的Pytorch实现 105 | 2. [Pytorch代码实现](https://wmathor.com/index.php/archives/1455/) 106 | 3. [b站-Transformer从零详细解读](https://urlify.cn/qiqAJf) 107 | 4. [知乎-Transformer面试题系列](https://zhuanlan.zhihu.com/p/148656446) -------------------------------------------------------------------------------- /dataset/airplane_data.csv: -------------------------------------------------------------------------------- 1 | "Month","International airline passengers: monthly totals in thousands. Jan 49 ? Dec 60" 2 | "1949-01",112 3 | "1949-02",118 4 | "1949-03",132 5 | "1949-04",129 6 | "1949-05",121 7 | "1949-06",135 8 | "1949-07",148 9 | "1949-08",148 10 | "1949-09",136 11 | "1949-10",119 12 | "1949-11",104 13 | "1949-12",118 14 | "1950-01",115 15 | "1950-02",126 16 | "1950-03",141 17 | "1950-04",135 18 | "1950-05",125 19 | "1950-06",149 20 | "1950-07",170 21 | "1950-08",170 22 | "1950-09",158 23 | "1950-10",133 24 | "1950-11",114 25 | "1950-12",140 26 | "1951-01",145 27 | "1951-02",150 28 | "1951-03",178 29 | "1951-04",163 30 | "1951-05",172 31 | "1951-06",178 32 | "1951-07",199 33 | "1951-08",199 34 | "1951-09",184 35 | "1951-10",162 36 | "1951-11",146 37 | "1951-12",166 38 | "1952-01",171 39 | "1952-02",180 40 | "1952-03",193 41 | "1952-04",181 42 | "1952-05",183 43 | "1952-06",218 44 | "1952-07",230 45 | "1952-08",242 46 | "1952-09",209 47 | "1952-10",191 48 | "1952-11",172 49 | "1952-12",194 50 | "1953-01",196 51 | "1953-02",196 52 | "1953-03",236 53 | "1953-04",235 54 | "1953-05",229 55 | "1953-06",243 56 | "1953-07",264 57 | "1953-08",272 58 | "1953-09",237 59 | "1953-10",211 60 | "1953-11",180 61 | "1953-12",201 62 | "1954-01",204 63 | "1954-02",188 64 | "1954-03",235 65 | "1954-04",227 66 | "1954-05",234 67 | "1954-06",264 68 | "1954-07",302 69 | "1954-08",293 70 | "1954-09",259 71 | "1954-10",229 72 | "1954-11",203 73 | "1954-12",229 74 | "1955-01",242 75 | "1955-02",233 76 | "1955-03",267 77 | "1955-04",269 78 | "1955-05",270 79 | "1955-06",315 80 | "1955-07",364 81 | "1955-08",347 82 | "1955-09",312 83 | "1955-10",274 84 | "1955-11",237 85 | "1955-12",278 86 | "1956-01",284 87 | "1956-02",277 88 | "1956-03",317 89 | "1956-04",313 90 | "1956-05",318 91 | "1956-06",374 92 | "1956-07",413 93 | "1956-08",405 94 | "1956-09",355 95 | "1956-10",306 96 | "1956-11",271 97 | "1956-12",306 98 | "1957-01",315 99 | "1957-02",301 100 | "1957-03",356 101 | "1957-04",348 102 | "1957-05",355 103 | "1957-06",422 104 | "1957-07",465 105 | "1957-08",467 106 | "1957-09",404 107 | "1957-10",347 108 | "1957-11",305 109 | "1957-12",336 110 | "1958-01",340 111 | "1958-02",318 112 | "1958-03",362 113 | "1958-04",348 114 | "1958-05",363 115 | "1958-06",435 116 | "1958-07",491 117 | "1958-08",505 118 | "1958-09",404 119 | "1958-10",359 120 | "1958-11",310 121 | "1958-12",337 122 | "1959-01",360 123 | "1959-02",342 124 | "1959-03",406 125 | "1959-04",396 126 | "1959-05",420 127 | "1959-06",472 128 | "1959-07",548 129 | "1959-08",559 130 | "1959-09",463 131 | "1959-10",407 132 | "1959-11",362 133 | "1959-12",405 134 | "1960-01",417 135 | "1960-02",391 136 | "1960-03",419 137 | "1960-04",461 138 | "1960-05",472 139 | "1960-06",535 140 | "1960-07",622 141 | "1960-08",606 142 | "1960-09",508 143 | "1960-10",461 144 | "1960-11",390 145 | "1960-12",432 146 | 147 | International airline passengers: monthly totals in thousands. Jan 49 ? Dec 60 148 | 149 | -------------------------------------------------------------------------------- /dataset/diabetes.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/diabetes.csv.gz -------------------------------------------------------------------------------- /dataset/mnist/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/mnist/.DS_Store -------------------------------------------------------------------------------- /dataset/mnist/processed/test.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/mnist/processed/test.pt -------------------------------------------------------------------------------- /dataset/mnist/processed/training.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/mnist/processed/training.pt -------------------------------------------------------------------------------- /dataset/mnist/raw/t10k-images-idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/mnist/raw/t10k-images-idx3-ubyte -------------------------------------------------------------------------------- /dataset/mnist/raw/t10k-labels-idx1-ubyte: -------------------------------------------------------------------------------- 1 | '                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             -------------------------------------------------------------------------------- /dataset/mnist/raw/train-images-idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/mnist/raw/train-images-idx3-ubyte -------------------------------------------------------------------------------- /dataset/mnist/raw/train-labels-idx1-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/mnist/raw/train-labels-idx1-ubyte -------------------------------------------------------------------------------- /dataset/names_test.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/names_test.csv.gz -------------------------------------------------------------------------------- /dataset/names_train.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricPengShuai/Pytorch-Learning/528caf054bd2c779f6953b89313d2794e48eccfe/dataset/names_train.csv.gz -------------------------------------------------------------------------------- /softmax_classifier.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.6.12-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3.6.12 64-bit ('pytorch_dl': conda)", 19 | "metadata": { 20 | "interpreter": { 21 | "hash": "6b962ed846ab7a2f9c4286c5c9f447c6ba721d3b496bd62adf19906831909870" 22 | } 23 | } 24 | } 25 | }, 26 | "nbformat": 4, 27 | "nbformat_minor": 2, 28 | "cells": [ 29 | { 30 | "source": [ 31 | "来源： [b站刘二大人--多分类问题](https://www.bilibili.com/video/BV1Y7411d7Ys?p=9)\n", 32 | "# 0. Test CrossEntropyLoss" 33 | ], 34 | "cell_type": "markdown", 35 | "metadata": {} 36 | }, 37 | { 38 | "source": [ 39 | "import torch\n", 40 | "criterion = torch.nn.CrossEntropyLoss()\n", 41 | "Y = torch.LongTensor([2, 0, 1])\n", 42 | "\n", 43 | "Y_pred1 = torch.Tensor([\n", 44 | " [0.1, 0.2, 0.9],\n", 45 | " [1.1, 0.1, 0.2],\n", 46 | " [0.2, 2.1, 0.1]\n", 47 | "])\n", 48 | "Y_pred2 = torch.Tensor([\n", 49 | " [0.8, 0.2, 0.3],\n", 50 | " [0.2, 0.3, 0.5],\n", 51 | " [0.2, 0.2, 0.5]\n", 52 | "])\n", 53 | "\n", 54 | "l1 = criterion(Y_pred1, Y)\n", 55 | "l2 = criterion(Y_pred2, Y)\n", 56 | "print(\"Batch Loss1 = \", l1.data)\n", 57 | "print(\"Batch Loss2 = \", l2.data)" 58 | ], 59 | "cell_type": "code", 60 | "metadata": {}, 61 | "execution_count": 1, 62 | "outputs": [ 63 | { 64 | "output_type": "stream", 65 | "name": "stdout", 66 | "text": [ 67 | "Batch Loss1 = tensor(0.4966)\nBatch Loss2 = tensor(1.2389)\n" 68 | ] 69 | } 70 | ] 71 | }, 72 | { 73 | "source": [ 74 | "# 1. Implementation for MINIST\n", 75 | "## 1.1 Prepare Dataset" 76 | ], 77 | "cell_type": "markdown", 78 | "metadata": {} 79 | }, 80 | { 81 | "source": [ 82 | "import torch\n", 83 | "from torchvision import transforms\n", 84 | "from torchvision import datasets\n", 85 | "from torch.utils.data import DataLoader\n", 86 | "import torch.nn.functional as F\n", 87 | "import torch.optim as optim\n", 88 | "\n", 89 | "batch_size = 64\n", 90 | "transform = transforms.Compose([\n", 91 | " transforms.ToTensor(), \n", 92 | " transforms.Normalize((0.1307,), (0.3081,))\n", 93 | "]) # 归一化,均值和方差\n", 94 | " \n", 95 | "train_dataset = datasets.MNIST(root='./dataset/mnist/', train=True, download=True, transform=transform)\n", 96 | "train_loader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size)\n", 97 | "test_dataset = datasets.MNIST(root='./dataset/mnist/', train=False, download=True, transform=transform)\n", 98 | "test_loader = DataLoader(test_dataset, shuffle=False, batch_size=batch_size)" 99 | ], 100 | "cell_type": "code", 101 | "metadata": {}, 102 | "execution_count": 2, 103 | "outputs": [] 104 | }, 105 | { 106 | "source": [ 107 | "## 1.2 Design Model" 108 | ], 109 | "cell_type": "markdown", 110 | "metadata": {} 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 3, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "class Net(torch.nn.Module):\n", 119 | " def __init__(self):\n", 120 | " super(Net, self).__init__()\n", 121 | " self.l1 = torch.nn.Linear(784, 512)\n", 122 | " self.l2 = torch.nn.Linear(512, 256)\n", 123 | " self.l3 = torch.nn.Linear(256, 128)\n", 124 | " self.l4 = torch.nn.Linear(128, 64)\n", 125 | " self.l5 = torch.nn.Linear(64, 10)\n", 126 | "\n", 127 | " def forward(self, x):\n", 128 | " x = x.view(-1, 784)\n", 129 | " x = F.relu(self.l1(x))\n", 130 | " x = F.relu(self.l2(x))\n", 131 | " x = F.relu(self.l3(x))\n", 132 | " x = F.relu(self.l4(x))\n", 133 | " # 最后一层不需要激活，注意CrossEntropyLoss包含了logSoftmax+NLLloss\n", 134 | " return self.l5(x)\n", 135 | "\n", 136 | "model = Net()\n" 137 | ] 138 | }, 139 | { 140 | "source": [ 141 | "## 1.3 Construct Loss ans Optimizer" 142 | ], 143 | "cell_type": "markdown", 144 | "metadata": {} 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 5, 149 | "metadata": {}, 150 | "outputs": [ 151 | { 152 | "output_type": "stream", 153 | "name": "stdout", 154 | "text": [ 155 | "\n\n\nlayer1_weight.shape torch.Size([512, 784])\nlayer1_bias.shape torch.Size([512])\nlayer2_weight.shape torch.Size([256, 512])\nlayer2_bias.shape torch.Size([256])\n" 156 | ] 157 | } 158 | ], 159 | "source": [ 160 | "# 查看模型参数shape\n", 161 | "\n", 162 | "print(train_loader)\n", 163 | "print(test_loader)\n", 164 | "print(model.parameters())\n", 165 | "# 参数说明\n", 166 | "# 第一层的参数：\n", 167 | "layer1_weight = model.l1.weight.data\n", 168 | "layer1_bias = model.l1.bias.data\n", 169 | "# print(\"layer1_weight\", layer1_weight)\n", 170 | "print(\"layer1_weight.shape\", layer1_weight.shape)\n", 171 | "# print(\"layer1_bias\", layer1_bias)\n", 172 | "print(\"layer1_bias.shape\", layer1_bias.shape)\n", 173 | "\n", 174 | "print(\"layer2_weight.shape\", model.l2.weight.data.shape)\n", 175 | "print(\"layer2_bias.shape\", model.l2.bias.data.shape)\n", 176 | "\n", 177 | "criterion = torch.nn.CrossEntropyLoss()\n", 178 | "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)" 179 | ] 180 | }, 181 | { 182 | "source": [ 183 | "## 1.4 Train and Test" 184 | ], 185 | "cell_type": "markdown", 186 | "metadata": {} 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 6, 191 | "metadata": { 192 | "tags": [] 193 | }, 194 | "outputs": [ 195 | { 196 | "output_type": "stream", 197 | "name": "stdout", 198 | "text": [ 199 | "[1, 300] loss: 2.260\n", 200 | "[1, 600] loss: 1.088\n", 201 | "[1, 900] loss: 0.429\n", 202 | "accuracy on test set: 89 % \n", 203 | "[2, 300] loss: 0.319\n", 204 | "[2, 600] loss: 0.264\n", 205 | "[2, 900] loss: 0.228\n", 206 | "accuracy on test set: 94 % \n", 207 | "[3, 300] loss: 0.187\n", 208 | "[3, 600] loss: 0.165\n", 209 | "[3, 900] loss: 0.157\n", 210 | "accuracy on test set: 95 % \n", 211 | "[4, 300] loss: 0.130\n", 212 | "[4, 600] loss: 0.126\n", 213 | "[4, 900] loss: 0.115\n", 214 | "accuracy on test set: 96 % \n", 215 | "[5, 300] loss: 0.096\n", 216 | "[5, 600] loss: 0.095\n", 217 | "[5, 900] loss: 0.099\n", 218 | "accuracy on test set: 96 % \n" 219 | ] 220 | } 221 | ], 222 | "source": [ 223 | "def train(epoch):\n", 224 | " running_loss = 0.0\n", 225 | " for batch_idx, data in enumerate(train_loader, 0):\n", 226 | " inputs, target = data\n", 227 | " optimizer.zero_grad()\n", 228 | "\n", 229 | " # forward + backward + update\n", 230 | " outputs = model(inputs)\n", 231 | " loss = criterion(outputs, target)\n", 232 | " loss.backward()\n", 233 | " optimizer.step()\n", 234 | "\n", 235 | " running_loss += loss.item()\n", 236 | " if batch_idx % 300 == 299:\n", 237 | " print('[%d, %5d] loss: %.3f' % (epoch+1, batch_idx+1, running_loss/300))\n", 238 | " running_loss = 0.0\n", 239 | " \n", 240 | " \n", 241 | "def test():\n", 242 | " correct = 0\n", 243 | " total = 0\n", 244 | " with torch.no_grad():\n", 245 | " for data in test_loader:\n", 246 | " images, labels = data\n", 247 | " outputs = model(images)\n", 248 | " _, predicted = torch.max(outputs.data, dim=1) # dim = 1 列是第0个维度，行是第1个维度\n", 249 | " total += labels.size(0)\n", 250 | " correct += (predicted == labels).sum().item() # 张量之间的比较运算\n", 251 | " print('accuracy on test set: %d %% ' % (100*correct/total))\n", 252 | "\n", 253 | "for epoch in range(5):\n", 254 | " train(epoch)\n", 255 | " test()\n" 256 | ] 257 | }, 258 | { 259 | "source": [ 260 | "## 1.5 理解维度" 261 | ], 262 | "cell_type": "markdown", 263 | "metadata": {} 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 7, 268 | "metadata": {}, 269 | "outputs": [ 270 | { 271 | "output_type": "stream", 272 | "name": "stdout", 273 | "text": [ 274 | "target.shape torch.Size([64]) tensor([6, 2, 0, 6, 5, 8, 0, 6, 1, 4, 1, 7, 7, 0, 7, 3, 0, 1, 0, 7, 4, 2, 2, 5,\n 7, 2, 6, 3, 2, 9, 7, 2, 3, 4, 1, 3, 0, 8, 3, 7, 6, 7, 5, 1, 8, 2, 3, 8,\n 4, 2, 5, 4, 5, 0, 8, 0, 7, 3, 2, 6, 2, 7, 7, 5])\noutput.shape torch.Size([64, 10]) tensor([ 0.0467, 0.1483, 0.0750, -0.0185, -0.0073, 0.0327, 0.0647, 0.0828,\n 0.0023, -0.0072], grad_fn=)\n" 275 | ] 276 | } 277 | ], 278 | "source": [ 279 | "\n", 280 | "for batch_idx, data in enumerate(train_loader, 0):\n", 281 | " inputs, target = data\n", 282 | " optimizer.zero_grad()\n", 283 | " print(\"target.shape\", target.shape, target)\n", 284 | " # forward + backward + update\n", 285 | " outputs = model(inputs)\n", 286 | " print(\"output.shape\", outputs.shape, outputs[0])\n", 287 | " break\n", 288 | " loss = criterion(outputs, target)\n", 289 | " loss.backward()\n", 290 | " optimizer.step()\n", 291 | "\n", 292 | " running_loss += loss.item()\n", 293 | " if batch_idx % 300 == 299:\n", 294 | " print('[%d, %5d] loss: %.3f' % (epoch+1, batch_idx+1, running_loss/300))\n", 295 | " running_loss = 0.0" 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": null, 301 | "metadata": {}, 302 | "outputs": [], 303 | "source": [] 304 | } 305 | ] 306 | } --------------------------------------------------------------------------------