├── README.md ├── deeplearning ├── resnet.ipynb └── resnet.py └── nlp ├── Bert_Blend_CNN.ipynb ├── at.py ├── bert_blend_cnn.py ├── bert_classify.ipynb ├── bert_classify.py └── focal_loss.py /README.md: -------------------------------------------------------------------------------- 1 | # 存放博客代码，希望能坚持下去呀hh 2 | 3 | - [博客园](https://www.cnblogs.com/qingyao/) 4 | - [知乎](https://www.zhihu.com/people/sheng-jian-93-86) 5 | 6 | -------------------------------------------------------------------------------- /deeplearning/resnet.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "resnet.ipynb", 7 | "provenance": [] 8 | }, 9 | "kernelspec": { 10 | "name": "python3", 11 | "display_name": "Python 3" 12 | }, 13 | "language_info": { 14 | "name": "python" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "code", 21 | "metadata": { 22 | "id": "sSw_-JAWi_nk" 23 | }, 24 | "source": [ 25 | "# -*- coding:utf-8 -*-\n", 26 | "# handwritten digits recognition\n", 27 | "# Data: MINIST\n", 28 | "# model: resnet\n", 29 | "# date: 2021.10.8 14:18\n", 30 | "\n", 31 | "import math\n", 32 | "import torch\n", 33 | "import torchvision\n", 34 | "import torchvision.transforms as transforms\n", 35 | "import torch.nn as nn\n", 36 | "import torch.utils.data as Data\n", 37 | "import torch.optim as optim\n", 38 | "import pandas as pd\n", 39 | "import matplotlib.pyplot as plt" 40 | ], 41 | "execution_count": 1, 42 | "outputs": [] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "metadata": { 47 | "id": "wAMLAt-7jlnL" 48 | }, 49 | "source": [ 50 | "train_curve = []\n", 51 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" 52 | ], 53 | "execution_count": 2, 54 | "outputs": [] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "metadata": { 59 | "id": "MjTOGiQaqw4A" 60 | }, 61 | "source": [ 62 | "# param\n", 63 | "batch_size = 100\n", 64 | "n_class = 10\n", 65 | "padding_size = 15\n", 66 | "epoches = 10" 67 | ], 68 | "execution_count": 20, 69 | "outputs": [] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "metadata": { 74 | "id": "Pz0nsjE-jwVt" 75 | }, 76 | "source": [ 77 | "train_dataset = torchvision.datasets.MNIST('./data/', train=True, transform=transforms.ToTensor(), download=True)\n", 78 | "test_dataset = torchvision.datasets.MNIST('./data/', train=False, transform=transforms.ToTensor(), download=False)\n", 79 | "train = Data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=5)\n", 80 | "test = Data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=5)" 81 | ], 82 | "execution_count": null, 83 | "outputs": [] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "metadata": { 88 | "id": "5t9GsIw5CCBx" 89 | }, 90 | "source": [ 91 | "def gelu(x):\n", 92 | " \"Implementation of the gelu activation function by Hugging Face\"\n", 93 | " return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))" 94 | ], 95 | "execution_count": 5, 96 | "outputs": [] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "metadata": { 101 | "id": "VHOBzUgztqQE" 102 | }, 103 | "source": [ 104 | "class ResBlock(nn.Module):\n", 105 | " def __init__(self, in_size, out_size1, out_size2):\n", 106 | " super(ResBlock, self).__init__()\n", 107 | " self.conv1 = nn.Conv2d(\n", 108 | " in_channels = in_size,\n", 109 | " out_channels = out_size1,\n", 110 | " kernel_size = 3,\n", 111 | " stride = 2,\n", 112 | " padding = padding_size\n", 113 | " )\n", 114 | " self.conv2 = nn.Conv2d(\n", 115 | " in_channels = out_size1,\n", 116 | " out_channels = out_size2,\n", 117 | " kernel_size = 3,\n", 118 | " stride = 2,\n", 119 | " padding = padding_size\n", 120 | " )\n", 121 | " self.batchnorm1 = nn.BatchNorm2d(out_size1)\n", 122 | " self.batchnorm2 = nn.BatchNorm2d(out_size2)\n", 123 | " \n", 124 | " def conv(self, x):\n", 125 | " x = gelu(self.batchnorm1(self.conv1(x)))\n", 126 | " x = gelu(self.batchnorm2(self.conv2(x)))\n", 127 | " return x\n", 128 | " \n", 129 | " def forward(self, x):\n", 130 | " return x + self.conv(x)" 131 | ], 132 | "execution_count": 6, 133 | "outputs": [] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "metadata": { 138 | "id": "rWj7lSNRqj7a" 139 | }, 140 | "source": [ 141 | "# resnet\n", 142 | "class Resnet(nn.Module):\n", 143 | " def __init__(self, n_class = n_class):\n", 144 | " super(Resnet, self).__init__()\n", 145 | " self.res1 = ResBlock(1, 8, 16)\n", 146 | " self.res2 = ResBlock(16, 32, 16)\n", 147 | " self.conv = nn.Conv2d(\n", 148 | " in_channels = 16,\n", 149 | " out_channels = n_class,\n", 150 | " kernel_size = 3,\n", 151 | " stride = 2,\n", 152 | " padding = padding_size\n", 153 | " )\n", 154 | " self.batchnorm = nn.BatchNorm2d(n_class)\n", 155 | " self.max_pooling = nn.AdaptiveAvgPool2d(1)\n", 156 | "\n", 157 | " def forward(self, x):\n", 158 | " x = x.view(-1, 1, 28, 28)\n", 159 | " x = self.res1(x)\n", 160 | " x = self.res2(x)\n", 161 | " x = self.max_pooling(self.batchnorm(self.conv(x)))\n", 162 | "\n", 163 | " return x.view(x.size(0), -1)\n" 164 | ], 165 | "execution_count": 7, 166 | "outputs": [] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "metadata": { 171 | "colab": { 172 | "base_uri": "https://localhost:8080/" 173 | }, 174 | "id": "jk63HsdJ5Pv_", 175 | "outputId": "d1a36e02-2bc5-46c1-ab8d-cf04ef5b3e3f" 176 | }, 177 | "source": [ 178 | "resnet = Resnet().to(device)\n", 179 | "resnet" 180 | ], 181 | "execution_count": 11, 182 | "outputs": [ 183 | { 184 | "output_type": "execute_result", 185 | "data": { 186 | "text/plain": [ 187 | "Resnet(\n", 188 | " (res1): ResBlock(\n", 189 | " (conv1): Conv2d(1, 8, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n", 190 | " (conv2): Conv2d(8, 16, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n", 191 | " (batchnorm1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 192 | " (batchnorm2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 193 | " )\n", 194 | " (res2): ResBlock(\n", 195 | " (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n", 196 | " (conv2): Conv2d(32, 16, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n", 197 | " (batchnorm1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 198 | " (batchnorm2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 199 | " )\n", 200 | " (conv): Conv2d(16, 10, kernel_size=(3, 3), stride=(2, 2), padding=(15, 15))\n", 201 | " (batchnorm): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", 202 | " (max_pooling): AdaptiveAvgPool2d(output_size=1)\n", 203 | ")" 204 | ] 205 | }, 206 | "metadata": {}, 207 | "execution_count": 11 208 | } 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "metadata": { 214 | "id": "br_L1zb65YZr" 215 | }, 216 | "source": [ 217 | "loss_fn = nn.CrossEntropyLoss()\n", 218 | "optimizer = optim.SGD(params=resnet.parameters(), lr=1e-2, momentum=0.9)" 219 | ], 220 | "execution_count": 16, 221 | "outputs": [] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "metadata": { 226 | "colab": { 227 | "base_uri": "https://localhost:8080/" 228 | }, 229 | "id": "2_DGwLWC6x4p", 230 | "outputId": "d0a9cc2d-a12a-4e15-972b-2a7bd1bef09d" 231 | }, 232 | "source": [ 233 | "# train\n", 234 | "total_step = len(train)\n", 235 | "sum_loss = 0\n", 236 | "for epoch in range(epoches):\n", 237 | " for i, (images, targets) in enumerate(train):\n", 238 | " optimizer.zero_grad()\n", 239 | " images = images.to(device)\n", 240 | " targets = targets.to(device)\n", 241 | " preds = resnet(images)\n", 242 | " \n", 243 | " loss = loss_fn(preds, targets)\n", 244 | " sum_loss += loss.item()\n", 245 | " loss.backward()\n", 246 | " optimizer.step()\n", 247 | " if (i+1)%100==0:\n", 248 | " print('[{}|{}] step:{}/{} loss:{:.4f}'.format(epoch+1, epoches, i+1, total_step, loss.item()))\n", 249 | " train_curve.append(sum_loss)\n", 250 | " sum_loss = 0\n", 251 | " " 252 | ], 253 | "execution_count": 22, 254 | "outputs": [ 255 | { 256 | "output_type": "stream", 257 | "name": "stderr", 258 | "text": [ 259 | "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 5 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", 260 | " cpuset_checked))\n" 261 | ] 262 | }, 263 | { 264 | "output_type": "stream", 265 | "name": "stdout", 266 | "text": [ 267 | "[1|10] step:100/600 loss:0.0104\n", 268 | "[1|10] step:200/600 loss:0.0106\n", 269 | "[1|10] step:300/600 loss:0.0732\n", 270 | "[1|10] step:400/600 loss:0.0317\n", 271 | "[1|10] step:500/600 loss:0.1365\n", 272 | "[1|10] step:600/600 loss:0.0139\n", 273 | "[2|10] step:100/600 loss:0.1351\n", 274 | "[2|10] step:200/600 loss:0.0275\n", 275 | "[2|10] step:300/600 loss:0.0894\n", 276 | "[2|10] step:400/600 loss:0.0616\n", 277 | "[2|10] step:500/600 loss:0.0297\n", 278 | "[2|10] step:600/600 loss:0.0832\n", 279 | "[3|10] step:100/600 loss:0.0885\n", 280 | "[3|10] step:200/600 loss:0.0124\n", 281 | "[3|10] step:300/600 loss:0.0781\n", 282 | "[3|10] step:400/600 loss:0.0477\n", 283 | "[3|10] step:500/600 loss:0.0048\n", 284 | "[3|10] step:600/600 loss:0.0412\n", 285 | "[4|10] step:100/600 loss:0.0146\n", 286 | "[4|10] step:200/600 loss:0.0193\n", 287 | "[4|10] step:300/600 loss:0.0526\n", 288 | "[4|10] step:400/600 loss:0.0025\n", 289 | "[4|10] step:500/600 loss:0.0876\n", 290 | "[4|10] step:600/600 loss:0.0551\n", 291 | "[5|10] step:100/600 loss:0.0240\n", 292 | "[5|10] step:200/600 loss:0.0036\n", 293 | "[5|10] step:300/600 loss:0.0077\n", 294 | "[5|10] step:400/600 loss:0.0169\n", 295 | "[5|10] step:500/600 loss:0.0079\n", 296 | "[5|10] step:600/600 loss:0.0342\n", 297 | "[6|10] step:100/600 loss:0.0029\n", 298 | "[6|10] step:200/600 loss:0.0772\n", 299 | "[6|10] step:300/600 loss:0.0368\n", 300 | "[6|10] step:400/600 loss:0.0408\n", 301 | "[6|10] step:500/600 loss:0.0082\n", 302 | "[6|10] step:600/600 loss:0.0354\n", 303 | "[7|10] step:100/600 loss:0.0042\n", 304 | "[7|10] step:200/600 loss:0.0313\n", 305 | "[7|10] step:300/600 loss:0.0376\n", 306 | "[7|10] step:400/600 loss:0.0500\n", 307 | "[7|10] step:500/600 loss:0.0020\n", 308 | "[7|10] step:600/600 loss:0.0330\n", 309 | "[8|10] step:100/600 loss:0.0175\n", 310 | "[8|10] step:200/600 loss:0.0473\n", 311 | "[8|10] step:300/600 loss:0.0029\n", 312 | "[8|10] step:400/600 loss:0.0147\n", 313 | "[8|10] step:500/600 loss:0.0300\n", 314 | "[8|10] step:600/600 loss:0.0006\n", 315 | "[9|10] step:100/600 loss:0.0401\n", 316 | "[9|10] step:200/600 loss:0.0286\n", 317 | "[9|10] step:300/600 loss:0.0258\n", 318 | "[9|10] step:400/600 loss:0.0167\n", 319 | "[9|10] step:500/600 loss:0.0201\n", 320 | "[9|10] step:600/600 loss:0.0574\n", 321 | "[10|10] step:100/600 loss:0.0066\n", 322 | "[10|10] step:200/600 loss:0.0097\n", 323 | "[10|10] step:300/600 loss:0.0616\n", 324 | "[10|10] step:400/600 loss:0.0515\n", 325 | "[10|10] step:500/600 loss:0.0019\n", 326 | "[10|10] step:600/600 loss:0.0381\n" 327 | ] 328 | } 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "metadata": { 334 | "colab": { 335 | "base_uri": "https://localhost:8080/" 336 | }, 337 | "id": "qbngK_gf9RBl", 338 | "outputId": "cdbcda1b-3562-48fd-b52e-a77e141e4d22" 339 | }, 340 | "source": [ 341 | "# test\n", 342 | "resnet.eval()\n", 343 | "with torch.no_grad():\n", 344 | " correct = 0\n", 345 | " total = 0\n", 346 | " for images, labels in test:\n", 347 | " images = images.to(device)\n", 348 | " labels = labels.to(device)\n", 349 | " outputs = resnet(images)\n", 350 | " _, maxIndexes = torch.max(outputs, dim=1)\n", 351 | " correct += (maxIndexes==labels).sum().item()\n", 352 | " total += labels.size(0)\n", 353 | " \n", 354 | " print('in 1w test_data correct rate = {:.4f}'.format((correct/total)*100))" 355 | ], 356 | "execution_count": 23, 357 | "outputs": [ 358 | { 359 | "output_type": "stream", 360 | "name": "stderr", 361 | "text": [ 362 | "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 5 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", 363 | " cpuset_checked))\n" 364 | ] 365 | }, 366 | { 367 | "output_type": "stream", 368 | "name": "stdout", 369 | "text": [ 370 | "in 1w test_data correct rate = 98.5100\n" 371 | ] 372 | } 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "metadata": { 378 | "colab": { 379 | "base_uri": "https://localhost:8080/", 380 | "height": 283 381 | }, 382 | "id": "418J4zPhBlu6", 383 | "outputId": "041cfecc-0197-40fa-c716-3a1505515baf" 384 | }, 385 | "source": [ 386 | "pd.DataFrame(train_curve).plot() # loss曲线" 387 | ], 388 | "execution_count": 24, 389 | "outputs": [ 390 | { 391 | "output_type": "execute_result", 392 | "data": { 393 | "text/plain": [ 394 | "" 395 | ] 396 | }, 397 | "metadata": {}, 398 | "execution_count": 24 399 | }, 400 | { 401 | "output_type": "display_data", 402 | "data": { 403 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xcZb3v8c9vZnJp0qSXSXpNSVrbDVIQKYEW2ZvjEeS2laKClmu5bKoCHtjsfRRf7O11q7jdiiIIh1KkYCkgisUtt1pQz1FaSKGUXoCG0tKUXtL7NU0y85w/1pNmmiZtkklmklnf9+uV16z1rDWzfp1Mv2vlWc+sZc45REQkHCLZLkBERDJHoS8iEiIKfRGREFHoi4iEiEJfRCREYtku4EjKyspcVVVVtssQEelXFi9evMU5V97esj4d+lVVVdTU1GS7DBGRfsXM1na0TN07IiIhotAXEQkRhb6ISIj06T59EZFsaWpqoq6ujoaGhmyX0qHCwkIqKirIy8vr9HMU+iIi7airq6OkpISqqirMLNvlHMY5x9atW6mrq2Ps2LGdfp66d0RE2tHQ0EA8Hu+TgQ9gZsTj8S7/JaLQFxHpQF8N/Bbdqe+ooW9mD5rZZjNbltL2IzN7y8yWmtlTZjY4ZdnXzazWzN42s3NT2s/zbbVmdluXK+2CA80JfvDsSuq27+vNzYiI9DudOdJ/CDivTdt84ATn3EeAd4CvA5jZ8cA0YKJ/zi/MLGpmUeAe4HzgeOBSv26v2LzrAI8ufJ+bHn2dxuZkb21GRKTXPffccxx77LGMHz+eO+64I+3XO2roO+f+Amxr0/aCc67Zzy4EKvz0VOAx59wB59x7QC1wmv+pdc6tds41Ao/5dXvFmKFF/OiSj7Bk3Q6+/8zK3tqMiEivSiQS3HjjjTz77LOsWLGCuXPnsmLFirResyf69K8FnvXTo4F1KcvqfFtH7YcxsxlmVmNmNfX19d0u6rwTRnLtGWN56G9r+MPSDd1+HRGRbHnllVcYP34848aNIz8/n2nTpjFv3ry0XjOtIZtmdjvQDMxJq4oUzrn7gfsBqqur07qX423nH8fr67bztd8s5cMjSxhXPrBHahSRcPn275ez4oNdPfqax48q5ZufnnjEddavX8+YMWMOzldUVLBo0aK0ttvtI30zuxr4FHC5a73R7npgTMpqFb6to/ZelR+LcM9lk8iLGjfMeY2GpkRvb1JEpE/r1pG+mZ0HfBX4H8651CEyTwOPmtlPgFHABOAVwIAJZjaWIOynAZelU3hnjRo8gDu/8FGueehVvjFvGf958UmZ2KyI5JCjHZH3ltGjR7NuXWvPeF1dHaNHt9sz3mmdGbI5F3gZONbM6szsOuBuoASYb2ZLzOw+AOfccuAJYAXwHHCjcy7hT/reBDwPrASe8OtmxMePHcZN/3M8T9TU8euadUd/gohIH3DqqaeyatUq3nvvPRobG3nssce48MIL03rNox7pO+cubad51hHW/x7wvXbanwGe6VJ1PeiWs/+OxWu38+/zlnFixSCOG1GarVJERDolFotx9913c+6555JIJLj22muZODG9vzpC843caMT42bSTKS3M44ZfvcbuhqZslyQiclQXXHAB77zzDu+++y6333572q8XmtAHKC8p4OeXnszabfu47bdv0nr+WUQkHEIV+gCTx8X513OO5Q9LN/Dwyx3eUUxEJCeFLvQBvnjmOM46bhj/8YcVLFm3I9vliEgf1dd7A7pTXyhDPxIxfvz5kxhWUsiNc15jx77GbJckIn1MYWEhW7du7bPB33I9/cLCwi49L7Q3URlclM8vLp/EJfe9zK1PvMEDV1UTifTty6iKSOZUVFRQV1dHOpeD6W0td87qitCGPsBJYwbzb5/6MN+Yt5z7/vIuN3x8fLZLEpE+Ii8vr0t3pOovQtm9k+rKKZV8+qRR/Nfzb7Nw9dZslyMi0qtCH/pmxg8+eyJVZcV8Ze7rbN7dd2+CLCKSrtCHPsDAghj3Xn4KuxuauHnuEhLJvnniRkQkXQp979gRJfzHRSfy8uqt3Dn/nWyXIyLSKxT6KS4+pYIvVI/h7pdqefGtTdkuR0Skxyn02/j21IlMHFXKDXNe4+V3dWJXRHKLQr+Nwrwos689jTFDirhu9qu8umbb0Z8kItJPKPTbUTawgDnXT2bEoEKufvAVFq/dnu2SRER6hEK/A8NKCpl7/RSGlQbB/4au0SMiOUChfwTDSwt59PrJDCnO58pZi1i2fme2SxIRSYtC/yhGDhrAo9dPpqQwjytmLWLFB7uyXZKISLcp9DuhYkgRc6+fwoC8KFfMWsTbG3dnuyQRkW5R6HfSMfEg+POixuUPLKR2s4JfRPofhX4XVJUV8+j1UwDj0pmLWF2/J9sliYh0iUK/iz5UPpC5108mmXRcNnMRa7fuzXZJIiKdptDvhgnDS5hz/WQONCe4bOYi1m3bl+2SREQ6RaHfTceNKOVX/zSZPQeauXTmQtbv2J/tkkREjuqooW9mD5rZZjNbltI21Mzmm9kq/zjEt5uZ3WVmtWa21MwmpTxnul9/lZlN751/TmZNHDWIX103mZ37m7hs5kI27tS1+EWkb+vMkf5DwHlt2m4DFjjnJgAL/DzA+cAE/zMDuBeCnQTwTWAycBrwzZYdRX93YsUgHr72NLbuaeTSmQvZvEvBLyJ911FD3zn3F6DtVcemArP99GzgopT2h11gITDYzEYC5wLznXPbnHPbgfkcviPpt04+ZggPXXMqm3Y1cOnMhdTvPpDtkkRE2tXdPv3hzrkNfnojMNxPjwbWpaxX59s6aj+Mmc0wsxozq+nLd6Fvq7pqKL+8+lQ+2BEEv7p6RKQvSvtErnPOAT12f0Hn3P3OuWrnXHV5eXlPvWxGTB4X55fXnMrGnQ187t6/aRy/iPQ53Q39Tb7bBv+42bevB8akrFfh2zpqzzlTxsV5bMYUGpoSXHLfy7xZp4u0iUjf0d3QfxpoGYEzHZiX0n6VH8UzBdjpu4GeB84xsyH+BO45vi0nnTB6EL/+0ukU5kW5dOZC/vbulmyXJCICdG7I5lzgZeBYM6szs+uAO4BPmtkq4Gw/D/AMsBqoBWYCNwA457YB3wVe9T/f8W05a1z5QH7z5Y8xanAhVz/4Ks8t25jtkkREsKBLvm+qrq52NTU12S4jLTv2NXLNQ6/yxrod/OCzJ/KFU4/JdkkikuPMbLFzrrq9ZfpGbi8bXJTPnH+azD9MKOdrv3mTe//0Ln15RysiuU2hnwFF+TFmXlXNhSeN4ofPvcX3n1mp4BeRrIhlu4CwyI9F+OkXPsqQojxm/t/32L6viTs+eyKxqPa7IpI5Cv0MikSMb104kaHFBdz5x3fYsa+Juy87mcK8aLZLE5GQ0GFmhpkZN589ge9OnciCtzZx1YOvsKuhKdtliUhIKPSz5MrTq7hr2sm8/v52pv0fXa9HRDJDoZ9Fnz5pFLOmn8p7W/Zy8X1/081YRKTXKfSz7My/K2fO9cE1+T937994a+OubJckIjlMod8HTDpmCL/+4ulEzLjkvpf5a60u2yAivUOh30dMGF7Ck18+nVGDBnDVg68wZ9HabJckIjlIod+HVAwp4skvn86ZE8q4/allfOf3K0gk9SUuEek5Cv0+pqQwjwemn8q1Z4zlwb++x/UP17BbQzpFpIco9PugaMT4xqeP53ufOYE/v1PPxfe+rJE9ItIjFPp92OWTK5l9zWls2Lmfz/ziryxeuz3bJYlIP6fQ7+P+fkIZT914BgMLYlw6cyG/ez0nbzgmIhmi0O8HPlQ+kKduOIOTxwzmlseX8JMX3iapE7wi0g0K/X5iSHE+j1w3mS9Uj+GuF2v5ytzX2d+YyHZZItLP6Cqb/Uh+LMIdnzuR8cMG8v1nV1K3fR8zr6pmWGlhtksTkX5CR/r9jJlx/ZnjuP/KalZt3sPUe/7K8g92ZrssEeknFPr91CePH86TX/oYBlxy38u8sFw3XheRo1Po92PHjyrldzedwYThJXzxV4u578+6/66IHJlCv58bVlLI4zOmcMGJI7nj2bdYuHpbtksSkT5MoZ8DCvOifPvCiQC6NLOIHJFCP0fEi/MZWBBj7VZdrkFEOpZW6JvZP5vZcjNbZmZzzazQzMaa2SIzqzWzx80s369b4Odr/fKqnvgHSMDMqIwXsWbr3myXIiJ9WLdD38xGA/8LqHbOnQBEgWnAD4E7nXPjge3Adf4p1wHbffudfj3pQVXxYh3pi8gRpdu9EwMGmFkMKAI2AJ8AnvTLZwMX+empfh6//CwzszS3Lykq40Ws27aP5kQy26WISB/V7dB3zq0H/gt4nyDsdwKLgR3OuWa/Wh0w2k+PBtb55zb79eNtX9fMZphZjZnV1NfXd7e8UKoqK6Y56fhgR0O2SxGRPiqd7p0hBEfvY4FRQDFwXroFOefud85VO+eqy8vL0325UKmKFwOoX19EOpRO987ZwHvOuXrnXBPwW+AMYLDv7gGoAFquBbweGAPglw8CtqaxfWmjKl4EwFqFvoh0IJ3Qfx+YYmZFvm/+LGAF8BJwsV9nOjDPTz/t5/HLX3T6+miPKi8pYEBelDU6mSsiHUinT38RwQnZ14A3/WvdD3wNuNXMagn67Gf5p8wC4r79VuC2NOqWdrQM29SRvoh0JK1LKzvnvgl8s03zauC0dtZtAC5JZ3tydFXxYmrr92S7DBHpo/SN3BxTWVbE+1v3kdCdtUSkHQr9HFMVL6YxkWTjLg3bFJHDKfRzTGXLCJ4t6tcXkcMp9HNM61h9jeARkcMp9HPMiNJC8mMRjeARkXYp9HNMJGJUDtXVNkWkfQr9HFSpq22KSAcU+jmoyl9XX194FpG2FPo5qLKsmIamJJt3H8h2KSLSxyj0c1DLhdfWaNimiLSh0M9BLcM21a8vIm0p9HPQyEGF5EVNI3hE5DAK/RwUi0YYM6RIR/oichiFfo6qjGusvogcTqGfo1rG6mvYpoikUujnqKp4EXsONLN1b2O2SxGRPkShn6Mqy1pG8KiLR0RaKfRz1MGrbW7RyVwRaaXQz1GjBw8gGjEd6YvIIRT6OSo/FmH04AG6rr6IHEKhn8Mq40U60heRQyj0c1hVvFhH+iJyCIV+DquMF7FzfxM79mnYpogEFPo5TPfLFZG20gp9MxtsZk+a2VtmttLMTjezoWY238xW+cchfl0zs7vMrNbMlprZpJ75J0hHqsqCSyyrX19EWqR7pP8z4Dnn3HHAScBK4DZggXNuArDAzwOcD0zwPzOAe9PcthxFxZAizDRWX0RadTv0zWwQcCYwC8A51+ic2wFMBWb71WYDF/npqcDDLrAQGGxmI7tduRxVYV6UUYMG6EhfRA5K50h/LFAP/NLMXjezB8ysGBjunNvg19kIDPfTo4F1Kc+v822HMLMZZlZjZjX19fVplCegq22KyKHSCf0YMAm41zl3MrCX1q4cAFxwiccuXebROXe/c67aOVddXl6eRnkCwdU2dSJXRFqkE/p1QJ1zbpGff5JgJ7CppdvGP272y9cDY1KeX+HbpBdVxYvYtreRnfubsl2KiPQB3Q5959xGYJ2ZHeubzgJWAE8D033bdGCen34auMqP4pkC7EzpBpJeUumHbb6vo30RIeiiScdXgDlmlg+sBq4h2JE8YWbXAWuBz/t1nwEuAGqBfX5d6WUtwzbXbN3LiRWDslyNiGRbWqHvnFsCVLez6Kx21nXAjelsT7rumKEaqy8irfSN3BxXlB9jeGmBTuaKCKDQD4Xgfrk60hcRhX4oVMWLdKQvIoBCPxQq48XU7z7A3gPN2S5FRLJMoR8CLVfbXKujfZHQU+iHQGVcI3hEJKDQD4GW0Fe/vogo9EOgpDCPsoH5OtIXEYV+WAQXXlPoi4SdQj8kKuNFOpErIgr9sKiKF7NhZwMNTYlslyIiWaTQD4mWk7nvb9PRvkiYKfRDomWs/pot6tcXCTOFfkjoC1oiAgr90BhUlMfgojyN4BEJOYV+iARX29SRvkiYKfRDJLjapo70RcJMoR8ilfFiPtixnwPNGrYpElYK/RCpiheRdFC3fX+2SxGRLFHoh0jlwRE86uIRCSuFfohUtVxtc4tO5oqElUI/RIYW51NSENORvkiIKfRDxMyoLNP9ckXCTKEfMsFYfR3pi4RV2qFvZlEze93M/tvPjzWzRWZWa2aPm1m+by/w87V+eVW625auq4oXUbd9P02JZLZLEZEs6Ikj/ZuBlSnzPwTudM6NB7YD1/n264Dtvv1Ov55kWGW8mOak44MdGrYpEkZphb6ZVQD/CDzg5w34BPCkX2U2cJGfnurn8cvP8utLBh282qb69UVCKd0j/Z8CXwVa+griwA7nXLOfrwNG++nRwDoAv3ynX/8QZjbDzGrMrKa+vj7N8qStlmGb6tcXCaduh76ZfQrY7Jxb3IP14Jy73zlX7ZyrLi8v78mXFqC8pIABeVGN1RcJqVgazz0DuNDMLgAKgVLgZ8BgM4v5o/kKYL1ffz0wBqgzsxgwCNiaxvalG8zM3y9XR/oiYdTtI33n3NedcxXOuSpgGvCic+5y4CXgYr/adGCen37az+OXv+icc93dvnRfVbxYV9sUCaneGKf/NeBWM6sl6LOf5dtnAXHffitwWy9sWzqhsqyIddv2k0hqnysSNul07xzknPsT8Cc/vRo4rZ11GoBLemJ7kp6qeDGNiSQbdu6nYkhRtssRkQzSN3JDqPLgCB6dzBUJG4V+CLWO1Ve/vkjYKPRDaERpIfmxiI70RUJIoR9CkYhRObSINVt0pC8SNgr9kAqutqkjfZGwUeiHVFW8iLXb9pLUsE2RUFHoh1RlWTENTUk27z6Q7VJEJIMU+iF18H65GsEjEioK/ZBqGbapa/CIhItCP6RGDiokL2q6rr5IyCj0QyoWjTBmiK62KRI2Cv0Qq4wX6br6IiGj0A+xYKz+XnSFa5HwUOiHWFW8iL2NCbbsacx2KSKSIQr9EKss0wgekbBR6IdY69U21a8vEhYK/RAbPXgA0YjpSF8kRBT6IZYfizB68AAd6YuEiEI/5CrjGqsvEiYK/ZCrihfz3hYN2xQJC4V+yFXGi9jd0MyOfU3ZLkVEMkChH3K6X65IuCj0Q66qLLjEsu6iJRIOCv2QqxhShJmO9EXCotuhb2ZjzOwlM1thZsvN7GbfPtTM5pvZKv84xLebmd1lZrVmttTMJvXUP0K6rzAvyqhBA3SkLxIS6RzpNwP/4pw7HpgC3GhmxwO3AQuccxOABX4e4Hxggv+ZAdybxralB1XGi3SkLxIS3Q5959wG59xrfno3sBIYDUwFZvvVZgMX+empwMMusBAYbGYju1259Jjgaps60hcJgx7p0zezKuBkYBEw3Dm3wS/aCAz306OBdSlPq/NtbV9rhpnVmFlNfX19T5QnR1EVL2Lb3kZ27tewTZFcl3bom9lA4DfALc65XanLXPCNny5968c5d79zrto5V11eXp5uedIJlX7Y5vs62hfJeWmFvpnlEQT+HOfcb33zppZuG/+42bevB8akPL3Ct0mWtQzbVL++SO5LZ/SOAbOAlc65n6QsehqY7qenA/NS2q/yo3imADtTuoEkiyqHFmMGC1dvzXYpItLL0jnSPwO4EviEmS3xPxcAdwCfNLNVwNl+HuAZYDVQC8wEbkhj29KDBuRHuXzyMcxZ9D5/WKr9sEgui3X3ic65/wdYB4vPamd9B9zY3e1J7/r3Tx3Pig928b+ffIPxwwZy7IiSbJckIr1A38gVAApiUe694hSKC2LMeKSGnboAm0hOUujLQcNLC7nvikl8sGM/Nz/+OomkLrcskmsU+nKIUyqH8q0LJ/Knt+u5c/472S5HRHqYQl8Oc9lpxzDt1DHc/VItzy3TiV2RXKLQl8OYGd+eOpGPjhnMvzzxBqs27c52SSLSQxT60q6CWJT7rjiFAfkxZjyymF0NOrErkgsU+tKhEYMK+cXlk1i3bR///NgSkjqxK9LvKfTliE4bO5Rvfvp4Fry1mZ8uWJXtckQkTQp9OaorplRyySkV3LVgFS8s35jtckQkDQp9OSoz47sXncBHKgZx6xNvULt5T7ZLEpFuUuhLpxTmBSd2C2IRZjxSw26d2BXplxT60mmjBg/gnssnsXbrPm594g2d2BXphxT60iVTxsX5t3/8MPNXbOLul2qzXY6IdJFCX7rs6o9V8dlJo7nzj++wYOWmbJcjIl2g0JcuMzO+/5kTmTiqlFseW8Lqep3YFekvFPrSLS0ndvNiEWY8spg9B5qzXZKIdEK3b6IiUjGkiLsvO5krZ73C2T/+M8eNLGFc2UDGlRczrqyYceUDGV5aQHBnTRHpCxT6kpaPfaiMX1w+id+/8QGr6/eyaPU29jclDi4vzo8ytry4dWdQPtDvEIopytfHTyTT9L9O0nbuxBGcO3EEAMmkY+OuBt7bspfV9Xt4t34vq7fs5bX3t/P7pR/gUkZ5jhxUyNiyYirjxQwvLWB4aSHDSvxjaQHx4gKiEf2VINKTFPrSoyIRY9TgAYwaPIAzxpcdsqyhKcGarXtZXR/sEFbX7+XdLXt5fvlGtu1tPPy1DMpLChhWUsjw0gLK/eOwlMdhpQUMGpBHQSyibiSRTlDoS8YU5kU5bkQpx40oPWxZY3OSLXsOsGlXA5t3H2Czf9y0q4FNuw6wfkcDr7+/g63t7BwA8qJGaWEepQPyKCmM+engsXU+aCspaF2vOD9GYX6EAXlRBuRFiUU1tkFym0Jf+oT8WOTgXwhH0rJzaNkh1O8+wK6GJnY3NLNrfxO7GprZ3dDErv1NbNzV4KebDznPcCR5UQt2APnBTqAwZXpAXpTC/ChFvq0gFqEg5h/zDp/Oj7bfXhCLkB+LkBeNkB8NptWNJZmi0Jd+pbM7h7aaEsmUHUOwI9jd0MS+xgT7mxI0NCXY35hgn39saAra9ze2Pu7c3xSs69c70JSkoTlxyHmK7ooYh+wE8qIR8mJ2eFvUyG/ZofidR8vy/FhKm28vONgePdgWjRhmEDHzP8F3L6KR1ulI6vJI63Qsaoe8bp7fdixi6l7rJxT6Egp50QhDi/MZWpzfo6/rnKM56TjQnKSxOcmB5mBncKBlujnp5xN+eWt7Y3OSpoSjKZGkKRHMN/rppmZHY8LPN/vlvn3n/qZg3eZEsM7BbQePzVm4JpIZwc4nZQfUspM5dMcQ7FTMPydirTug4HX8jofWZWBEIxzc2eXH7JDXbXlM3XZrm/m/pILtRyOW8hjsAKNt26OHLk9nXxax4HUPPvaBv+gyHvpmdh7wMyAKPOCcuyPTNYj0FDMjLxockVOQ7WoCyWSww0jdEbXsUBqbkyQdJJ3DORdMJx0J53C+PXV5InnodHOy9XUbmw/dWR3ymLIDa9khJZLBNhzBNlwSEgT1tNTiAA5OO5LJoC2RDHaQbV83Wzu57jLjkJ1AsJOJtJkPHieOGsTPLz25x2vIaOibWRS4B/gkUAe8amZPO+dWZLIOkVwWiRiFkeB8RBi07ORadgStO4cEjf4vpkTSkUg6mpNJ/+hIJPyjb086R3PCtS5PBn+FpcM5/GslSSSDnVfrNl1KXYevc8zQrnVhdlamj/RPA2qdc6sBzOwxYCqg0BeRbgnbTi5dmR6fNhpYlzJf59sOMrMZZlZjZjX19fUZLU5EJNf1uUHJzrn7nXPVzrnq8vLybJcjIpJTMh3664ExKfMVvk1ERDIg06H/KjDBzMaaWT4wDXg6wzWIiIRWRk/kOueazewm4HmCIZsPOueWZ7IGEZEwy/g4fefcM8Azmd6uiIj0wRO5IiLSexT6IiIhYq4nrhbVS8ysHlibxkuUAVt6qJyepLq6RnV1jerqmlysq9I51+6Y9z4d+ukysxrnXHW262hLdXWN6uoa1dU1YatL3TsiIiGi0BcRCZFcD/37s11AB1RX16iurlFdXROqunK6T19ERA6V60f6IiKSQqEvIhIi/T70zew8M3vbzGrN7LZ2lheY2eN++SIzq8pATWPM7CUzW2Fmy83s5nbW+biZ7TSzJf7nG71dV8q215jZm367Ne0sNzO7y79nS81sUgZqOjblvVhiZrvM7JY262TkPTOzB81ss5ktS2kbambzzWyVfxzSwXOn+3VWmdn0DNT1IzN7y/+enjKzwR0894i/816o61tmtj7ld3VBB8894v/fXqjr8ZSa1pjZkg6e25vvV7v5kLHPmPP3v+yPPwQXbXsXGAfkA28Ax7dZ5wbgPj89DXg8A3WNBCb56RLgnXbq+jjw31l639YAZUdYfgHwLMH9qacAi7Lwe91I8AWTjL9nwJnAJGBZStt/Arf56duAH7bzvKHAav84xE8P6eW6zgFifvqH7dXVmd95L9T1LeBfO/F7PuL/356uq83yHwPfyML71W4+ZOoz1t+P9A/eftE51wi03H4x1VRgtp9+EjjLLJ372x+dc26Dc+41P70bWEmbO4T1cVOBh11gITDYzEZmcPtnAe8659L5Nna3Oef+Amxr05z6OZoNXNTOU88F5jvntjnntgPzgfN6sy7n3AvOuWY/u5DgHhUZ1cH71Rmd+f/bK3X5DPg8MLenttdZR8iHjHzG+nvoH/X2i6nr+P8cO4F4RqoDfHfSycCidhafbmZvmNmzZjYxUzUBDnjBzBab2Yx2lnfmfe1N0+j4P2O23rPhzrkNfnojMLyddbL9vl1L8Bdae472O+8NN/lupwc76KrI5vv1D8Am59yqDpZn5P1qkw8Z+Yz199Dv08xsIPAb4Bbn3K42i18j6L44Cfg58LsMlvb3zrlJwPnAjWZ2Zga3fUQW3FznQuDX7SzO5nt2kAv+zu5TY53N7HagGZjTwSqZ/p3fC3wI+CiwgaArpS+5lCMf5ff6+3WkfOjNz1h/D/3O3H7x4DpmFgMGAVt7uzAzyyP4hc5xzv227XLn3C7n3B4//QyQZ2ZlvV2X3956/7gZeIrgz+xU2byt5fnAa865TW0XZPM9Aza1dHH5x83trJOV983MrgY+BVzuw+Iwnfid9yjn3CbnXMI5lwRmdrC9bL1fMeCzwOMdrdPb71cH+ZCRz1h/D/3O3H7xaaDlDPfFwJXrcbMAAAFxSURBVIsd/cfoKb6/cBaw0jn3kw7WGdFybsHMTiP4XWRiZ1RsZiUt0wQnApe1We1p4CoLTAF2pvzZ2ds6PALL1nvmpX6OpgPz2lnneeAcMxviuzPO8W29xszOA74KXOic29fBOp35nfd0XanngD7TwfaydfvUs4G3nHN17S3s7ffrCPmQmc9Yb5ydzuQPwUiTdwhGAdzu275D8J8AoJCgq6AWeAUYl4Ga/p7gT7OlwBL/cwHwJeBLfp2bgOUEIxYWAh/L0Ps1zm/zDb/9lvcstTYD7vHv6ZtAdYZqKyYI8UEpbRl/zwh2OhuAJoI+0+sIzgMtAFYBfwSG+nWrgQdSnnut/6zVAtdkoK5agj7els9Zy0i1UcAzR/qd93Jdj/jPzlKCMBvZti4/f9j/396sy7c/1PKZSlk3k+9XR/mQkc+YLsMgIhIi/b17R0REukChLyISIgp9EZEQUeiLiISIQl9EJEQU+iIiIaLQFxEJkf8PYkCLlTQG8KMAAAAASUVORK5CYII=\n", 404 | "text/plain": [ 405 | "
" 406 | ] 407 | }, 408 | "metadata": { 409 | "needs_background": "light" 410 | } 411 | } 412 | ] 413 | } 414 | ] 415 | } -------------------------------------------------------------------------------- /deeplearning/resnet.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | # handwritten digits recognition 3 | # Data: MINIST 4 | # model: resnet 5 | # date: 2021.10.8 14:18 6 | 7 | import math 8 | import torch 9 | import torchvision 10 | import torchvision.transforms as transforms 11 | import torch.nn as nn 12 | import torch.utils.data as Data 13 | import torch.optim as optim 14 | import pandas as pd 15 | import matplotlib.pyplot as plt 16 | 17 | train_curve = [] 18 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 19 | 20 | # param 21 | batch_size = 100 22 | n_class = 10 23 | padding_size = 15 24 | epoches = 10 25 | 26 | train_dataset = torchvision.datasets.MNIST('./data/', train=True, transform=transforms.ToTensor(), download=True) 27 | test_dataset = torchvision.datasets.MNIST('./data/', train=False, transform=transforms.ToTensor(), download=False) 28 | train = Data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=5) 29 | test = Data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=5) 30 | 31 | def gelu(x): 32 | "Implementation of the gelu activation function by Hugging Face" 33 | return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) 34 | 35 | class ResBlock(nn.Module): 36 | # 残差块 37 | def __init__(self, in_size, out_size1, out_size2): 38 | super(ResBlock, self).__init__() 39 | self.conv1 = nn.Conv2d( 40 | in_channels = in_size, 41 | out_channels = out_size1, 42 | kernel_size = 3, 43 | stride = 2, 44 | padding = padding_size 45 | ) 46 | self.conv2 = nn.Conv2d( 47 | in_channels = out_size1, 48 | out_channels = out_size2, 49 | kernel_size = 3, 50 | stride = 2, 51 | padding = padding_size 52 | ) 53 | self.batchnorm1 = nn.BatchNorm2d(out_size1) 54 | self.batchnorm2 = nn.BatchNorm2d(out_size2) 55 | 56 | def conv(self, x): 57 | # gelu效果比relu好呀哈哈 58 | x = gelu(self.batchnorm1(self.conv1(x))) 59 | x = gelu(self.batchnorm2(self.conv2(x))) 60 | return x 61 | 62 | def forward(self, x): 63 | # 残差连接 64 | return x + self.conv(x) 65 | 66 | # resnet 67 | class Resnet(nn.Module): 68 | def __init__(self, n_class = n_class): 69 | super(Resnet, self).__init__() 70 | self.res1 = ResBlock(1, 8, 16) 71 | self.res2 = ResBlock(16, 32, 16) 72 | self.conv = nn.Conv2d( 73 | in_channels = 16, 74 | out_channels = n_class, 75 | kernel_size = 3, 76 | stride = 2, 77 | padding = padding_size 78 | ) 79 | self.batchnorm = nn.BatchNorm2d(n_class) 80 | self.max_pooling = nn.AdaptiveAvgPool2d(1) 81 | 82 | def forward(self, x): 83 | # x: [bs, 1, h, w] 84 | # x = x.view(-1, 1, 28, 28) 85 | x = self.res1(x) 86 | x = self.res2(x) 87 | x = self.max_pooling(self.batchnorm(self.conv(x))) 88 | 89 | return x.view(x.size(0), -1) 90 | 91 | resnet = Resnet().to(device) 92 | 93 | loss_fn = nn.CrossEntropyLoss() 94 | optimizer = optim.SGD(params=resnet.parameters(), lr=1e-2, momentum=0.9) 95 | 96 | # train 97 | total_step = len(train) 98 | sum_loss = 0 99 | for epoch in range(epoches): 100 | for i, (images, targets) in enumerate(train): 101 | optimizer.zero_grad() 102 | images = images.to(device) 103 | targets = targets.to(device) 104 | preds = resnet(images) 105 | 106 | loss = loss_fn(preds, targets) 107 | sum_loss += loss.item() 108 | loss.backward() 109 | optimizer.step() 110 | if (i+1)%100==0: 111 | print('[{}|{}] step:{}/{} loss:{:.4f}'.format(epoch+1, epoches, i+1, total_step, loss.item())) 112 | train_curve.append(sum_loss) 113 | sum_loss = 0 114 | 115 | # test 116 | resnet.eval() 117 | with torch.no_grad(): 118 | correct = 0 119 | total = 0 120 | for images, labels in test: 121 | images = images.to(device) 122 | labels = labels.to(device) 123 | outputs = resnet(images) 124 | _, maxIndexes = torch.max(outputs, dim=1) 125 | correct += (maxIndexes==labels).sum().item() 126 | total += labels.size(0) 127 | 128 | print('in 1w test_data correct rate = {:.4f}'.format((correct/total)*100)) 129 | 130 | pd.DataFrame(train_curve).plot() # loss曲线 131 | -------------------------------------------------------------------------------- /nlp/Bert_Blend_CNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Bert_Blend-CNN.ipynb", 7 | "provenance": [] 8 | }, 9 | "kernelspec": { 10 | "name": "python3", 11 | "display_name": "Python 3" 12 | }, 13 | "language_info": { 14 | "name": "python" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "code", 21 | "metadata": { 22 | "id": "UahhCDlxtpWr" 23 | }, 24 | "source": [ 25 | "# -*- coding:utf-8 -*-\n", 26 | "# bert融合textcnn思想的Bert+Blend-CNN\n", 27 | "# model: Bert+Blend-CNN\n", 28 | "# date: 2021.10.11 18:06:11\n", 29 | "\n", 30 | "import os\n", 31 | "import numpy as np\n", 32 | "import pandas as pd\n", 33 | "import torch\n", 34 | "import torch.nn as nn\n", 35 | "import torch.utils.data as Data\n", 36 | "import torch.nn.functional as F\n", 37 | "import torch.optim as optim\n", 38 | "import transformers\n", 39 | "from transformers import AutoModel, AutoTokenizer\n", 40 | "import matplotlib.pyplot as plt" 41 | ], 42 | "execution_count": 62, 43 | "outputs": [] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "metadata": { 48 | "id": "dL4eT_MTS9JY" 49 | }, 50 | "source": [ 51 | "train_curve = []\n", 52 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" 53 | ], 54 | "execution_count": 63, 55 | "outputs": [] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "metadata": { 60 | "id": "ZdVB3Lt6TAEs" 61 | }, 62 | "source": [ 63 | "# # 定义一些参数，模型选择了最基础的bert中文模型\n", 64 | "batch_size = 2\n", 65 | "epoches = 100\n", 66 | "model = \"bert-base-chinese\"\n", 67 | "hidden_size = 768\n", 68 | "n_class = 2\n", 69 | "maxlen = 8\n", 70 | "\n", 71 | "encode_layer=12\n", 72 | "filter_sizes = [2, 2, 2]\n", 73 | "num_filters = 3" 74 | ], 75 | "execution_count": 64, 76 | "outputs": [] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "id": "gQ3SK8_rTFGX" 82 | }, 83 | "source": [ 84 | "# data，构造一些训练数据\n", 85 | "sentences = [\"我喜欢打篮球\", \"这个相机很好看\", \"今天玩的特别开心\", \"我不喜欢你\", \"太糟糕了\", \"真是件令人伤心的事情\"]\n", 86 | "labels = [1, 1, 1, 0, 0, 0] # 1积极, 0消极." 87 | ], 88 | "execution_count": 65, 89 | "outputs": [] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "metadata": { 94 | "id": "d8eBhZbDTJSz" 95 | }, 96 | "source": [ 97 | "class MyDataset(Data.Dataset):\n", 98 | " def __init__(self, sentences, labels=None, with_labels=True,):\n", 99 | " self.tokenizer = AutoTokenizer.from_pretrained(model)\n", 100 | " self.with_labels = with_labels\n", 101 | " self.sentences = sentences\n", 102 | " self.labels = labels\n", 103 | " def __len__(self):\n", 104 | " return len(sentences)\n", 105 | "\n", 106 | " def __getitem__(self, index):\n", 107 | " # Selecting sentence1 and sentence2 at the specified index in the data frame\n", 108 | " sent = self.sentences[index]\n", 109 | "\n", 110 | " # Tokenize the pair of sentences to get token ids, attention masks and token type ids\n", 111 | " encoded_pair = self.tokenizer(sent,\n", 112 | " padding='max_length', # Pad to max_length\n", 113 | " truncation=True, # Truncate to max_length\n", 114 | " max_length=maxlen, \n", 115 | " return_tensors='pt') # Return torch.Tensor objects\n", 116 | "\n", 117 | " token_ids = encoded_pair['input_ids'].squeeze(0) # tensor of token ids\n", 118 | " attn_masks = encoded_pair['attention_mask'].squeeze(0) # binary tensor with \"0\" for padded values and \"1\" for the other values\n", 119 | " token_type_ids = encoded_pair['token_type_ids'].squeeze(0) # binary tensor with \"0\" for the 1st sentence tokens & \"1\" for the 2nd sentence tokens\n", 120 | "\n", 121 | " if self.with_labels: # True if the dataset has labels\n", 122 | " label = self.labels[index]\n", 123 | " return token_ids, attn_masks, token_type_ids, label\n", 124 | " else:\n", 125 | " return token_ids, attn_masks, token_type_ids" 126 | ], 127 | "execution_count": 66, 128 | "outputs": [] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "metadata": { 133 | "id": "q-fhfJi7xkXd" 134 | }, 135 | "source": [ 136 | "train = Data.DataLoader(dataset=MyDataset(sentences, labels), batch_size=batch_size, shuffle=True, num_workers=1)" 137 | ], 138 | "execution_count": 67, 139 | "outputs": [] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "metadata": { 144 | "id": "aqjKaE6-UPE1" 145 | }, 146 | "source": [ 147 | "class TextCNN(nn.Module):\n", 148 | " def __init__(self):\n", 149 | " super(TextCNN, self).__init__()\n", 150 | " self.num_filter_total = num_filters * len(filter_sizes)\n", 151 | " self.Weight = nn.Linear(self.num_filter_total, n_class, bias=False)\n", 152 | " self.bias = nn.Parameter(torch.ones([n_class]))\n", 153 | " self.filter_list = nn.ModuleList([\n", 154 | " nn.Conv2d(1, num_filters, kernel_size=(size, hidden_size)) for size in filter_sizes\n", 155 | " ])\n", 156 | "\n", 157 | " def forward(self, x):\n", 158 | " # x: [bs, seq, hidden]\n", 159 | " x = x.unsqueeze(1) # [bs, channel=1, seq, hidden]\n", 160 | " \n", 161 | " pooled_outputs = []\n", 162 | " for i, conv in enumerate(self.filter_list):\n", 163 | " h = F.relu(conv(x)) # [bs, channel=1, seq-kernel_size+1, 1]\n", 164 | " mp = nn.MaxPool2d(\n", 165 | " kernel_size = (encode_layer-filter_sizes[i]+1, 1)\n", 166 | " )\n", 167 | " # mp: [bs, channel=3, w, h]\n", 168 | " pooled = mp(h).permute(0, 3, 2, 1) # [bs, h=1, w=1, channel=3]\n", 169 | " pooled_outputs.append(pooled)\n", 170 | " \n", 171 | " h_pool = torch.cat(pooled_outputs, len(filter_sizes)) # [bs, h=1, w=1, channel=3 * 3]\n", 172 | " h_pool_flat = torch.reshape(h_pool, [-1, self.num_filter_total])\n", 173 | " \n", 174 | " output = self.Weight(h_pool_flat) + self.bias # [bs, n_class]\n", 175 | "\n", 176 | " return output" 177 | ], 178 | "execution_count": 68, 179 | "outputs": [] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "metadata": { 184 | "id": "AAqolyEvTNYJ" 185 | }, 186 | "source": [ 187 | "# model\n", 188 | "class Bert_Blend_CNN(nn.Module):\n", 189 | " def __init__(self):\n", 190 | " super(Bert_Blend_CNN, self).__init__()\n", 191 | " self.bert = AutoModel.from_pretrained(model, output_hidden_states=True, return_dict=True)\n", 192 | " self.linear = nn.Linear(hidden_size, n_class)\n", 193 | " self.textcnn = TextCNN()\n", 194 | " \n", 195 | " def forward(self, X):\n", 196 | " input_ids, attention_mask, token_type_ids = X[0], X[1], X[2]\n", 197 | " outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids) # 返回一个output字典\n", 198 | " # 取每一层encode出来的向量\n", 199 | " # outputs.pooler_output: [bs, hidden_size]\n", 200 | " hidden_states = outputs.hidden_states # 13*[bs, seq_len, hidden] 第一层是embedding层不需要\n", 201 | " cls_embeddings = hidden_states[1][:, 0, :].unsqueeze(1) # [bs, 1, hidden]\n", 202 | " # 将每一层的第一个token(cls向量)提取出来，拼在一起当作textcnn的输入\n", 203 | " for i in range(2, 13):\n", 204 | " cls_embeddings = torch.cat((cls_embeddings, hidden_states[i][:, 0, :].unsqueeze(1)), dim=1)\n", 205 | " # cls_embeddings: [bs, encode_layer=12, hidden]\n", 206 | " logits = self.textcnn(cls_embeddings)\n", 207 | " return logits" 208 | ], 209 | "execution_count": 69, 210 | "outputs": [] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "metadata": { 215 | "id": "_E6qpfSATZd1", 216 | "outputId": "baa36957-5382-42f5-b0d0-9d334070732b", 217 | "colab": { 218 | "base_uri": "https://localhost:8080/" 219 | } 220 | }, 221 | "source": [ 222 | "bert_blend_cnn = Bert_Blend_CNN().to(device)" 223 | ], 224 | "execution_count": 70, 225 | "outputs": [ 226 | { 227 | "output_type": "stream", 228 | "name": "stderr", 229 | "text": [ 230 | "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n", 231 | "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", 232 | "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" 233 | ] 234 | } 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "metadata": { 240 | "id": "QblwR6DexAFe" 241 | }, 242 | "source": [ 243 | "optimizer = optim.Adam(bert_blend_cnn.parameters(), lr=1e-3, weight_decay=1e-2)\n", 244 | "loss_fn = nn.CrossEntropyLoss()" 245 | ], 246 | "execution_count": 71, 247 | "outputs": [] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "metadata": { 252 | "id": "bBzQBHt8xbKm", 253 | "outputId": "a43c4a60-17af-4d63-f3c4-afd329aa8605", 254 | "colab": { 255 | "base_uri": "https://localhost:8080/" 256 | } 257 | }, 258 | "source": [ 259 | "# train\n", 260 | "sum_loss = 0\n", 261 | "total_step = len(train)\n", 262 | "for epoch in range(epoches):\n", 263 | " for i, batch in enumerate(train):\n", 264 | " optimizer.zero_grad()\n", 265 | " batch = tuple(p.to(device) for p in batch)\n", 266 | " pred = bert_blend_cnn([batch[0], batch[1], batch[2]])\n", 267 | " loss = loss_fn(pred, batch[3])\n", 268 | " sum_loss += loss.item()\n", 269 | "\n", 270 | " loss.backward()\n", 271 | " optimizer.step()\n", 272 | " if epoch % 10 == 0:\n", 273 | " print('[{}|{}] step:{}/{} loss:{:.4f}'.format(epoch+1, epoches, i+1, total_step, loss.item()))\n", 274 | " train_curve.append(sum_loss)\n", 275 | " sum_loss = 0" 276 | ], 277 | "execution_count": 72, 278 | "outputs": [ 279 | { 280 | "output_type": "stream", 281 | "name": "stdout", 282 | "text": [ 283 | "[1|100] step:1/3 loss:1.3587\n", 284 | "[1|100] step:2/3 loss:0.5860\n", 285 | "[1|100] step:3/3 loss:1.3804\n", 286 | "[11|100] step:1/3 loss:0.7330\n", 287 | "[11|100] step:2/3 loss:0.9912\n", 288 | "[11|100] step:3/3 loss:0.5007\n", 289 | "[21|100] step:1/3 loss:0.6944\n", 290 | "[21|100] step:2/3 loss:0.6947\n", 291 | "[21|100] step:3/3 loss:0.6936\n", 292 | "[31|100] step:1/3 loss:0.7441\n", 293 | "[31|100] step:2/3 loss:0.6923\n", 294 | "[31|100] step:3/3 loss:0.6735\n", 295 | "[41|100] step:1/3 loss:0.6875\n", 296 | "[41|100] step:2/3 loss:0.7020\n", 297 | "[41|100] step:3/3 loss:0.6898\n", 298 | "[51|100] step:1/3 loss:0.4228\n", 299 | "[51|100] step:2/3 loss:0.2038\n", 300 | "[51|100] step:3/3 loss:0.0154\n", 301 | "[61|100] step:1/3 loss:0.0064\n", 302 | "[61|100] step:2/3 loss:0.0058\n", 303 | "[61|100] step:3/3 loss:0.0060\n", 304 | "[71|100] step:1/3 loss:0.0039\n", 305 | "[71|100] step:2/3 loss:0.0036\n", 306 | "[71|100] step:3/3 loss:0.0039\n", 307 | "[81|100] step:1/3 loss:0.0021\n", 308 | "[81|100] step:2/3 loss:0.0020\n", 309 | "[81|100] step:3/3 loss:0.0020\n", 310 | "[91|100] step:1/3 loss:0.0029\n", 311 | "[91|100] step:2/3 loss:0.0025\n", 312 | "[91|100] step:3/3 loss:0.0256\n" 313 | ] 314 | } 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "metadata": { 320 | "id": "QkDiBPY3xhxK", 321 | "outputId": "f1cfed2a-c23d-4d2e-8269-90c1437c7831", 322 | "colab": { 323 | "base_uri": "https://localhost:8080/" 324 | } 325 | }, 326 | "source": [ 327 | "# test\n", 328 | "bert_blend_cnn.eval()\n", 329 | "with torch.no_grad():\n", 330 | " test_text = ['我不喜欢打篮球']\n", 331 | " test = MyDataset(test_text, labels=None, with_labels=False)\n", 332 | " x = test.__getitem__(0)\n", 333 | " x = tuple(p.unsqueeze(0).to(device) for p in x)\n", 334 | " pred = bert_blend_cnn([x[0], x[1], x[2]])\n", 335 | " pred = pred.data.max(dim=1, keepdim=True)[1]\n", 336 | " if pred[0][0] == 0:\n", 337 | " print('消极')\n", 338 | " else:\n", 339 | " print('积极')" 340 | ], 341 | "execution_count": 74, 342 | "outputs": [ 343 | { 344 | "output_type": "stream", 345 | "name": "stdout", 346 | "text": [ 347 | "消极\n" 348 | ] 349 | } 350 | ] 351 | }, 352 | { 353 | "cell_type": "code", 354 | "metadata": { 355 | "id": "qELdtXhu2qOw", 356 | "outputId": "b8bfc850-d8e8-4c5b-b624-7ae77b0b7bfb", 357 | "colab": { 358 | "base_uri": "https://localhost:8080/", 359 | "height": 283 360 | } 361 | }, 362 | "source": [ 363 | "pd.DataFrame(train_curve).plot() # loss曲线" 364 | ], 365 | "execution_count": 75, 366 | "outputs": [ 367 | { 368 | "output_type": "execute_result", 369 | "data": { 370 | "text/plain": [ 371 | "" 372 | ] 373 | }, 374 | "metadata": {}, 375 | "execution_count": 75 376 | }, 377 | { 378 | "output_type": "display_data", 379 | "data": { 380 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXSc9X3v8fd3di2jxZJsS7LBJl7AhtgQEyBkJyyhCSG3CZc0S5f0uu0lN0vb01Pa2yWnSZOmaVKSNDnlhjZbA03bUBNKSQhLyEIMMpDgDWywsSXLtmRrl2b/3T9mRpZsyZJsyfPMM5/XQcea0TMzv4fH56Ovf89vMeccIiLiXYFSN0BERE5PQS0i4nEKahERj1NQi4h4nIJaRMTjQgvxps3NzW7FihUL8dYiIr60bdu2Xudcy1Q/W5CgXrFiBR0dHQvx1iIivmRmL0/3M3V9iIh4nIJaRMTjFNQiIh63IH3UIiKlkE6n6ezsJJFIlLop04rFYixbtoxwODzr1yioRcQ3Ojs7icfjrFixAjMrdXNO4Zzj2LFjdHZ2snLlylm/Tl0fIuIbiUSCpqYmT4Y0gJnR1NQ054pfQS0ivuLVkC46k/aVRVBve/k4u7oHS90MEZGSKIug/rP/3MHnHnqh1M0QEZmVBx98kLVr17Jq1So+/elPn/X7zepmopntB4aALJBxzm0660+eg7F0ltFU5lx+pIjIGclms9x222089NBDLFu2jMsvv5ybbrqJdevWnfF7zqWifpNzbuO5DmmAZDpLMp071x8rIjJnTz75JKtWreKCCy4gEolw6623smXLlrN6z7IYnpfI5EhksqVuhoiUkY9/bwc7D83vva11bXX8xdvXn/aYrq4uli9fPv542bJlbN269aw+d7YVtQN+YGbbzGzzVAeY2WYz6zCzjp6enrNq1MmS6SwJVdQiUqFmW1G/1jnXZWaLgYfMbLdz7vGJBzjn7gTuBNi0adO87pibzORIqqIWkTmYqfJdKO3t7Rw8eHD8cWdnJ+3t7Wf1nrOqqJ1zXYU/jwL3Aq8+q0+dg0w2RybnVFGLSFm4/PLL2bNnD/v27SOVSnHPPfdw0003ndV7zhjUZlZjZvHi98B1wPaz+tQ5SGbyAZ1Iq6IWEe8LhUJ86Utf4vrrr+eiiy7illtuYf36s6vuZ9P1sQS4tzCbJgR82zn34Fl96hwUg7r4p4iI1914443ceOON8/Z+Mwa1c+4lYMO8feIcFfumU5kcuZwjEPD29FARkfnm+ZmJE/umU1lV1SJSeTwf1BNHe6ifWkRm4ty8Djqbd2fSPu8H9YSKWiM/ROR0YrEYx44d82xYF9ejjsVic3qd52cmTryJqLHUInI6y5Yto7Ozk/medDefiju8zIXng3pid4cqahE5nXA4PKedU8qF97s+MhO7PlRRi0jlKYOgzk74XhW1iFQe7wd1WhW1iFQ2zwd1QhW1iFQ4zwe1KmoRqXTeD2rdTBSRClcGQa2uDxGpbJ4P6oS6PkSkwnk+qJOZLJFQoPC9KmoRqTxlENQ56mIhzPJ7J4qIVBrvB3U6RzQUJBoKkFBFLSIVyPNBnchkiYYDRENBVdQiUpE8H9TJdI5YKEgsHNCiTCJSkbwf1IWKOhYOTpqlKCJSKcogqHNEQwGiocCkWYoiIpXC+0GdzhINBVVRi0jF8n5QZ3LEwqqoRaRylUVQq6IWkUrm/aBOZwt91EGN+hCRiuT5oE5kcvlx1OGANrcVkYrk+c1tk+lsfhx1KKc+ahGpSJ6vqJOFijo/4UUVtYhUHk9X1JlsjkzOFdb6yGn1PBGpSJ6uqIvBHA2pohaRylUWQR0LB4mGgmRyjkxWVbWIVJZZB7WZBc3sGTO7fyEbNFFxlEexos4/p6AWkcoyl4r6I8CuhWrIVIqjPIqLMoG24xKRyjOroDazZcCvAF9d2OZMlhivqPMbB4AqahGpPLOtqP8e+CNg2pQ0s81m1mFmHT09PfPSuGJFHVNFLSIVbMagNrO3AUedc9tOd5xz7k7n3Cbn3KaWlpZ5adyJUR/B8T5qTSMXkUozm4r6auAmM9sP3AO82cy+taCtKph4MzEaCk56TkSkUswY1M65251zy5xzK4BbgUecc+9b8JZxonqOhoJEVVGLSIXy9MzEYvUcCwdIZVVRi0hlmlNQO+ceAx5bkJZMIZme2Eed/14VtYhUGo9X1CfGUadzqqhFpDJ5OqiLQ/GioQDpbGEctSpqEakwng7qiWt9ZHIOQNtxiUjF8fiiTPlQjgRPTHhRRS0ilcbjQZ0jEgwQCNj4FHLNTBSRSuPpoE6ks+Pjp8PBAMGAqetDRCqOp4M6mcmNz0iE/E1FdX2ISKXxdlCnc+NdHpC/qaiKWkQqjaeDOpE50fUBEAsFNOFFRCqOp4M6mc4Rm9j1EQ5qPWoRqTjeDuqTKupoaHYb3GayOQbG0gvZNBGRc8bjQX1qH/VsKuqvP/Ey1/zdYzjnFrJ5IiLnhLeDOp09ZdTHbCrqfb3D9A6nGNOYaxHxAW8HdSY3vrMLFCrqWYRv32i+22M4kVmwtomInCueD+pTxlHPouujfzQFwFBSQS0i5c/bQZ3OnjqOejYV9Ui+oh5SRS0iPuDpoE5kcpPHUYdnN466WFGr60NE/MDTQZ1MZyePow4FZ7VxwHgfdVJD9ESk/Hk7qM+gok6ks+OjPQZVUYuID3gmqHM5x0fveYbvPt0J5CetZHLupJuJ+Yr6dOOj+0dPVNHq+hARP/BMUAcCxuN7enlqfx8wYb/E0OSKOucgnZ0+qPsK/dMAwxr1ISI+4JmgBmhriHGofwyYvA1XUfH7062gNzGohxLqoxaR8uetoK6vonugGNQnNrYtKn5/ujWpJ3V9qKIWER/wVlA3VNHVN4ZzbjyMJy3KVKyoTzOWulhRV0eCGkctIr7gqaBub6hiJJVlMJEZ796YeDNxfIPb08xOLFbU5y2qVlCLiC94KqhbG2IAHOofG6+oY+FTuz6KFfUTLx5jy7Ndk96jbyRFVThIc21UXR8i4gueCuq2hiqgENTjoz6mqqjzQX3Hwy/wqQd2T3qPvtE0jdVhaqMhDc8TEV/wVFC3F4N6IDHjzUTnHDsPDXJ0KEEme6IrpH80RUN1hNpYSKM+RMQXPBXULbVRwkHjUP/Y+AzEqSrqRCZLV/8Yg4kMOQdHhpLjx/SNpmisCROPhbR6noj4gqeCOhAwltTFCl0f+Yo6dtIUcoBEOsfOQ4PjzxfHXkP+ZmJDdYR4NMRwMqNdXkSk7M0Y1GYWM7MnzewXZrbDzD6+kA1qa6iadDPx5CnkkO+j3tk9dVD3jabyfdSxEM7BSEq7vIhIeZtNRZ0E3uyc2wBsBG4wsysXqkHtDVUc6k+cuJl4mop6aV1+lEj3QALIrxcyMJamsTpCPBYGtN6HiJS/GYPa5Q0XHoYLXwvWn9DWEOPwYILRVD5gJy5zWvw+mc5X1JtWNBKPheguVNSDiTQ5R/5mYjQEaKlTESl/s+qjNrOgmT0LHAUecs5tneKYzWbWYWYdPT09Z9ygtoYqsjnHweOjwMkzE/PfHx1K0tk3xrq2Otrqq+jqz1fUxXWoi10foKVORaT8zSqonXNZ59xGYBnwajO7eIpj7nTObXLObWppaTnjBrXV54fovdQ7AkAkOHF4Xr6ifuZAPwDrWutoa4iNrw9SnD7eWB2hrhDU6voQkXI3p1Efzrl+4FHghoVpzolJL/t6R4gEAwQCNv6zYMAIB41fdBaCuq2O1oaq8T7q4hZcDdVhaqOFPmoN0RORMjebUR8tZtZQ+L4KuBbYffpXnbm2wjTyrv6xSd0eRbFQkNFUlubaKIvjMdrqYxwfSTGWyo5vattYmPACWupURMpfaBbHtAJfN7Mg+WD/jnPu/oVqUDxWmKySyEwamlcUDQcZSmZY11YHnKjAuwfGJnV9WCHjtTCTiJS7GYPaOfdL4NJz0JZxbfVVPJ8YmjR9vKj43LrWfFC31heDOkH/aJqAQTwWGh+Woq4PESl3npqZWFTs/piy66Pw3ImK+kRXSV9hnY9AwAgGjBqtSS0iPuDRoM5XybEpuj6K630UK+ql9YVJL/2JwvTx8Pix8VhYoz5EpOx5OqinqqijoQCxcICVzTWFx/m1p4t91I3VkfFja2MhdX2ISNnzZFAXlzudqo+6uTbKhmUNBCcM22triBW6PvJrURfVRkMMatSHiJS52Yz6OOdaC90ZU436+JtffSW5k1bEa62P8WLPCCPJDOsLfdeQv6moilpEyp0nK+rxPuopuj4aayI01UZPOb67f2x85byieEy7vIhI+fNkRb20PobZ1BX1VNrqq8aXM22Y2EcdDWnUh4iUPU9W1OFg/mZh80mV83SKm+ICk24mxmNhdX2ISNnzZEUN8K+br6I6MsuKutBVApxyM3E4mSGXc5PWDBERKSeerKgBWuJRaqKz+z1SXHEPJnd9xIsr6KVUVYtI+fJsUM9FSzxKqFAxN9ZMvpkIWupURMqbL4I6WNgUFyb3UWupUxHxA18ENZxY82PiFHItdSoifuCboG6tr6I6Epw0pC8+HtSqqEWkfHl21Mdc3bJpOWuW1E56Lj6+wa2CWkTKl2+C+rWrm3nt6uZJz9WqohYRH/BN18dU4rHCzUQFtYiUMV8HdXU4iJluJopIefN1UAcCRm0kxJD6qEWkjPk6qGHhVtB7sWeYgVFV6iKy8Hwf1LWx+V9BbyyV5eYv/ZQ/v2/7vL6viMhU/B/U0ak3D9hzZIgPfu0pfrDjMO6kjQhm8vDuIwwlM/xgxxFG1K0iIgvM90Edj4VP6aMeSWb4nW9t4+HdR9n8zW28/64nef7w0Kzf875nDxEJBhhLZ/nhriPz3WQRkUl8H9T5ro8TfcnOOf7k3ufY3zvCNz/4av7i7et4rmuAG+54nHd95Wd8+bG9vHBk+tAeGEvz2PM9/NoV59FaH+O+Zw+di9MQkQrm+6CORyffTLz7yYNsefYQH3vLGl63uoXfvHolj/3hG/nINatJZnJ85sHnue7zj/ONJ/ZP+X7f33GYVDbHzZe28/YNbTy+p4f+0dS5ORkRqUj+D+rCBreJdJa7nzzAX35vB69f08Jtb1o1fkxjTYSPvmUN3/s/r2Xrn1zDG9e28In7d7Gre/CU97vv2UOc31TNhmX13LShjXTW8d/bD5/LU5rWXPvaRaQ8+D6oa6NhRlNZXveZR7n9u89xUWsdn79lw7Q7viypi/F3795AQ3WYD9/9DGOFvRgBjg4l+NmLvbz9lW2YGevb6riguYYtz3adq9OZ1vauAS7/5A95/11b2Xno1F8wIlK+fLPWx3SWL8rv/nLh0jh3/M+NXPWKJsxOvy1XU22Uz92ykffdtZVPPrCTT9x8CQAP/LKbnIN3bGwDwMx4+4Y2vvDIHg4PJFhaH5vy/YaTGfYcGWIwkaGpJkJLPMqimgjh4OTfk845xtJZqiNzuyw7Dw3yvru2Eg0F+GXnAL/yxR/zzkvb+a2rV7K+rQ4zwznHk/uO842fv8zRwQSLaiIsqonQVl/Fxe31rG+rY3Hd1O0XkdKyhfjn8qZNm1xHR8e8v++ZyOYcfaOpWW+UO9GnHtjFPz7+Eq9esYi2hhjPHOynKhzkwY++fvyYF3uGuebvfsSbL1xMW0OMkWSWkWSGsXT+zyODSbr6x05571DAuKClhrVL61hUHWbX4SF2HRpkOJXh0uUNXLtuKa9b3Ty+HdloKsMzB/p5ct9xdnYPckl7PdevX8LS+ip+62tPEQ0FuGfzlTRURfjyY3v555/tJ5XJsaQuyutWt7Dz0CA7uwdpqA5z4dI4/aNpjo2k6BlKjrepKhykKhIkFgrQWBNh7dI4Fy2tY3FdlO6BBF19Y3QPJOgdTnJsJEkinWPDsnquWNnEZec3UhcLEQkFiIQCBM3Gf0F09Y+x/9gIB46NETCojoaoiQTJOUhmsiQzORZVR1jRXMPK5hrisRDpbI5M1lFXFSao/S6lApjZNufcpil/NlNQm9ly4BvAEsABdzrn7jjda7wU1Gcjlcnxqf/exY6uQboHxzg6mOTP3raO9115/qTj3n/XVra+dJyaaJCaaIiaSIjqaJDqSJDm2ihrlsRZvbiWxpoIx4ZTHBtJ0tk3xp4jQ+w+PETfSIo1S+Osb6ujoSrCj17o4bmugSnbtDgeZV1bHc8e7Ke/MDNySV2UezZfxcrmmvHjeoeTPLr7KI8938Pje3porY/xm1ev5OaN7VRN2DR4KJFmV/cQz3UNcKh/jGQmSyKdo2coye7DgxwZPBHkdbEQbQ1VtMSjNNVECASMp1/uY/+x0fn83z7J6sW1fOV9r2LV4tqZDxYpY2cb1K1Aq3PuaTOLA9uAm51zO6d7jV+CupS6B8bo2N9HNpe/PsGAcUl7Pec3VWNmZLI5ntx3nJ++2Mu7XrV8UkifzDk3Y3fPdI6PpOgdTtJaHxtfjfBkRwYTPNc5wFg6SzqbI5XJkXOQdQ6co7W+ihXNNSxfVEXAjNFklpFUhoAZ0VCAcCjAseEk+3pH2N87wmg6SyQYIOcc//ijl0iks3z23Rt46yWtZ3QOIuXgrIJ6ijfbAnzJOffQdMcoqGW+dA+M8bvfeppfHOznN16zgo9du4b6qql/YYiUs9MF9ZxGfZjZCuBSYOsUP9tsZh1m1tHT03Mm7RQ5RWt9Fd/5nSv5wFXn8/Un9vOmzz7GN5/YTyabK3XTRM6ZWVfUZlYL/Aj4pHPuu6c7VhW1LITtXQN84r928vOXjnP1qib+5bevLHWTRObNWVfUZhYG/gP4l5lCWmShXNxez93/60o+fM1qfrr3GC8fGyl1k0TOiRmD2vJ3oe4CdjnnPrfwTRKZnpnx7lctA+ChnVoQSyrDbCrqq4H3A282s2cLXzcucLtEprV8UTUXLo3zgx0KaqkMM06Bc879BNCMA/GU69Yt4UuP7uX4SIpFNZFSN0dkQfl+rQ/xp+vWLyXn4GGtBy4VQEEtZWl9Wx1t9TH1U0tFUFBLWTIz3rJuCT/e00sinZ35BSJlTEEtZevadUsYS2f5yZ7eUjdFZEEpqKVsXbGyiXg0pO4P8T0FtZStSCjAGy9czAPPdfPIboW1+JeCWsra71+7hvbGKn7rax38/neeZWA0PfOLRMqMglrK2srmGu770Gv58JtXseXZQ1z7+R/xoEf2sBSZLwpqKXuRUIDfv24tW267mubaKL/7rW383re2cXQwUeqmicwLBbX4xsXt9Wz50NX80Q1reXj3UW78wo8ZSWZK3SyRs6agFl8JBwP87zeu4jO/+kp6h1O8vIDbhImcKwpq8aXi7vM9w8kZjhTxPgW1+FJLbQxA/dTiCwpq8aWWeBRQRS3+oKAWX6qKBIlHQxwdVFBL+VNQi2+1xKOqqMUXFNTiWy3xKD1DCmopfwpq8S0FtfiFglp8S0EtfqGgFt9aHI8xnMwwmtLsRClvCmrxrfEheqqqpcwpqMW3FNTiFwpq8a3FCmrxCQW1+Faxoj6qoJYyp6AW31pUHSEYMFXUUvYU1OJbgYDRXBvh6JAWZpLypqAWX9NYavEDBbX4Wkut1vuQ8qegFl9bHI9pBT0pewpq8bWWeJRjIymyOVfqpoicsRmD2sz+ycyOmtn2c9EgkfnUEo+SzTn6RlOlborIGZtNRf014IYFbofIgihOelH3h5SzGYPaOfc4cPwctEVk3mlLLvGDeeujNrPNZtZhZh09PT3z9bYiZ0XrfYgfzFtQO+fudM5tcs5tamlpma+3FTkrJ6aRa9KLlC+N+hBfq46EqI2GVFFLWVNQi+8t1uxEKXOzGZ53N/AEsNbMOs3sgwvfLJH50xyPagU9KWuhmQ5wzr3nXDREZKG0xKPsOjRY6maInDF1fYjvLVZFLWVOQS2+1xKPapNbKWsKavG9ltr8EL0jmp0oZUpBLb63YXkDZvD1n+0vdVNEzoiCWnxvzZI477vifL7xxH62dw2Uujkic6aglorwh9evZVFNlD+99zkteSplR0EtFaG+Ksz//ZWL+EXnAN9+8kCpmyMyJwpqqRjv2NjGa17RxGce3E2vVtOTMqKglophZvzlTesZSmT47tOdpW6OyKwpqKWirFkSZ+PyBu595lCpmyIyawpqqTjvvLSdXd2D7D6saeVSHhTUUnHe9spWQgHj3me6St0UkVlRUEvFaaqN8oY1LWx55hA5DdWTMqCglop086XtHB5M8POXjpW6KSIzUlBLRbp23RJqoyF1f0hZUFBLRYqFg7z14qX89/bDJNLZUjdH5LQU1FKx3nlpO8PJDN/7hYbqibcpqKViXXlBE+vb6vjiI3tJZ3Olbo7ItBTUUrECAeMPrlvDgeOj/FuHZiqKdymopaK9ae1iLjuvgS8+skd91eJZCmqpaGbGH163lu6BBN/eqlX1xJsU1FLxXrOqmde8ookvP7ZX+yqKJymoRYA/uG4tvcMp/nzLDpzTbEXxFgW1CPCq8xv58DWr+fdtnXzyv3YprMVTQqVugIhXfOwtqxkcS/PVn+yjsSbCbW9aVeomiQAKapFxZsafv20dA2Np/vb7z2MGv/eGV2BmpW6aVDgFtcgEgYDxmXe9kmzO8ZkHn2fv0WH++p2XEAsHS900qWAKapGThIMB7rh1I6sW1/K5h15gf+8IX37vq1haHyt106RC6WaiyBTMjA9fs5qvvPcydnUP8Ya/fZS/fmAXx0dSpW6aVCBV1CKn8dZLWlnfVs/f//AF/t+PX+LbWw9w08Y2Xr+6hate0UR9VficteXAsVF+sreXJ146xvauAd69aZn60CuEzWYYkpndANwBBIGvOuc+fbrjN23a5Do6OuanhSIesefIEF98ZC8P7zrCSCpLMGBc1BrnkvZ6Lm6vZ82SOO0NVSypixEMzF94bu8a4AsP7+EHO48AsDgepb2ximcO9HPr5cv5q5svJhysrH8c942kGBhLc35T9Wl/UTnn6BtNc2QwwfGRFPVVYZbWx1hUHSEwj9doPpjZNufcpil/NlNQm1kQeAG4FugEngLe45zbOd1rFNTiZ+lsjmcO9PPjPT08faCP5zoHGEycmNEYDBiL41EaqyMsqonQWBOhLhaivipMPBamOhKkKhIkFg4SDth4qA8lMgyMpRlMpBlNZRlJZni5UEXXxUL8xtUrecfGNi5orgHg8w+9wBce2cvr17Twe294xaQ2moGRvzkaMAiYkXOQyeZIZx3pbI5kJkcqmyOTzZHNObI5R86Bw1GMBSu8NmAQCgQIBY1wMEA4GCAayn9VR0LURIPURkNEQ0Gi4QCRYGDOQeicI511ZHI5cg4CBoZxbCTJ9q4BtncNsuPQALu6hzg8mACgqSbCq1cu4rLzGllSH6O5JkIwYHS83MdP9/by9IE+EulTV0aMBANsPK+B161q5jWrmsk5R2ffKIf6EzTXRrikvYHVS2rn9Aswl3MkMzmqImd24/lsg/oq4C+dc9cXHt8O4Jz71HSvUVBLJXHOcfD4GC/1DnOoP8Gh/jEODyboG0lxfDRF/2iawbE0A2NpMrPco7EqHKQmGqQuFuZ/XNbOB16zgrrYqd0s33nqILff+xzZMtj7MRgwgmYUC2AzcA5y7sQviZlev6qllota41zUWkc8Fqbj5eNsfek4Xf1jpxx/UWsdV6xcxHmLqllaH6OxOsLAWIojg0kOHh/l5/uOsb1r+p3oo6EATTURgkEjHAiQyuYYSWYYSWZxOGLhIFWF0UAjyQyj6SyL41G2/slbzuj/z+mCejZ91O3AwQmPO4ErpviQzcBmgPPOO+8MmilSnsyM85qqOa+p+rTHOedIpHOMpjKMprIk0lmyzpHJ5hMqPqHqnm3XyS2XL+fVKxfRPZA48Tk4Cv/hHGSdI1coyCKFajgcNCKFijgUCORDNGAECkFq4+914j0y2RyZXL4aT2XyX8lMPrxGU1mGk5l8lZ7JkcxkmVgDOufIOkc2N7l9xYo9aPnPDweNUDBAYDzE8/9fLm6v58Kl8VOGSf7aFfms6R9N0TucpGcoRSKdZcPyBhbVRGb8/3dsOMlT+48TCwdZ1lhNe0MVhwcT/LKznx2HBukbSZHJOVLZHJFggJpokJpoiIAZiXT+GjoHNdEQNdEQjdULc89iNhX1u4AbnHO/XXj8fuAK59yHpnuNKmoRkbk5XUU9mw6YLmD5hMfLCs+JiMg5MJugfgpYbWYrzSwC3Arct7DNEhGRohn7qJ1zGTP7EPB98sPz/sk5t2PBWyYiIsAsJ7w45x4AHljgtoiIyBQqa5S8iEgZUlCLiHicglpExOMU1CIiHjerRZnm/KZmPcDLZ/jyZqB3HptTDirxnKEyz7sSzxkq87znes7nO+dapvrBggT12TCzjulm5/hVJZ4zVOZ5V+I5Q2We93yes7o+REQ8TkEtIuJxXgzqO0vdgBKoxHOGyjzvSjxnqMzznrdz9lwftYiITObFilpERCZQUIuIeJxngtrMbjCz581sr5n9canbs1DMbLmZPWpmO81sh5l9pPD8IjN7yMz2FP5sLHVb55uZBc3sGTO7v/B4pZltLVzzfy0so+srZtZgZv9uZrvNbJeZXeX3a21mHyv83d5uZnebWcyP19rM/snMjprZ9gnPTXltLe8LhfP/pZldNpfP8kRQFzbQ/QfgrcA64D1mtq60rVowGeAPnHPrgCuB2wrn+sfAw8651cDDhcd+8xFg14THfwN83jm3CugDPliSVi2sO4AHnXMXAhvIn79vr7WZtQMfBjY55y4mvzTyrfjzWn8NuOGk56a7tm8FVhe+NgNfmdMnOedK/gVcBXx/wuPbgdtL3a5zdO5byO/w/jzQWniuFXi+1G2b5/NcVviL+2bgfvLb8vUCoan+DvjhC6gH9lG4aT/hed9ea07ssbqI/DLK9wPX+/VaAyuA7TNdW+AfgfdMddxsvjxRUTP1BrrtJWrLOWNmK4BLga3AEudcd+FHh4ElJWrWQvl74I+AXOFxE9DvnMsUHvvxmq8EeoB/LnT5fNXMavDxtXbOdQGfBQ4A3cAAsA3/X+ui6a7tWWWcV4K64phZLfAfwEedc5P2rHf5X7m+GTdpZm8DjjrntpW6LedYCLgM+Hb7d+QAAAGOSURBVIpz7lJghJO6OXx4rRuBd5D/JdUG1HBq90BFmM9r65WgrqgNdM0sTD6k/8U5993C00fMrLXw81bgaKnatwCuBm4ys/3APeS7P+4AGsysuMuQH695J9DpnNtaePzv5IPbz9f6LcA+51yPcy4NfJf89ff7tS6a7tqeVcZ5JagrZgNdMzPgLmCXc+5zE350H/Drhe9/nXzftS845253zi1zzq0gf20fcc69F3gUeFfhMF+dM4Bz7jBw0MzWFp66BtiJj681+S6PK82suvB3vXjOvr7WE0x3be8DPlAY/XElMDChi2Rmpe6Mn9C5fiPwAvAi8Kelbs8Cnudryf9z6JfAs4WvG8n32T4M7AF+CCwqdVsX6PzfCNxf+P4C4ElgL/BvQLTU7VuA890IdBSu938CjX6/1sDHgd3AduCbQNSP1xq4m3w/fJr8v54+ON21JX/z/B8K+fYc+VExs/4sTSEXEfE4r3R9iIjINBTUIiIep6AWEfE4BbWIiMcpqEVEPE5BLSLicQpqERGP+/9NI9qzd1LGKQAAAABJRU5ErkJggg==\n", 381 | "text/plain": [ 382 | "
" 383 | ] 384 | }, 385 | "metadata": { 386 | "needs_background": "light" 387 | } 388 | } 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "metadata": { 394 | "id": "gIaPMvXe2wrS" 395 | }, 396 | "source": [ 397 | "" 398 | ], 399 | "execution_count": null, 400 | "outputs": [] 401 | } 402 | ] 403 | } -------------------------------------------------------------------------------- /nlp/at.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class FGM(): 5 | def __init__(self, model): 6 | self.model = model 7 | self.backup = {} 8 | 9 | def attack(self, epsilon=1., emb_name='emb'): 10 | for name, param in self.model.named_parameters(): 11 | if param.requires_grad and emb_name in name: 12 | self.backup[name] = param.data.clone() 13 | norm = torch.norm(param.grad) 14 | if norm != 0: 15 | r_at = epsilon * param.grad / norm 16 | param.data.add_(r_at) 17 | 18 | def restore(self, emb_name='emb'): 19 | for name, param in self.model.named_parameters(): 20 | if param.requires_grad and emb_name in name: 21 | assert name in self.backup 22 | param.data = self.backup[name] 23 | self.backup = {} 24 | 25 | 26 | # 初始化 27 | fgm = FGM(model) 28 | for batch_input, batch_label in data: 29 | # 正常训练 30 | loss = model(batch_input, batch_label) 31 | loss.backward() 32 | # 对抗训练 33 | fgm.attack() # 修改embedding 34 | # optimizer.zero_grad() # 梯度累加，不累加去掉注释 35 | loss_sum = model(batch_input, batch_label) 36 | loss_sum.backward() # 累加对抗训练的梯度 37 | fgm.restore() # 恢复Embedding的参数 38 | 39 | optimizer.step() 40 | optimizer.zero_grad() 41 | -------------------------------------------------------------------------------- /nlp/bert_blend_cnn.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | # bert融合textcnn思想的Bert+Blend-CNN 3 | # model: Bert+Blend-CNN 4 | # date: 2021.10.11 18:06:11 5 | 6 | import os 7 | import numpy as np 8 | import pandas as pd 9 | import torch 10 | import torch.nn as nn 11 | import torch.utils.data as Data 12 | import torch.nn.functional as F 13 | import torch.optim as optim 14 | import transformers 15 | from transformers import AutoModel, AutoTokenizer 16 | import matplotlib.pyplot as plt 17 | 18 | train_curve = [] 19 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 20 | 21 | # # 定义一些参数，模型选择了最基础的bert中文模型 22 | batch_size = 2 23 | epoches = 100 24 | model = "bert-base-chinese" 25 | hidden_size = 768 26 | n_class = 2 27 | maxlen = 8 28 | 29 | encode_layer=12 30 | filter_sizes = [2, 2, 2] 31 | num_filters = 3 32 | 33 | # data，构造一些训练数据 34 | sentences = ["我喜欢打篮球", "这个相机很好看", "今天玩的特别开心", "我不喜欢你", "太糟糕了", "真是件令人伤心的事情"] 35 | labels = [1, 1, 1, 0, 0, 0] # 1积极, 0消极. 36 | 37 | class MyDataset(Data.Dataset): 38 | def __init__(self, sentences, labels=None, with_labels=True,): 39 | self.tokenizer = AutoTokenizer.from_pretrained(model) 40 | self.with_labels = with_labels 41 | self.sentences = sentences 42 | self.labels = labels 43 | def __len__(self): 44 | return len(sentences) 45 | 46 | def __getitem__(self, index): 47 | # Selecting sentence1 and sentence2 at the specified index in the data frame 48 | sent = self.sentences[index] 49 | 50 | # Tokenize the pair of sentences to get token ids, attention masks and token type ids 51 | encoded_pair = self.tokenizer(sent, 52 | padding='max_length', # Pad to max_length 53 | truncation=True, # Truncate to max_length 54 | max_length=maxlen, 55 | return_tensors='pt') # Return torch.Tensor objects 56 | 57 | token_ids = encoded_pair['input_ids'].squeeze(0) # tensor of token ids 58 | attn_masks = encoded_pair['attention_mask'].squeeze(0) # binary tensor with "0" for padded values and "1" for the other values 59 | token_type_ids = encoded_pair['token_type_ids'].squeeze(0) # binary tensor with "0" for the 1st sentence tokens & "1" for the 2nd sentence tokens 60 | 61 | if self.with_labels: # True if the dataset has labels 62 | label = self.labels[index] 63 | return token_ids, attn_masks, token_type_ids, label 64 | else: 65 | return token_ids, attn_masks, token_type_ids 66 | 67 | train = Data.DataLoader(dataset=MyDataset(sentences, labels), batch_size=batch_size, shuffle=True, num_workers=1) 68 | 69 | class TextCNN(nn.Module): 70 | def __init__(self): 71 | super(TextCNN, self).__init__() 72 | self.num_filter_total = num_filters * len(filter_sizes) 73 | self.Weight = nn.Linear(self.num_filter_total, n_class, bias=False) 74 | self.bias = nn.Parameter(torch.ones([n_class])) 75 | self.filter_list = nn.ModuleList([ 76 | nn.Conv2d(1, num_filters, kernel_size=(size, hidden_size)) for size in filter_sizes 77 | ]) 78 | 79 | def forward(self, x): 80 | # x: [bs, seq, hidden] 81 | x = x.unsqueeze(1) # [bs, channel=1, seq, hidden] 82 | 83 | pooled_outputs = [] 84 | for i, conv in enumerate(self.filter_list): 85 | h = F.relu(conv(x)) # [bs, channel=1, seq-kernel_size+1, 1] 86 | mp = nn.MaxPool2d( 87 | kernel_size = (encode_layer-filter_sizes[i]+1, 1) 88 | ) 89 | # mp: [bs, channel=3, w, h] 90 | pooled = mp(h).permute(0, 3, 2, 1) # [bs, h=1, w=1, channel=3] 91 | pooled_outputs.append(pooled) 92 | 93 | h_pool = torch.cat(pooled_outputs, len(filter_sizes)) # [bs, h=1, w=1, channel=3 * 3] 94 | h_pool_flat = torch.reshape(h_pool, [-1, self.num_filter_total]) 95 | 96 | output = self.Weight(h_pool_flat) + self.bias # [bs, n_class] 97 | 98 | return output 99 | 100 | # model 101 | class Bert_Blend_CNN(nn.Module): 102 | def __init__(self): 103 | super(Bert_Blend_CNN, self).__init__() 104 | self.bert = AutoModel.from_pretrained(model, output_hidden_states=True, return_dict=True) 105 | self.linear = nn.Linear(hidden_size, n_class) 106 | self.textcnn = TextCNN() 107 | 108 | def forward(self, X): 109 | input_ids, attention_mask, token_type_ids = X[0], X[1], X[2] 110 | outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids) # 返回一个output字典 111 | # 取每一层encode出来的向量 112 | # outputs.pooler_output: [bs, hidden_size] 113 | hidden_states = outputs.hidden_states # 13*[bs, seq_len, hidden] 第一层是embedding层不需要 114 | cls_embeddings = hidden_states[1][:, 0, :].unsqueeze(1) # [bs, 1, hidden] 115 | # 将每一层的第一个token(cls向量)提取出来，拼在一起当作textcnn的输入 116 | for i in range(2, 13): 117 | cls_embeddings = torch.cat((cls_embeddings, hidden_states[i][:, 0, :].unsqueeze(1)), dim=1) 118 | # cls_embeddings: [bs, encode_layer=12, hidden] 119 | logits = self.textcnn(cls_embeddings) 120 | return logits 121 | 122 | bert_blend_cnn = Bert_Blend_CNN().to(device) 123 | 124 | optimizer = optim.Adam(bert_blend_cnn.parameters(), lr=1e-3, weight_decay=1e-2) 125 | loss_fn = nn.CrossEntropyLoss() 126 | 127 | # train 128 | sum_loss = 0 129 | total_step = len(train) 130 | for epoch in range(epoches): 131 | for i, batch in enumerate(train): 132 | optimizer.zero_grad() 133 | batch = tuple(p.to(device) for p in batch) 134 | pred = bert_blend_cnn([batch[0], batch[1], batch[2]]) 135 | loss = loss_fn(pred, batch[3]) 136 | sum_loss += loss.item() 137 | 138 | loss.backward() 139 | optimizer.step() 140 | if epoch % 10 == 0: 141 | print('[{}|{}] step:{}/{} loss:{:.4f}'.format(epoch+1, epoches, i+1, total_step, loss.item())) 142 | train_curve.append(sum_loss) 143 | sum_loss = 0 144 | 145 | # test 146 | bert_blend_cnn.eval() 147 | with torch.no_grad(): 148 | test_text = ['我不喜欢打篮球'] 149 | test = MyDataset(test_text, labels=None, with_labels=False) 150 | x = test.__getitem__(0) 151 | x = tuple(p.unsqueeze(0).to(device) for p in x) 152 | pred = bert_blend_cnn([x[0], x[1], x[2]]) 153 | pred = pred.data.max(dim=1, keepdim=True)[1] 154 | if pred[0][0] == 0: 155 | print('消极') 156 | else: 157 | print('积极') 158 | 159 | pd.DataFrame(train_curve).plot() # loss曲线 160 | 161 | -------------------------------------------------------------------------------- /nlp/bert_classify.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "bert_classify.ipynb", 7 | "provenance": [] 8 | }, 9 | "kernelspec": { 10 | "name": "python3", 11 | "display_name": "Python 3" 12 | }, 13 | "language_info": { 14 | "name": "python" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "code", 21 | "metadata": { 22 | "id": "c3EUFvW1YZ23" 23 | }, 24 | "source": [ 25 | "# -*- coding:utf-8 -*-\n", 26 | "# bert文本分类baseline模型\n", 27 | "# model: bert\n", 28 | "# date: 2021.10.10 10:01\n", 29 | "\n", 30 | "import os\n", 31 | "import numpy as np\n", 32 | "import pandas as pd\n", 33 | "import torch\n", 34 | "import torch.nn as nn\n", 35 | "import torch.utils.data as Data\n", 36 | "import torch.optim as optim\n", 37 | "import transformers\n", 38 | "from transformers import AutoModel, AutoTokenizer\n", 39 | "import matplotlib.pyplot as plt" 40 | ], 41 | "execution_count": 38, 42 | "outputs": [] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "metadata": { 47 | "id": "honSDblyKbxc" 48 | }, 49 | "source": [ 50 | "train_curve = []\n", 51 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" 52 | ], 53 | "execution_count": 39, 54 | "outputs": [] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "metadata": { 59 | "id": "hNAVAq8o7Xbs" 60 | }, 61 | "source": [ 62 | "batch_size = 2\n", 63 | "epoches = 100\n", 64 | "model = \"bert-base-chinese\"\n", 65 | "hidden_size = 768\n", 66 | "n_class = 2\n", 67 | "maxlen = 8" 68 | ], 69 | "execution_count": 40, 70 | "outputs": [] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "metadata": { 75 | "id": "wiq4EFV9LKO1" 76 | }, 77 | "source": [ 78 | "# data\n", 79 | "sentences = [\"我喜欢打篮球\", \"这个相机很好看\", \"今天玩的特别开心\", \"我不喜欢你\", \"太糟糕了\", \"真是件令人伤心的事情\"]\n", 80 | "labels = [1, 1, 1, 0, 0, 0] # 1积极, 0消极.\n", 81 | "\n", 82 | "# word_list = ' '.join(sentences).split()\n", 83 | "# word_list = list(set(word_list))\n", 84 | "# word_dict = {w: i for i, w in enumerate(word_list)}\n", 85 | "# num_dict = {i: w for w, i in word_dict.items()}\n", 86 | "# vocab_size = len(word_list)" 87 | ], 88 | "execution_count": 41, 89 | "outputs": [] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "metadata": { 94 | "id": "xG31wox1EKVX" 95 | }, 96 | "source": [ 97 | "class MyDataset(Data.Dataset):\n", 98 | " def __init__(self, sentences, labels=None, with_labels=True,):\n", 99 | " self.tokenizer = AutoTokenizer.from_pretrained(model)\n", 100 | " self.with_labels = with_labels\n", 101 | " self.sentences = sentences\n", 102 | " self.labels = labels\n", 103 | " def __len__(self):\n", 104 | " return len(sentences)\n", 105 | "\n", 106 | " def __getitem__(self, index):\n", 107 | " # Selecting sentence1 and sentence2 at the specified index in the data frame\n", 108 | " sent = self.sentences[index]\n", 109 | "\n", 110 | " # Tokenize the pair of sentences to get token ids, attention masks and token type ids\n", 111 | " encoded_pair = self.tokenizer(sent,\n", 112 | " padding='max_length', # Pad to max_length\n", 113 | " truncation=True, # Truncate to max_length\n", 114 | " max_length=maxlen, \n", 115 | " return_tensors='pt') # Return torch.Tensor objects\n", 116 | "\n", 117 | " token_ids = encoded_pair['input_ids'].squeeze(0) # tensor of token ids\n", 118 | " attn_masks = encoded_pair['attention_mask'].squeeze(0) # binary tensor with \"0\" for padded values and \"1\" for the other values\n", 119 | " token_type_ids = encoded_pair['token_type_ids'].squeeze(0) # binary tensor with \"0\" for the 1st sentence tokens & \"1\" for the 2nd sentence tokens\n", 120 | "\n", 121 | " if self.with_labels: # True if the dataset has labels\n", 122 | " label = self.labels[index]\n", 123 | " return token_ids, attn_masks, token_type_ids, label\n", 124 | " else:\n", 125 | " return token_ids, attn_masks, token_type_ids" 126 | ], 127 | "execution_count": 42, 128 | "outputs": [] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "metadata": { 133 | "id": "nMRNgixZLXLj" 134 | }, 135 | "source": [ 136 | "train = Data.DataLoader(dataset=MyDataset(sentences, labels), batch_size=batch_size, shuffle=True, num_workers=1)" 137 | ], 138 | "execution_count": 43, 139 | "outputs": [] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "metadata": { 144 | "id": "u25BycPb8IhR" 145 | }, 146 | "source": [ 147 | "# model\n", 148 | "class BertClassify(nn.Module):\n", 149 | " def __init__(self):\n", 150 | " super(BertClassify, self).__init__()\n", 151 | " self.bert = AutoModel.from_pretrained(model, output_hidden_states=True, return_dict=True)\n", 152 | " self.linear = nn.Linear(hidden_size, n_class)\n", 153 | " self.dropout = nn.Dropout(0.5)\n", 154 | " \n", 155 | " def forward(self, X):\n", 156 | " input_ids, attention_mask, token_type_ids = X[0], X[1], X[2]\n", 157 | " outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids) # 返回一个output字典\n", 158 | " # 用最后一层cls向量做分类\n", 159 | " # outputs.pooler_output: [bs, hidden_size]\n", 160 | " logits = self.linear(self.dropout(outputs.pooler_output))\n", 161 | " \n", 162 | " return logits" 163 | ], 164 | "execution_count": 44, 165 | "outputs": [] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "metadata": { 170 | "colab": { 171 | "base_uri": "https://localhost:8080/" 172 | }, 173 | "id": "DWmpHgE38dPM", 174 | "outputId": "2a0d44a5-0d89-4478-8f33-d0e326aeb1d8" 175 | }, 176 | "source": [ 177 | "bc = BertClassify().to(device)" 178 | ], 179 | "execution_count": 45, 180 | "outputs": [ 181 | { 182 | "output_type": "stream", 183 | "name": "stderr", 184 | "text": [ 185 | "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertModel: ['cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight']\n", 186 | "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", 187 | "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" 188 | ] 189 | } 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "metadata": { 195 | "id": "oKn15V6oKG2B" 196 | }, 197 | "source": [ 198 | "optimizer = optim.Adam(bc.parameters(), lr=1e-3, weight_decay=1e-2)\n", 199 | "loss_fn = nn.CrossEntropyLoss()" 200 | ], 201 | "execution_count": 46, 202 | "outputs": [] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "metadata": { 207 | "colab": { 208 | "base_uri": "https://localhost:8080/" 209 | }, 210 | "id": "sFFxX8t2C4G0", 211 | "outputId": "38cff6f7-c1a9-48fb-ad18-dd28626382ca" 212 | }, 213 | "source": [ 214 | "# train\n", 215 | "sum_loss = 0\n", 216 | "total_step = len(train)\n", 217 | "for epoch in range(epoches):\n", 218 | " for i, batch in enumerate(train):\n", 219 | " optimizer.zero_grad()\n", 220 | " batch = tuple(p.to(device) for p in batch)\n", 221 | " pred = bc([batch[0], batch[1], batch[2]])\n", 222 | " loss = loss_fn(pred, batch[3])\n", 223 | " sum_loss += loss.item()\n", 224 | "\n", 225 | " loss.backward()\n", 226 | " optimizer.step()\n", 227 | " if epoch % 10 == 0:\n", 228 | " print('[{}|{}] step:{}/{} loss:{:.4f}'.format(epoch+1, epoches, i+1, total_step, loss.item()))\n", 229 | " train_curve.append(sum_loss)\n", 230 | " sum_loss = 0" 231 | ], 232 | "execution_count": 47, 233 | "outputs": [ 234 | { 235 | "output_type": "stream", 236 | "name": "stdout", 237 | "text": [ 238 | "[1|100] step:1/3 loss:1.1500\n", 239 | "[1|100] step:2/3 loss:1.2046\n", 240 | "[1|100] step:3/3 loss:2.5194\n", 241 | "[11|100] step:1/3 loss:1.1471\n", 242 | "[11|100] step:2/3 loss:0.7017\n", 243 | "[11|100] step:3/3 loss:0.8367\n", 244 | "[21|100] step:1/3 loss:1.0529\n", 245 | "[21|100] step:2/3 loss:1.4519\n", 246 | "[21|100] step:3/3 loss:1.4188\n", 247 | "[31|100] step:1/3 loss:1.2525\n", 248 | "[31|100] step:2/3 loss:1.2894\n", 249 | "[31|100] step:3/3 loss:0.6796\n", 250 | "[41|100] step:1/3 loss:1.2206\n", 251 | "[41|100] step:2/3 loss:0.7903\n", 252 | "[41|100] step:3/3 loss:0.9456\n", 253 | "[51|100] step:1/3 loss:0.4095\n", 254 | "[51|100] step:2/3 loss:0.4227\n", 255 | "[51|100] step:3/3 loss:1.7919\n", 256 | "[61|100] step:1/3 loss:2.2079\n", 257 | "[61|100] step:2/3 loss:1.0230\n", 258 | "[61|100] step:3/3 loss:1.0512\n", 259 | "[71|100] step:1/3 loss:0.5453\n", 260 | "[71|100] step:2/3 loss:0.2571\n", 261 | "[71|100] step:3/3 loss:1.0071\n", 262 | "[81|100] step:1/3 loss:1.1741\n", 263 | "[81|100] step:2/3 loss:0.6857\n", 264 | "[81|100] step:3/3 loss:0.2532\n", 265 | "[91|100] step:1/3 loss:0.5687\n", 266 | "[91|100] step:2/3 loss:0.4335\n", 267 | "[91|100] step:3/3 loss:0.8573\n" 268 | ] 269 | } 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "metadata": { 275 | "colab": { 276 | "base_uri": "https://localhost:8080/" 277 | }, 278 | "id": "y03sGa73O-Xb", 279 | "outputId": "b2b0f256-c55e-42c7-b9e1-121823ba5f67" 280 | }, 281 | "source": [ 282 | "# test\n", 283 | "bc.eval()\n", 284 | "with torch.no_grad():\n", 285 | " test_text = ['我不喜欢打篮球']\n", 286 | " test = MyDataset(test_text, labels=None, with_labels=False)\n", 287 | " x = test.__getitem__(0)\n", 288 | " x = tuple(p.unsqueeze(0).to(device) for p in x)\n", 289 | " pred = bc([x[0], x[1], x[2]])\n", 290 | " pred = pred.data.max(dim=1, keepdim=True)[1]\n", 291 | " if pred[0][0] == 0:\n", 292 | " print('消极')\n", 293 | " else:\n", 294 | " print('积极')" 295 | ], 296 | "execution_count": 52, 297 | "outputs": [ 298 | { 299 | "output_type": "stream", 300 | "name": "stdout", 301 | "text": [ 302 | "消极\n" 303 | ] 304 | } 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "metadata": { 310 | "colab": { 311 | "base_uri": "https://localhost:8080/", 312 | "height": 285 313 | }, 314 | "id": "u_d5OUCPSXgH", 315 | "outputId": "d266ed38-b7fa-4590-8640-c828d255ece6" 316 | }, 317 | "source": [ 318 | "pd.DataFrame(train_curve).plot() # loss曲线" 319 | ], 320 | "execution_count": 51, 321 | "outputs": [ 322 | { 323 | "output_type": "execute_result", 324 | "data": { 325 | "text/plain": [ 326 | "" 327 | ] 328 | }, 329 | "metadata": {}, 330 | "execution_count": 51 331 | }, 332 | { 333 | "output_type": "display_data", 334 | "data": { 335 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD6CAYAAACxrrxPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9ebRkV102/Owz1XTnsTt9O93p7pDOHEKDgYDG5EVCgKACwqei6It8+ILix+tScL2i8L0udamIgsobQT9YyKDMIEEDCciY0IGQqTN2hr6d7r63b9+pxjPU/v44Z5+zz1jn1K2qW1V3P2tl5fa9Neyqc87vPPv5Pb/fj1BKISAgICAw+JC2ewECAgICAp2BCOgCAgICQwIR0AUEBASGBCKgCwgICAwJREAXEBAQGBKIgC4gICAwJEgV0AkhTxJC7iOE3EMIORrxd0II+VtCyGOEkHsJIVd3fqkCAgICAklQMjz2pymlZ2P+9lIAFzr//QSAf3D+H4uZmRm6f//+DG8vICAgIHD33XefpZTORv0tS0BPwisBfJTaVUrfJ4RMEEJ2U0pPxT1h//79OHo0RPYFBAQEBBJACHkq7m9pNXQK4D8JIXcTQt4U8fc9AE5w/150ficgICAg0COkZegvpJSeJITMAbiNEPIQpfS/sr6ZczN4EwCcf/75WZ8uICAgIJCAVAydUnrS+f8SgM8BeF7gIScB7OX+veD8Lvg6t1BKj1BKj8zORkpAAgICAgJtoiVDJ4SUAEiU0k3n558B8J7Aw74I4K2EkE/CToauJ+nnAgICAtsJwzCwuLiIer2+3UuJRT6fx8LCAlRVTf2cNJLLPIDPEULY4z9OKf0qIeTNAEAp/SCArwC4CcBjAKoAfi3j2gUEBAR6hsXFRYyOjmL//v1wYltfgVKKlZUVLC4u4oILLkj9vJYBnVJ6HMCVEb//IPczBfCW1O8qICAgsI2o1+t9G8wBgBCC6elpLC8vZ3qeqBQVEBDYkejXYM7QzvoGLqA/fHoTf/WfD2Ol3NjupQgICAj0FQYuoD++XMb7b38MyyKgCwgIDDC++tWv4qKLLsKhQ4fwZ3/2Zx15zYEL6JpsL1k3m9u8EgEBAYH2YFkW3vKWt+DWW2/Fgw8+iE984hN48MEHt/y6gxfQFRHQBQQEBht33XUXDh06hAMHDkDTNLzuda/DF77whS2/bqd6ufQMqmDoAgICHcS7v/QAHnxmo6Ovecl5Y/ijV1wa+/eTJ09i716vFnNhYQF33nnnlt93cBm6JQK6gICAAI+BY+g5IbkICAh0EElMulvYs2cPTpzw+hkuLi5iz56t9zMUDF1AQECgx3juc5+LRx99FE888QR0XccnP/lJ3HzzzVt+3YFj6MLlIiAgMOhQFAUf+MAH8JKXvASWZeHXf/3XcemlW98pDF5AF5KLgIDAEOCmm27CTTfd1NHXHDjJxXW5CMlFQEBAwIeBC+iCoQsICAhEY+ACek4kRQUEBDoAu0ls/6Kd9Q1cQBdJUQEBga0in89jZWWlb4M664eez+czPW/gkqKSRKBIRAR0AQGBtrGwsIDFxcXM/cZ7CTaxKAsGLqADto4uArqAgEC7UFU10ySgQcHASS6AE9CFhi4gICDgw0AGdFUWDF1AQEAgiIEM6JosGLqAgIBAEAMZ0HNCQxcQEBAIIXVAJ4TIhJAfEUK+HPG3NxBClgkh9zj/vbGzy/RDJEUFBAQEwsjicnkbgGMAxmL+/ilK6Vu3vqTWEElRAQEBgTBSMXRCyAKAlwH4UHeXkw6aSIoKCAgIhJBWcnkfgN8DkBRFX0UIuZcQ8mlCyN6Ex20ZwuUiICAgEEbLgE4IeTmAJUrp3QkP+xKA/ZTSKwDcBuAjMa/1JkLIUULI0a1UaAnJRUBAQCCMNAz9WgA3E0KeBPBJANcTQj7GP4BSukIpbTj//BCA50S9EKX0FkrpEUrpkdnZ2bYXLZKiAgICAmG0DOiU0ndSShcopfsBvA7A7ZTSX+YfQwjZzf3zZtjJ065BMHQBAQGBMNru5UIIeQ+Ao5TSLwL4bULIzQBMAOcAvKEzy4tGTmjoAgICAiFkCuiU0m8A+Ibz87u4378TwDs7ubAkCMlFQEBAIIyBrBQVkouAgIBAGAMZ0IVtUUBAQCCMgQzoQnIREBAQCGMwA7oswWxSNJv9OT5KQEBAYDswmAFdDIoWEBAQCGEgA3pOBHQBAQGBEAYyoLsMXejoAgICAi4GM6DLIqALDC/+zzcfx20PntnuZQgMIAYyoKsioAsMMT7y3SfxuR8tbvcyBAYQAxnQmeRiCA1dYAihW02s14ztXobAAGKgA3pDMHSBIYRuNrFWFQFdIDsGOqALl4vAMEK3REAXaA8DGdBzQkMXGGIYFhWSi0BbGMiALmyLAsMKq0lhNSnKDVPkiAQyYyADunC5CAwr+CC+IVi6QEYMZEAXGrrAsIJP9K+JgC6QEQMd0MWWVGDYwJ/TIjEqkBWDGdBlYVsUGE7wMqKQXASyYiADek4kRQWGFD6GXtO3cSUCg4iBDOjC5SIwrODPaSG5CGTFYAd0oaELDBl0oaELbAEDGdCFbVFgWMGf06K4SCArUgd0QohMCPkRIeTLEX/LEUI+RQh5jBByJyFkfycXGYQiERAiArrA8MGwvLGKIqALZEUWhv42AMdi/vbfAaxSSg8B+GsAf77VhSWBEAJNloRtUWDo4NfQRVJUIBtSBXRCyAKAlwH4UMxDXgngI87PnwZwAyGEbH158dAUSdgWBYYOjKSM5hXB0AUyIy1Dfx+A3wMQF0H3ADgBAJRSE8A6gOnggwghbyKEHCWEHF1eXm5juR5yiiSSogJDB0ZSZkdzolJUIDNaBnRCyMsBLFFK797qm1FKb6GUHqGUHpmdnd3Sa2myJDR0gaEDY+hzozmsC5eLQEakYejXAriZEPIkgE8CuJ4Q8rHAY04C2AsAhBAFwDiAlQ6uMwRNEQFdYPiguww9j7WaAUppi2cICHhoGdAppe+klC5QSvcDeB2A2ymlvxx42BcB/Krz86udx3T1TFR3GEOnlIqLeweAMfTZkRysJkVFt7Z5RQKDhLZ96ISQ9xBCbnb++WEA04SQxwC8HcA7OrG4JGjKznK5vP7Dd+FPb31ou5ch0GWwvNDsaA6AcLoIZIOS5cGU0m8A+Ibz87u439cBvKaTC2sFbYclRZ86V0FRk7d7GQJdBtt1zrkB3cDC5HauSGCQMJCVooCdFN1JtkXDpDvq8+5UBBm6sC4KZMHgBvQdlhQ1rCbqhtBThx2GaedJZkZEQBfIjoEN6LkdFtB1qykY+g6AbllQJIKpkgZANOgSyIaBDeiqvLM0dEME9B0Bw6LQFAkTRRWA6IkukA0DG9B3nuRC0RCSy9BDN5tQZQl5VUZOkURxkUAmDG5A30HNuawmhdUUSdGdgIbZdPv9jxdUoaELZMLgBvQdxNDZjathpmfon7zrabzmg9/t1pIEugTDarozcyeKqtDQBTJBBPQBAMsV1I30n/f+Z9bx48X1bi1JoEvQOYY+UdCEhi6QCQMd0Bs7RHIxzOwMvapb0M2maBcwYDCsJlTZ7jw9Lhi6QEYMbEDPOb1cdkLAYlNsDMvW0tOAedbNlI8X6A/oAQ19Q2joAhkwsAGdzRXlR3YNK/jkb1qWXnWaOu2UxPGwQLea7rk9UVBFT3SBTBjYgM5YzE7wovOfsZFSR3cDujn8N7xhgm76k6JV3coktQnsbAx8QDe2KTFqWs2eWcr8DD3d5605AX0n3PCGCYbFSS5Fu1p0mKyLdcPC6fX6di9jaDHwAX27AtZHvvcUbvirb6DZA42aZ9lp+7lUddN+rgjoAwWdsy2OF+xq0WHS0T/87Sfw8vd/a7uX0XU0mxTvve0RLG309uY1uAHdOem3y7p44lwVZ8s6zvWgX7W+BYYuAvpgwTCpT0MHhqufy6n1Gs6W9Z4Qoe3E4moNf/v1R/GfD57p6fsObkB3GPp2VU9WGjYD7sX2sa2kqMPkd4pXf1igc5KL289liAJ6uW5fN8Ne9Vxzrr+yEyd6hcEN6NvM0CuOpHGmB1sqPqCnLS4SGvpggvVyAezCIgBD5XRhAW7YW0Gzz1cRAT0dtltDLzfsA3Zmo9H198rK0Pm+LzvB1jlM0H1JUZuhD1NSdNNh6PUhd+4whs4+b68w+AF9uyWXHjB0/jOmsS3WOPYjNPTBgt3Lxa4UHc0pIARYH6K5ooyhp7XfDirqQnLJBs0tLNregH6mBxq6zrHsNMyGOVyA7bN1CrQHvlJUkgjGh6y4yJVchpyhM2m0LBh6Omw3Q2dbqV4wdCMrQ9e9i0Vo6IMFg6sUBZxq0SFMimZpNDeIYNJoRe+zgE4IyRNC7iKE/JgQ8gAh5N0Rj3kDIWSZEHKP898bu7NcD9vuctmmpGiaz1vVeclFaOiDgmaTuhOLGMYKKjbqwxPQN3dIUpSRql5r6EqKxzQAXE8pLRNCVADfJoTcSin9fuBxn6KUvrXzS4xGbpuToq7k0nOXSxrJhWPoQnIZGBhN+1jxDD2vyn0d/B45s4miJmNhstjysQ3Tcs/Hfv5MnUDfaujURtn5p+r8t+20T91G22LDtGBYFEVNxmrV6PrJyWvoaRh6TRdJ0UEEO5dzSjCg9+8xfNsn78Gff/XhVI+tNLzzsp8/UydQN/tYQyeEyISQewAsAbiNUnpnxMNeRQi5lxDyaULI3pjXeRMh5Cgh5Ojy8vIWlr29Gjo7MQ/MlgAAS122Lma1LfIuF6GhDw6YPOZj6IrU12x2taKnbk3AB7dhbzjW1z50SqlFKb0KwAKA5xFCLgs85EsA9lNKrwBwG4CPxLzOLZTSI5TSI7Ozs1tZ97a6XNhBOjg7AqD7iVGWFJUlkorZ+FwuIqAPDBg50UIMvX+DX0U3U69vs+EF/mG3LbqVorrZ0zYHmVwulNI1AHcAuDHw+xVKKaOpHwLwnM4sLx7bydBZoqNnAd1qghCgqMrpGDovuQgNfWDAbr5+DV3qW3mCUoqqbrnyQivwDH3YbYvshkWp14ajF0jjcpklhEw4PxcAvBjAQ4HH7Ob+eTOAY51cZBS2s1KUOVxYQO92RzXdshs25VS5Jy6XSsPEb3z0KJ5eqWZ+rkD7aEQw9IIq923wa5hNuyo5ZcDiE4T9vOvoBPjP10sdPQ1D3w3gDkLIvQB+AFtD/zIh5D2EkJudx/y2Y2n8MYDfBvCG7izXA5NctsO2yE7MXeN5FFS56w262CT4XEo9dasa+mNLZdz24Bl86d5nMj9XoH0whs4qRYH+llwqGS2I/oDen7uOTsEX0Huoo7e0LVJK7wXw7Ijfv4v7+Z0A3tnZpSWDEALNmSvaa7ATeTSvYH4s1zHJhVKK13/4LvzyNftw42W73N/bDZsI8qqUkqGbkAjQpO1p6Izh/+DJc5mfK9A+ojT0nONyoZSCEBL31G0BMwfU0mro9Z3D0GvbFNAHtlIUAFSZbGtAL+UUzI/lO+ZFX68Z+PZjZ/HjxTXf71n1YE6RUyWTqrqFoqa0/f3UDPvz3f3Uauqh1AJbR5yGDvRnu1kmPaZl2yywpU3uDzL4z9dvkkvfQlMk6Fbv7/Ss0+KIpmDXeL5jHRfZ6wSDNhscnFOlVEnRumGhoMlQZWlLDH2zbuKRM5uZny/QHlyG7rMtygD6k9FmllzqJmSJYKKg9m1eoFOoGxZG87YAUm70rtJ34AP6dgxB9hi6jF1jeZzeqIPSra+DSTfBoG1YFDlFQj4TQ2cBPfu6+KTqUSG79Aws36HySVGNBfT+Y7QV5zxpmM1U53+5YWIkpyCvpjuPBxl1s4nZkRwAjwD2AgMf0LfD5VJumMgpEhRZwvxYHrrZ7EgDpTNuQPd/JsPMxtCruoWCagf0dr6fqnPDKmoyfvDkaubnC7SHSIbuSC79yNCrDb5QqPV5tlm3A3pOlYafoesWZlhA72EvnsEO6NuUFC03THc7NT+WB9AZL/pSXEC3mlAV4rhc0pX+FzUZmkza8qEz3+wLDk4Lht5DsN2Ur7DIkVzSJh57iaw2xHLDwGhecXaa/fd5Oom6aWF6xJ44JZKiKaEp6XzZnUalYaKUswP6rnH7LtyJgO5KLoGTnWno+ZSFRVXdtDV0pT0NvaZbIAR44aEZPLNex8m1WubXEMgOlg/SAs25gD5l6Jw0l+aG40ku/Vss1SnUDQtjeRWaIrkdJnuBwQ7oMtmewqKGiZLmZ+idKC5iSdFg5Z3ncklrW7RQUBVoW9DQi6qM514wBUDo6L0CywepSlRA778AmNVXXq6bGMkrfe2t7xTqRhN5VcJoTulpP5fBDuiKBH0btDjGNABgbtSRXNa37nRZimHohkWdwqJ0F0LN8JKi7exgqrqFgqbg8K4xjOSUWD/6R7/3JO4/uZ759buJTiSntwsNK0FD70PNme8ZlOa83HSum5wy/Bp6zbCQV2WM5BVhW0wLO6BvT7fFUk521zAzonVWcolk6FkKi5yA3rbkYqKoyZAlgqv3TeJoTGL0f//7MfzFf6RrndoL1A0Lz/2Tr+HW+0519HXvW1zHh751vKOvGQUjMinqMHS9/wKgvx1uCsmlvjNcLs0mhW42kVNllDRFaOhp0a6ksFXwGjpgs/StFhdZTYrlTceHHgjauskVFqUI6HXd9qFrMmkroFecGwIAPHffJB4+s4n1gIuHUvuk/c5jZ0N/2y4srtZwtqzjsaVy6wdnwL/dfQJ/dutDrR+4RTD5MNhtEehPhl7JKrlwtsV+/DydArtGCw5D7+XUosEO6NvE0Dc5yQWwe7pstZ/LSrkBVpQZTHzqVhOqYmvoVpMmBmlKKaoG70NvLynKAvqR/VOgFLj7ab/swoKP2aS47diZzO/RDbCbarnDcxxXKjrMJoXZ5XwNY+iqr5cLsy32H6Plk6KtGLrVtDsz2hr6cCdF2Xfhaug9nCs64AFd3rakKB/Q58fyWNrcWkBncstoXgltR1lzLsbWkli6btkd8OzSf8k37SgtqrqJopP0PThnD/E4uep3uvBr6LTE0S7YTbXTSajVig4gnKzuNHSrCYkAiuzvtgj0p8ul3DBdeajV+pjsYGvow50UrbkBXWjombAdPvSmwzR4yWXXWB5nyzo262FZ4vhyuu0/c7icP1WMKCxykqKsr0fCxcB6obPCorZ86I5kAyD2JsJuOqN5Bd969GxfDDJmN8VKhyvzzrGA3vVRg01fHxegv10uVd10vdatJJQy19Bu2DV0nqGXckJDTw1NIVvyoX/3sbOpAy4D2z7xDP26i2YhEeBdX3jA99i/+I+Hcf1ffRPHTm20fF0mF+ybLkaU/nuFRUAyU2Tb4IImI9duUtTwJBf2nqGA7qzxZZfvhm41cfuxpczv02kwht7pC2jFCei1LicmdbPp088B7/vvz8Iir3im1Q2HsdSRnIqcU+E9rI3f2HdRUGWM5oSGnho2Q2//RH/7v/4Y//CNxzM9h7E/nqFfuXcCb7vhWfjcj07iM3cvAgA+c/ci/t557cdT3DTObNQhEeC88UI4KcoVFgHJDJ0FdFtDb8+nX2l4AV2TJRASfk+2xp84MIVdY3l8pQ9kF4+hd+4CopS6kku352AyaY0HIfaNvB8rK6u6iemSXViXWnJxGDowvHNF2W4lp8oYySlomM2ejYIc7IC+xV4u6zUjs1RQ5hpz8Xjr9YfwExdM4Q+/cD/+9egJvPOz9+HIvkkAtvuiFc5s1DE7mkMxp0APNDviB1wAyRp6JySXGqehuwElwnnD3ufGy3bhG48s93RrGYUzXQjoG3UTpsMka3qXNfQIhg7075CLSsPEdCklQ+c0dLcl8JDKLsximldkl/j1qrho4AN6u7ZFw2qiZliZ9dZKIyy5AHaP5/e97ipoioTf+/S92DNZwId+9QgmiioWV1uPcjuz0cD8WD6y/7XBRtClaKXKij2KmgJVyZ4U5V0yDFF2SX5c2k2X74ZuNnHHQ9sru3RDcmH6OdB96yA7zkEUnCEX/YZKw8KkE9BbSUJMchnlGPqwWhfZ58qrEkacnk+9kl0GO6DLMqwmbUuLY4E568VfjgnoALB7vID3vfYqXLl3wgnmGvZMFFIz9LnRvBu0WcBkn0/lk6JJDN3wNHStDdui3QrVa9sKwGHoAcnFeZ+cIuM5+yYxXdJwx8PbF9ANq4nlsp1Y7mRSlA/o26GhA3Zg6DcN3WpS1AwLIzkFWgpJiPUE5xn6Vm5SlFK89z8fTkWWeg1XQ9dsDR3oXYOuliPo+hnuoGiz6QtAacDumFm3Qp7kEv3VXXfRHK67aM7998JkAY8vV1q+7pmNOo7sn+RkFQuA6k2xUUgo2EehFtDQswZ0V4NXuYCuSqHtMVtDTpEgSwSHd4/i8Q4X9GTB8mbDvhGpcke3tz6G3uWg2jDDLhegPyWXKmcOKKRYH7veRpxui8DWvs+lzQb+9vbHMFZQ8cYXHWj7dboB1+WiyC5DF5JLCrACjHasi+0G9DjJJQ4Lk0WcXK0l9hhpmBZWqwbmR/NeQHcCqMH193BdLqmTotkZOvt8TEMHkiUXdpM5MDOC48uVbeulwhKiB2ZLqOhmx9ZxruL16Ok2SzasaIaeU+Wue+Czwj3PcnKqQiGXCGmKu9PcSkBnxGW9tv122SB4Hzojfr3quDjQAd0Nfm2MoSu3KblUWjD0IBYmC6gZlo/pBbHkeNDnx/LIBXzfLEegpiwsqnKSC5tYlCW4sZOxmEuWXNhNlF2cB2ZL2GyYruzRa5xx9PODsyNo0s75ts9VvIDR7SSebjaRi9TQpb5j6P4kZ+tS/nLdRMnpD+Qx9Pa/T3aedmKwTKfBPherFAV6N1e0ZUAnhOQJIXcRQn5MCHmAEPLuiMfkCCGfIoQ8Rgi5kxCyvxuLDYKXXJLwqR88jWcCPb2ZplfRrUwBz50nmjKg75koAEh2ujB3xvx4PiC5eJ9N5V0uiYVFHsN2v58MLJ1n+AxRLhe2PmazOzA7AgA4nkJe6gYYQz/orKNTmmWvGbqqkNDv7UKc/gro1QY7T2wJJY1tkckPuQ4kRdmx6EeGXucY+ki+txp6GobeAHA9pfRKAFcBuJEQck3gMf8dwCql9BCAvwbw551dZjRYwEpyuixt1vH7n7nP9YczMMnFatJMxUmVhgmJeD02WmFhsgigVUBnDD0XsiZ6k+CJt1VNU1ikym6wzeIEYtpoQQ1ILnEaOmPoM3aLgG0L6Ot1aLKEhUn7BtopzXKlomOyqALoTaVo0IcO2FpsvyVFefuunbRNvob4/kedsC3W+1hyaRj2gJicIrmfuW80dGqDZbtU579ghHglgI84P38awA2EkDDV6DA02b7TJzH0x5fsALMWOPC8jSjL3bPsdFpM+/H2TDKGHp+NZ+xynne5OCc734EvTWFRTbegOYlKlmPI4kWvRTH0iFmmvMsFsHciOUXKXHnbKZzeqGN+PNdxRrRa0XGes8vqdFBdrxk+h5YemxTtv2ZW7MZva+IpGHrdxEjevjF2orConxl6zbCQUyQQQtxBOH1lWySEyISQewAsAbiNUnpn4CF7AJwAAEqpCWAdwHTE67yJEHKUEHJ0eXl5aytHOsmFVWluBA48f8FnuXuWG6ari6XBeEHFWF5JHOO2tFGHpkiYKKqcNdE+YaOSoq1siywYq+4OJsMOJLXk4rlcAECSCC6YKeH42e1j6LvG8h1nROcqOqZHctBSznNNi426gRf++e34+F1Pu7/TY5Ki/ehy4Rl6IYUkxF83nRir188B3Z5WZH9GSSIoaXJfSS6glFqU0qsALAB4HiHksnbejFJ6C6X0CKX0yOzsbDsv4YOnEcefGEwCCB54vpFWli872As9DRYmiy019PmxnFuVCXCSi+klRePK8Hmw8XHsOUA2Dd3V4HPJLhc9ENABW7/eLoZuf4d5rzKvQy1LVyo6pksa8kpnE5O3H1vCZt3E0yveDTCq9B/oz4DOpL1SyhmhbLgFAORdt9YWJBfnuf0Z0C23SyaAnnZczORyoZSuAbgDwI2BP50EsBcACCEKgHEAK51YYBLGC/YWbqUc7yBhDD144PkvOEshSrmNgL5nstBScpl3RtkFvea660OXYsvwedS4TontaegRPvSIwpGG2YQsEV+r1wOzJZxYrWW2kX7v8RW84zP3hpLTp9frePeXHmj5epRSnFqvY/d4HiXns5cTjulnf7iIbz2aboe4WtExVdI6HlRvvd/ufbPKuTSSS//7S3Lh7a2pXC5RSdEOMfTgebNSbmzrDbBuegwdsA0Une7RH4c0LpdZQsiE83MBwIsBBMe3fBHArzo/vxrA7bQHhuQLWCIuYZt//Gx0QN9sU3IJ9kJPg4VJu1o07itZ2mhgfpwFdL+ThU+K2n9PnlrE9zJX5eySC9+tkSEXMfquYVohNnlgtgSrSfH0uWyyyx0PL+GTPzgR2il97dgZ/PN3nsSDLbpVrtcMNMymn6EnHNP3fe1RfPCbrZuy1Q0LFd3CVElDQetcYrKqm/jmI/YNhbfdxZX+59X+607oNqnT5FQul8260VmG7pynVpOGzpubP/CdVMe3W6g7GjrDSK6/GPpuAHcQQu4F8APYGvqXCSHvIYTc7DzmwwCmCSGPAXg7gHd0Z7l+jBdUzIzkYisU64blSh3BpMRm3XSDZDbJxQo15mqFhckiqroV6ZmllPoZeqC8n9fQATvgtyoschl6Slsnj5puQSJ+KSWusCgXcPocmLEtg2kqY3kw+Yu5fRjY0OynVpJfjyWVd42nC+ibdQNPpFgjqx2YKmmpglZafPPhZdSNJkqajPWat7tMYuhAf3UnrOomcooERZZsl0tCWwRK7aA76jB0RZagSCT28zxxtoKXv/9bbpfLKPA3V56sNUwLJ9dqW54gthXUnQHRDCP53vVEb0k1KaX3Anh2xO/fxf1cB/Cazi4tHQ7OxifinlypgFJgZiQXKbnMjeZxcq2WOSmaXUP3vOismRFDRbdQ1S3Mj9ltSIOSi8fQ7Qs9ryYz9JphM0r7OU4lbUaGXtT8Lp64wqKcEmboQHbr4oZzs13aqOPQ3Ij7exbgnzyb3K+DXby7xnjJJfqYUkqxWTexWjV8u5ko+AK6lqp1DkMAACAASURBVCx7GFYTXz+2hJdcOt/SAfXVB05jqqThOfsm8SR37kYNuAD8jLaohf68LSj7bIjJlaw1w0KT+ms3kmSkexfXcP/JDTyxUgldL/xrMqzXDCzYjU3dY1btct+dJIQ09JyCs5u96Tkz0JWigF3QEtdvnFkWrz5/AuWG6ZsJudkwsMuRObLaFrO4XAA+oIcPKmOnLB8QLizykqLs70lMrap7J5OroWdg6HaQ8+9AcooMw/I3QWuYTffmwzCaVzE7msucGGXb0TOBMX7s360Y+hmOoTPGGHeTrhtNtx1uqxuFn6EnN8i646ElvPljd7eUhxqmhduPLeHFF89jZkRz7bRs6HYUQ2c7rn5KjFZ1y60mzqsydLOJZowkVOb6uDDkVSlWd2eOtHpCUOZ3BDxZY/m0ag/neAZhu1x4yUXtL5dLP+PgbAlrVSOytJ4FlqvOnwDgMUHAPsl2jdkBPW1SlFLanstlIr64yE0uOa8Z28vFqSDMtXAU8ElRz7aYLSkaCuhqWLppmFaIoQN2gVFW62Kc5OIy9BYB/ZTD0Occ2Wokp8QmRXl3E8uvxIEP6AUt2ZrHgsrSRnLrg+8+toLNhokbL9+F8YKG9aqd1GM3GU2OrhQF+mtqUblhuh7rVi0pWL5qJOCcirtBses06fPyz13npMyVPmHo/qRon9kW+xms1DuKpT++XMZ543k3cPNe9HLDxHjRHoeV1uLWMG12lzWgjxUUjOaUSIZecVsJ2CcAIcRuR8pcLmZAclHkRIbu86G3mRQtBGSI4K4BsG84UWzyQBvWRZbfYEyb4YyroScz6TMbdcyMaO56SjkllqHzN/VWOjoL6NOOhp4UYNj7rSTovoDtbhnNKXjBwWlMFFXoVhNV3XKPc2Rzrg50J+w0qrpHbPItmm3xvdAZojp4MrDrNOn7rhmW+135GXrDWd/2fVe1GA29F43rhiegRyRGj5+t4ODciCtn8Ad+o24naUYyDHHN2mmRgRCCPZOFyOKi6O6GUriwyDl5ky4EIOhyya6h14xoyQXwuxIaERo6YO+YVqtGYkIriE1XQ/fYbcO0G5qN5hSsVMIDuHmcXrc96AwlTYndcvPHutVO4lxFhywRjOVVFFpo6Kwgi+/9EoRpNXHbg2dww8VzyCkyJpzzcq1mhHIlPDrRP7zT4McUthpY4TXyUt3fJSWZ2RSxpERrjcs7RUku3e5dnwS+sAiwP7fVpD05fgMf0PdMFqApUujipJTi+HIFB2ZKGAsE9IZpM6LRnJLI5oKImieaFnHFRSwQBLejcUnRnBLv+W06J01IQ8+cFA0G9DBD1yM0dIBLjLaQM3h4kovH0Jc37cD4nP12tiuJpZ/eaGD3uBfQk27S7L0KqtwyoLM+LpJEWg6ZSMPQF1drWK0aeMGhGQDAhJPhXKvqiQy9ENHy4c9ufQi/+I/fT1x/N1FphHuzxAVRtxd6Lq2G3lpyqRkWpks5yBLxBfSzzg21aiRf0//8nSfwt19/NPEx7aJhWH4NvYcNugY+oMsSwQXTpRBDX9psoNwwIxl6heuYmCWge0wjm20RiPeiewzdX8jD2Itu+ZOi+QSG7ra+3YJtsdqI19AbQQ09okFZVuui1aTuTY1PijL9/HkXTAFoEdDXa36GnpNj8yIsuFy2ZwzHl8uJ22BWVAS0rtZkx/FcQpHbknOTYjefCafp13rV8ArIYipFAX+Ae+j0Bh4+vRn7Xt0Gc0MBaNkOl103o3m/yyVWcknB0OuOtDhRUH19mtIy9C/fe6prg83rZlhDB0RAT42Dc+FEHNPUD8x4AZ2dKIyljebVTAmLih7eOqbFwmQB5YYZsk9W9IiEEVfIE/ahx9sWg61v29LQjbCVL9gwDIiXXBYmC1Blktq6yL57TZZwZqPhBljmQX/efjugxyVG64Y9HGSXL6DH36TZsb9yYQKbdTORUZ+r6JgspgzoruQS/3psB8KStyygr9WMyFYKDF7vE+/7X60aWIuokuwVbNuiI7loLSSXujd+jiGpujSthl5QZYwXVN81xb7/VkaHlXKjKwHWtJowLOre5AAvXvSiuGgoAvqBmRE8fa7qY6KMIR6cK4UYOj8Oy7744w/+F+45ib/8j4cBeAcka2ER4Pei8wi6XAB/u1rDTYqyStH4wiLGSgqBStEsg6J5l4y3noikqNmEFiG5KLKE/dMlPHImHXtkAfaCmRJ0s+lut1nwu2CmhNnRXMi6aDUpvnr/afzyh+w+cfudqmGgleRi//6KvbbzKenGs1JpYHrEDugF1bZumjE3xzSSC2PoTPudKDDJxfANMgkiKum4VtVhNWnPJuEEUdVN95xtNVIuamxjUv+XNC6Xmm4hr8kYK6g+swNLitYMK9ZGaT9O70pLW+bHL2j+SlHAtkp3G0MR0A/OhUvOjy+XUdRk7Bqzh0ZosuQGdHcLmEJy+eI9z+ADdzyG7z2+kjgguhV2jdsBPejkYDeTUO8ULilKiC0tAcmFRUw33JoP3fKtha0HCEguRrRtEQCuOTCN7x9fSeXKYAH24JwdkJnscmazAVUmmCxq2D9dxJOc5LJSbuBn/vqbePPH7sbpjTre9fJL8LLLd7t/L2pJDN0J6HvGAQBPJGj9q1WDk1ySe9GzcyN5MpXdVZMRDMbQV3kNPUFy4RktSzqvVbofJIJomBYMi7pFXK36m282TGiK5MsPJNoWU/jQWa5ooqj6KrDPcpJX3A6gYVrYbJhdYej8cAsGJjUJhp4STLd9bMkL6I8vV3BgtgRCCAghvjv5pmujUjGiJbtc2E3gf//7g+7z2kmKsoMabEFQadiuEkkKVmay5lx2fw9WfZhUWBTsZc6m36SVXJrOJPewhh4uPdetaMkFAK4/PIeqbuHOJ861fE/2fRxy3Ershndmo4650TwkiWDfdMnH0P/9vlN4fLmC9/7ClfjG716HX3/hBb7vbyQno6JHM7RNZxTa3qkiNFmKZehWk2K1qmOq6DF0IJ6FMrlrJWEE39JmA3OjOfdY5lUZOcUmGnwTtiCCkovVpC6LXauldxN1CtWAOaCVT75cDxfjxTF0SqmnobfwoUdJLiuVhnuDibMuspuuYdGOt1PgB0QzdLoDaBKGI6BHOCuOL5fdQA8A4wXF3c6z8XNMcknyrK7XDIzlFTzwzAb+5c6nAGwxoAduHhUuucSQU/0uF5615Zy+3FHaqSe5tKeh100LlPrlH/aeQEBDN6JdLgDw/IPTyKsSbj92puV7smNxcI4FdDsgLm00MOdIE/uniziz0XCtiLc9eAYHZkr4+asXfN0eGdjxqUYEhM26gdG8Clki2DddjHW6rFV1UAqXobObWlyyje0IKroVG/RZi18ek0XN73JJIbnwAWy1RzM1711ccwt4KtxwC3t9yTe71aru7kYYbFkx/Pi60XTlp6QpSDXDlgb5gF7VTdSNpjslLO5Y8d1ZO82a2XeQU8OSi2DoKTGaVzE/lnNL/SsNEyfXaq5HHYDvwJc5G5XN5uJN/xt1AzdetgtX7Z3AA8/YZd0lLbuGPuokRoJ+6gqXXGLg29UaVtPVzwEvsER5y4NJUUViPvR0GnrUPFG2HiDocgk352LIqzKuPTiD2x9eapm0cxn6XJihs4Zl+6btG/bT56rYrBv4/vEV/LdL5mNfM6lB12bdaxJ1YLYUWwTlVomO2DeVQsQuhQe/y4uTXRhD58Ekg2BFMA/WB58Fi9Wq9/pr1e4z9GfWavj5v/8uPnCHbfML2ncLEUlbHitlHdMl/+eOS4pucNdHLYbRUmrvJPOOfLVRN9BsUjdQ73XyVXFE7Sy3i8rSOjsN2HfA93IZL6jIqxIe6oEraSgCOmDLLsfP2ja0P/riA6AUuPaQNzRpjAvoG1zlWimngNL4g79eMzBR1PCHL78YgM2WolhhK+RVu8Nc8C4d1SAqp0guYzMCDZuSphYxRsoCMiEEmiylti26DD+goQe7/TWbNHb+JcP1F8/hxLkaHovphMnAjsXsaA5jecV1t7ChHwCwb9pmXE+tVPFfj5yFYVG8OCGgu4woKqA3DDegX+Ak06MSnW5A51wuAFDT4x1GbL1xAT2KoY87tjuPoYfJAiHEV4jDa8a9mHr/0e89BbNJcc+JNQAeQ/d6uSRXiq5UdDe5zJBXpVB/IMBfzR0nuTTMJii13TXjBRWU2jdqFqj3TtnnS1xxGc/QO52ojNLQNUXCSy7dhS/fe6rrHTOHJqAfnLO96B+782l8+u5F/PYNF+KIY3kDAgy9YbfOzSlSIptrmBbqRhPjBRXP2TeFn73qPOxx5ktmBSEEI3klpKHzXesYeGtiIzBnMmk4AGM0fOm+pkipJRePoceV/vuHbsQxdAD46YvmAAC3P7SU+J5sxzKWVzE/lseZjQZquoWNuok5J/jtm7IZ+lMrFXzt2BlMFlVcff5k7GsmHdNy3cSoM9vywGwJhkUjC774Pi4AnxSNZ+h7na1+lNOlplvYrJuYjWDo6xxDVyMYOnt/xv7WfAy9uwG9plv4hDMm74FnNuy6gYA5oFWl6DnOz88Q1xKYMXRC4iUTdu4zDR2wiZfH0FlAT9bQge4x9HyAFL3q6gWs1wx8/Vjy9bBVDE1APzAzgo26iT/+4gO47qJZ/M4NF/r+zrZmgDcOixCSyOaY5j7mMLq/fM2V+Pxbrm17jaMRfZH5rnUM/FBmw6K+5GM+Qs/mXwvwO2ZUmWQI6OEiJ3s9fh+6N080Xno6b6KAi3eP4estA7oJRbJvrvNjeZzZrGPJcbowNjteVDFZVPH4UgW3P7SE6w/Pu66fKJQSCjk2697knAOO1fGJCB2dBWXetghEBxnDakI3mzjfYYZR5f/Bz8QwUdCwVtO9YeAxu56C6vWS4XXz1S5LLp+/5yTWawZe99y9qOoWji+XPWeWc56w8zNKcmHJ5ekR/40s7jnsmpsZycVq6LWYgM4C9d6pFpILd3zKHWboNZeh+4/jtYdmMD+Ww2d/uNjR9wtiaAI6S6qdN5HH+157lc/1ADgBvWZrbSwxBvBsLnzwGaNnrQMUWXKf1w5GcmokQw8mWW0N3fOhRzH0SMklYtqQKrfD0JN96Oz/cS4XhusPz+Lup1Z93fCCsI+FfXOdG8thaaPhJkaZhAHYOvpX7j+F9ZqBF18yl/i+IwnHdKNuujfoA06OJSoxymyBkwHJJWpnxFwfbKsfNRKRedCjNPTVquEezygfOnt/T3KxXz/o8Og0KKX45+88gUt2j+HXX3gBAODexXWXobOkKBuNGPXdrDrJ5ekYhh58DiNd82O5FPUW/oDOAvXCZHrJJWlUYTvgdw88ZIngZ5+9B994eNmn4XcaQxPQrz5/Ai+9bBduef0Rt0cGj/GCiiYFyrrpkzmS2FwwoG8Vo3kllBStNqxQkjXYy4XfhnvMJkpyCU8bUmXJ7aneCnGSiyIRSMS7ibCbTeuAPg+rSfHNhPmdvAQyP5bH0mYdp9Zr7r8Z9k8XsVm3/cwvujB5wLjrcom4oPmb+WRRxXhBjUyMrjiNwZh3Osmax+ZF7h7PQ5FIpIbOkr0hhl7UnIIq+7yI+05z3ECItaoBidhMtJsM/buPr+CRM2X82rX7cXB2BEVNxn0n193v1V8oFO0rD0pX3uOjz2P2Peway8dKLjVOp3b74dR0rJR1FDXZfa94l0sDM86OoVsul6DkAgCvvnoBZpPiC/c809H35DE0AX00r+Iffvk5uHj3WOTfx/Je3wx+2z2SoLduBIZPbHmNERWMUf3Vc4o9Q5IlH3nWltR7mpVD8xNzsmnoTIP3n4w2A/Pr+oC3W4jDVXsnMFXSEu2LvOtkfjQHw6KuG4C5XADP6XLtwemWttE4GU03m2g4TdnY59o/U8LT58J9YjZqBsY5q11S8UyVq4ScLGmRDJ3tOqIYOgAsO6wtnqF7MpxtA9Qcy2P3GPo/f+cJTJc0vOLK8yBLBJeeN4b7Tq67bQ74iulCTEBnbDSUFI3p/+IlyfOxDDteQ7cre9nOIUlDZ4n2TleL1s34/NKF86O4YmG8q7LL0AT0Vhjj+rlsctvuJNM/YwudCujBpCilFBXdGxTAwIK2bjUTXC4RW/+IXuZZNPRgYRIPu22vX3JJcrkA9jbzyL5J1+4ZBV9Ad9jrfYvryCkSxgreZ9k/Y1+ASXZFhrikqFvpyzWJmgiUjrvrCiSrCwmJP76CeLqkRSZFlzbr0GQp5MdmLXSXnYAf1W2Rvb8rudQMTBRUTBa1rkkulYaJrz+0hNcc2euej5fvmcCDz2xgo2aAEL+sEFco5PWUD9sWgYikaM1AXrXtiHE2SOY0CkouKxXbHskISZxL5mxZd62NnW6dwKpboxg6APz8s/fggWc28NDp5MlW7WLHBHT+wPOSS3JS1HNgdALBpGjdaKJJw4VKfCGPYdFQYRH7WxDLm/WQVqlmsC3GaejsfYNDN5JcLgx7p4qRXSYZNuqG27yIuVruXVzD/Fjet9N40YWz+Lln78HLLz+v5XuypHBQH+WbsjGM5JXIi7oSCOj5hKQonyScHtGik6IbDcxyVaIM46kZupcUXXMKdWz9vTuSC9O+D3A9ci5fGEPNsHDv4jpKgbmzcZIL260EGXpOjUmK1g27/7wqQ7eakZZSPimaV+2WAutVw/G724NOFIlEsm9KKVYq9rEYydBpNS3iNHSGm6/aA1Um+Mzd3WHpOy6gb9QMbNYNl6UlWdzWO83Qcyo2616HPLfiLsLlArCeGf7CoqQJ8I8vV9yeKAyaI9+kgedyCUsakZJLCw0dsIs8aobl67HBg98tsSToBjcekGFmJIe/fu1VPhkkDpJEUNTk0DHd5OoPGEZi+r6UG2ZgBmZ88UyF05SnSrlIDX1ps+5L8jKwBl1suIcaMYLOfn+PAa9WDEwWNUw4SdGkJlTtIip/dPkeu6HZ0afORZyz0YOiVyo6CPGSywxxVseNmomxguo2t4p6TV5DJ4S4yWFfMzVNjpRcqrptRZ4eyaGUkzuvoZsWZInE3pinShr+8jVX4vXX7O/o+zK0vCIJIXsJIXcQQh4khDxACHlbxGOuI4SsE0Lucf57V1dWuwWw7Ttj6IylxbE59tiCKsdug7NiNK84/SPskzToFmBw29Wath0uTWGRbjbx9LmqrzoWyO5yUSQSMwZN4lwurW2LDMz5cSJi/B7gDN12Aifv0Z6LCH5ZENV0bcNl6FxAzyuRF3XQfSRLdpFW1Da+kkJyObPRCCVEAU9DZ5JMkMEz+AuLdIwXVUwUNVDqr67sFFzLLid7HZgpoeRMbgrJhIoU2UxrpdzAREEN2Uy9nWbY5TKWVxJtoi4LdnaS4wXVnSvM7JFFTY58Lp+kHckpbkK7U6gbTddaHIdXXrUH5zsafqeRJlKZAP4npfQSANcAeAsh5JKIx32LUnqV8997OrrKDoCx7OXNBgyLuttpSSIoRbA5gLGF7H1b4hBs0OWVUMfbBG2XS+vCoqfPVWA1aSiga7KUekh0NaJ1rve+npWSXYRpGDqzkEUV71BKfTfXnOI5FKKCXxZEtdBl3zsvoZVySmQjr/iGUhEBnUlVOXv9m3UzJHMtbdRDCVGA77hoJBIHn8ul5jB01k+9C4nR9Qi5UZIILnW6VAZrJ+JK+fkgG3w8EOVDNxyGrjh/TwjoqhfQF9eqMCzqSo5FTYns5cOStDMjTkDvMEMPzhPtNVpekZTSU5TSHzo/bwI4BmBPtxfWaYzkFMgSced68iwtroXues3omNzCvycLNJUI+xfgL7oIauj5GIbOOk2yRmUMWZOiUfq5vSbvgs0iubA+8CcinCRV3YLVpL5jwYJelDyRBfbUokBSNGIU2mhMUjzKfRSnE/MMnd2QeG27bvgrX3nwO8A4uYU9rm5YaJgWqrqFyaLqyhjd0NHjHF6s7XCQoce5XFYiqkSBJB+66WroQLRTJdiiYqKgul0zZ3wMPXxNu5p+KYeRfHc09L4O6DwIIfsBPBvAnRF/fj4h5MeEkFsJIZd2YG0dBSEEY3nFZYq+bXfMQIROB/SRQIOuqAHRgL94SDcD3RYDVZsM7oSmCMkldVLUCHd+dN+XK3bSM0guJUeGWIyQXPg2xgyMmW+VoZe08OCSzQjJJaqwrOmMxQsGdHtQdHRAZ64PxhD54pGlGMsiYJ+XzOmSxNDZ7oCx8Ymi5uYT1rrgdNmIqcG4fMEJ6Cnb4dqe74iAHlNPYTN0xdXQoySuWmCHOF5Q3cA/5TL0aA19peLZKEstWme3g4bRDFWJ9hKp35kQMgLgMwB+h1Ia9Nz8EMA+SumVAN4P4PMxr/EmQshRQsjR5eX4YpNuYaygugydHyMXx9BZxr1TCDa652eb8ghLLq0Lix5fLmPXWD70WmoGH3pNN2Oz87zLpZHB5QJ481SDYAGWTz4yZj432j3JJehyAfwl4IythyQXRY7R0C3X9cECCp8YPRNT9s/ApJO4RBpgM1qzSd0bxQTH0LvRcZFZE4PfweV74gJ6fGFREkPnd5qsF/pYXk10FdWc4SqsGpy/6XhJUcWVwnisVPwMvdMBfSAYOiFEhR3M/4VS+tng3ymlG5TSsvPzVwCohJCZiMfdQik9Qik9MjubXO3XDYwXVJxcZQGdZ2nRQ4U7z9Dt92TFE5W43imcrBIsLFJlCbJEQpJLlMMFsDX0tC6XSsOKHa9nu1yylf4zLEwVIyWXjQjXicfQO5AUDWy5NxumPb2KW7c3wNc7/sH2sAx5TY52uTRM93tjAYUP6C5Dj/lMzOnSiqEDXsUpc7kA3dHQN5wcQrCFxv7pknMz8V8XUQHdtJpYrRohDzoQ3cuFSYxjBTVxoEg9kOvhr1FXclHjJZeiJqOgyYmjCttF32voxE67fxjAMUrpe2Mes8t5HAghz3Ned6WTC+0ExguqGwjTSi6dKvsHvAQTe69qI6zpAv6hzMEBF/bf/VOLKKU4vlQOJUQBJymatvTfCBcmue+pRjD0FJILYHe/O7lWC7VKZd/DGHcsrt43iYOzJZzXZldLhqhdF1/27z5OC9tW+QEoPPJKtMulzBWHseDFV4u6Zf8xuw4mnSQVarEg8cya/VoTRRVjBRWEdGfIxXqgUpZBkgg+/sZr8NbrD/l+n4uQXNi6gh50wO6LpEjEl0jd4DpvJhUHsYpoBr5Yi+1aYiWXsmdtbOVD180mnl6JdmfFoR5YW6+RxsJxLYDXA7iPEHKP87s/AHA+AFBKPwjg1QB+kxBiAqgBeB3drnHkCeCDcygpGribN5u2A6OTAd3d3jMNnXNH8PD70GloK86mFjEsbzaw2TAjA7qqZEmKmtgdIwvwGjr7f1o7596pAgyLYmmzjt3jXqCOKvT56Yvm3Na7W8FITo6UXEYDQXok4DwCPLYeHDxS0GS3aRePKpdAHXcsej6GvtmIrBJlSKeh22s5vc4CugZZIhjLq1jvkuQSJzdecl64vUZesQuBrCZ1LYpMr46SXIAwq/d0ewVF1f4+oyWXZmiABGATA/YdFmJsi6yaFLCvezaGLkhOKKX4rU/8EN98ZBk//MMXx+aWgqgbTUyVtk9Db7lKSum3AcSn3+3HfADABzq1qG6BP0H9kksUmzNBaeeKivj39GyLrC97tORSc1wgwYBuD4r2TtbH3IRoWHJRM0gu1RYuF/aeumX71ZNa2PJg1sUT52qBgB6WXDqFUk5B3bArDdlAksiAHlFYFlcfkKihO8Ffkggmi6rPi760UY+sEmWYdAJeKw0dAE67kovXYKxbDD1L/ogx6obpJdbPlaPL/hmCiVSeoeeTkqK6X9Zg1+gMZ4+MZ+g6zpuwSQuf08qN+M/7f/rOk/iPB+weRA8+s+GbrZCEumm17HHUTeyYSlHAH5xHWkgung+3c8FGUyTkFMmzLTbC04oAT8pgjwsOPeATlABcy1as5JLBthjrQ+eTokb8gOgo7I2xLrpJ0TZmtLYCe03ei8xa9UY9jj/+7EYTklxinBzBfjxTJX/5/9JmIzEnwM7LRMnF+b5Pr9ehKZLnwS5q3XG51LPlj/IRmvjZSnTZP0NwrqhXzKS2LCziz1O28+Hfp6gpqBnh+oKVSsPdMXhym/89fvT0Kv70K8fwvAvsIH7v4nr0h45AXbd8A6J7jR0Z0O3Ayk3l1jw2x9DpTosMo3mFS4pakcGMSS4syAQv9OBW9fHlMoqaHCqXB1ilaDr1q6KHvdf8mngNPQsL2eME9KDTZbNu2/2CTLgTKEZo45t1053tylCKCOjBiTwMBS2Oofu/Nzug+zX0JNeO63JJIbmcWq9hsqi6bH+ioHbF5bLu2AfTIspXfo51WoyVXKQYDV1JbFcc1NDZNcrvBIoRGjyl9txRVujEjhk/hm6tquOtH/8R5sfy+MfXH8H8WA73Lq4lf3gOdbPpWi63AzsyoAetWGy7zNucOt3HhWE0rwYYenQjLMCzN0Zp6Px28vHlCg7MlkKOBPZcqxme3RhEs0lRD2iT/veUYTUpTKvpaI7pT52cImN+LBcq/990JkdFrXurcI9pIKAHWXdOkaDKxBfQywnJ6ijXRTngDpou5UJJ0SSG7rpcEhg6Y6Sn1+vu4wFbcsnqcnn0zCaufPd/RjqPGDZqZjaGHhGAzzl9XKLmE7Dn8PUUvPddle3jkkZyYXkuP0MPFyZt1EyYTa+alO3WeIb+j986jtMbdfzdL12N8aKKKxYmcO/JDAzdEAy9Z3ADegodNa6wYqsYyXlDLqKKVwDvwt6MCeiXnDeOO584h2ccT/3jMQ4XwJNrWsku7MKJ19A9K2XDzCa5ALbTJSy5mB31+fPwpBTvYuX7xjAQQkI5FBbQ0xYWVSMkF6ahJ1WJMjCGrsXMEwW8/uEV3fIlVyeKWuZK0QdPbWC9ZuDRpegp9LrZRM2wMh2bqIEVZys6Jp3kbRRySpCh+3MqeTU6sRmUXMYLKlSZ+Hz+zK3FP58laWcCDJ2vQXhyFAtqbgAAHj9JREFUpYp9U0VctdduRHbFnnEcX66k6pdDKR0MH/qwgG0hgywtquNi9xi6whUWmZG+bzbSy9XQAyXhb73+EECBv/nao6jpFk6u1WIDOrs5tEqMJrXOBQLeeLOZuWEZa6PLI0rT7hSCx9Rq+vvG8Aj29Cg3vDmnPPKKHJpU32xSVAM35qmS3afcsJq454S9XQ8Oh+aRSkPnirj4zoUTRXusYVSb2Tgwp07UIA6AkxtTdLZk8HoMees457SzjUNe9fv6WS90JocWY26gtjWQb1gn4+O/cQ1+9fn73d+5DN3wjmtwTmzUTZ8lsBmucAL7/SlYum7Z7bAHolJ0GMAunOBWOiox1q2AbjN0LqAnlNqzCysYPPdMFPBL15yPf7v7BL7mTAOKcrjwzzUc/XujbuCLPw6PwPLmNMZp6J6LwWbo2VjIwmQBp9Zrvp1ClOukUwge0yjPO//YoIY+kldCrhS3pSsXZFjSlb8xs1L3fzu6iF/75x/gvPE8rntWfCFdFpeL/XiOoXN9/tOC6ftRbX6B9uYAMKmOT3LyCcgohGyLgcpsfjA2j6CGDgDP3T/luwEVIiSXlbLfRumeI9zN3E5ge0yfVcbelyIxym5OgqH3CJ7kEp0Y47W0jboB2emr3Un4NPTEJKQcmxQFgLf89CHkVRl/9MUHAEQ7XAAvSLDE6BfueQa//YkfuXINg9sorBVDN7Jr6IAtuTQpcMopjAHsZFQ3HC5AmKFH9XFhCAb0csyNNkonrkTIM1NOcu4PPncfnjU/gs+/9dpkySWFD51vszDOa+glNlPT/nxfvf8UfulD30/skc6Y6rkYqWad84OnRVR/85WK7rMShp8jBXzo/rqPvBptPawZFvItrstShOTCevKzNY3k/ecIpdRJYHtrnipp2DtVSOV0YTczEdB7hFZJ0SBDHy+osd7hdmG7XOwLpppYai/FJkUB+6R84wsvcBNPF8xEM3QvoNvs4awzfT44ebzqMvT4pCjgaOhGM3UfF4aFKce6yCVG+QHRnUYwKeolOsPvF9LQY3YOUU4OV2/nbgC7HZ/zSy/bhU++6fkt+9IUNRmqHD8UAfBPwOHL7tk5zZwu/3Ln0/jOYyuRvXMYXIYeK7mYvtdOg3zEBKKVcnQfF/c5ihzyofM7qKicRavkPQMjYvxxZZ/brSZ1XoNNrNqom6gbzVDPnSv2TODek62dLoKh9xgseAQ19Kik6HrN7KgH3VuDzQZZL/AkycXzoUcfpjf+5AFMFFXsmSjEnkRMf2caOjupg0MYkhgsWw/QvuSy1+2L7gX03kgulvteQAxDD4yhi9s5RQX0akTfl2fvncCX3vpC/N0vXh17g+RBCMH/etkleNXVC7GP8UkuRS3081rVHtzy/eN2x41jCTMrW0kuUb3QW4ElbRkjNqwm1mtGrAcdsHehfD3FRqDVRiEiKcoe3ypoRrUOWCk3MF5Q3Z2QJBFf+f+y00Qt2HPnioVxnDhXi/2+GLxJSkJD7wlkieBll+/GCw5O+34fNSh6o8ONuRhGcoozZcZEw2zGSy6KzDH06F3CWF7F+/+vZ+MPXx41b8SGFmDobJsdZGcrLav64lv6psHu8TxkieDEOY85bnaRoRdUm/WynUii5KKFGXrUcSlEJP48R4wXYAghuHxhPJMd81dfsN9tTRsF1pQN8Ccr+QEZ3370rCutPXw62sECxN/UGdoZjh6UXFbdroYJAV2R/IVFAddTlIbOzxNNQpRt8WwlnKTlx9Cdcdsc+xk6Oy73tUiMfvZHiwDsBmbbhe7Qoz7G3/3S1aHfxSVFO21ZBLxdAmvYFOsqUSV3PFZS8HzRhcldK13JxWnQtRrDzvg+0XHrATgNPSMLUWQJu8fzruRSNyzoVrNrDJ0QgqvPn8R3HjsLILp1LkNwDF25YbrtCngw5hWpoXehOCr0/oqEim4FXC5eC927nljBaF7BeEFNnCrfMilaz27ZDUou7GYxFUMQ7Of4pxxtBIqZogq5Ugd0pxdMMCkaPL/5MXRuE7UAQ/cSo2v4qZjk9v0n1/Ghbz2B1x7Zi8v2xN+Yu40dxdDjkFNs9hP0oXcjoDO5hzVZiksK5hQJrL1ZkrbaCkyu0S37xI5jZytlHZoiJa4H4CWX7Gvivejd7OPCcN1Fc3jo9CZOr9ddXThKRguOoYtrIxzV0jVu6lQ3wFgwr6GP5hRIxJ5adMfDy/jJZ83isvPG8dCpaIbebFLXt54kuWiKlEkLDspR7o4vQXLJq5JrA+V7oTMUVDk0p5RJMK2Soq7kwu26lzYaIfbNW1aXNlmbY/9jRvMqDsyW8OOYxKhpNfH7n7kXUyUNf3DTxYnr6jZEQIdTXKLJIZdLNyQXFsBYk6ViguTCsJWA7vrQHYbusTN/UvRsWcfsSHwDKV9StA0NHbC79N1/cgOrFb2lZt8J/PRhm01985Elb/xcxPsFx9CVG2Zk8jRq6ELckJJugL0/X3kpSfbU+28/ehbLmw3ccHgOF+0axZMrlciinPWagSa1pZByw/Q1eWPYqGUv+MopEgjxnB7uji9BcmHunodPb/p6oTMUNDk0FzQ4TzQOmlMBzBg6pRTPrNewezwQ0LkxdGc26ig5fdKDuGLPeKx18R+/9QQeeGYD77n50kze/W5ABHQHvHWNUtrx4RYMLHgsbTCGnmwTBJIrCFtB4ypFKY1nZyuV8HY0aj11w4LeJkN/1dUL0K0mPn/PSfe7DvZW6SQumh/FrrE8vvHwMjYdG2pUIOBtq/b4OTPyuHg6saehV9mQkpjj2EkwWSN4Xk4WNfx4cR2E2LuSi3ePokkRWQnKdmYH52yb62ol7F8PSh9pwIrh2Hdzzi3iiZdcfvbZezBeUPGntx7zdVpkiEqKppVc2GNYQF+rGqgbTewO9Nnnx9AtbTRip0o9Z/8UTm/U8Z4vPeiOYKSU4qv3n8L7vvYIXnLpPF56+e6Wa+o2REB3UMopbjLIZQtdSNiNBBl6i0IeYIuSC5cU3WyYbtIsSnJJTGC5PdptDT1rpShgM/QrFsbxqR+c6InkQgjBdRfN4tuPnsVqVcdoRLEQ4B9DVzUsUBotobg6MRdkomyL3UJetdlj8LtnrPDZeycwVdJweJfdrzxKdmGB9kInoK8EdmpA+7tTvlR/paxDIh4Lj8JEUcNvXX8I33r0LL7kFLvxN5K844LhPfVeAVzr86+oKe4N95l1Oxl/XpChc0RuabMeW9H72iN78YYX7Mc/fecJvPqD38V/PbKMX/mnu/Dmj/0QF8yU8P++8rKW6+kFREB3cNGuUfzw6TU0m7RrVaKAl5Q7vW5fSK00ayD9IIko8AGdJUQlEsHQy41ENsUklqpuD91oR3IBgF84shcPnd50k5XdcrkwXHfRHDYbJv7rkbOxNw9+DJ3baTHisYWAkwOwk6J5VUrdG34ryKty5JAMliS94eJ5AMD5U0UUVDnSusiO+yEnoEfp6Fl7obvr45qXrTizRFs5fX7l+fuxb7qI9972CIAAQ9fC33ctQ/EO3xOdFbQFGTo/V/RMAkPXFAl/fPOl+D+vfw6ePFvBr/zTXbjnxBr++BWX4Mu/9cLEwrFeQgR0BzdcPIez5QbuO7neVqVcWrAA3tLlwgXxTjB03aIuKz9/quizLVJKbUtXCsmFad9ZXS4MN191HvKqhH+582kA3WXoAHDtoWkoEsHJtVqkLg54xUbluun1Qk/wofs09JgWyN1AKadE7qIYC2aTniSJ4Fm7RiOtix5DH/X9m0e7ll27HW4Tj5zZxB0PLfmGmcRBUyT8/o2H3cAb9KED/u87rYYO+KcWxTF0VlRGqT1Rq9Us25dcugtfeduL8I6XHsYdv3sd3nDtBe4AlX5A/6xkm/FTz5oDIcDXH1rqWi90wAsUpzdauVw6mxQ1TI+hH5obxWbDdLXAsvPzTILFjAV0NoSgHQ0dsBnYTZftdm+a3Q7oo3kVz3WmzcS9F18pnGRDzEf40IO90LuJ37/xIrw7Ymt/1fkTuPr8CVy8e9T93eH5URw7tYHgJEiWDGcMPapBV9Ze6Ax5VcZ9i2t41d9/Fxal+JOfSydDvPSyXTiybxIAQpWigN8mWmtR0cyDZ+jPrNWhyiTUimDEGUO3XG6gbjRbVvUC9gSuN//UwcS2BtsFEdAdTJU0XH3+JO54aAnr1e4FdFmyHTWs4CXW5cIx4KxFPDxY+1zdaroMnV3MLEGaxmKmOIUtLkPfQs/n1z53r/tzL9jtdRfZbpe4yl+WmOUDepTkIksEmiyFfOhp501uFZeeN+62deXxK8/fj8/+j2t9+YHDu0exWjWwvOnXyM9VDJQ0GXOjOdfuyMO2D2brhc6QV2U8uVLFnskCPv+Wa3HFQnitUSCE4E9+7nK8+jkL2Dvl+f+jbKLZGLriumROrdewazwfkoDY+feEM/UrWCU6aBABncP1h+dw38l1PLpkz+jsVq/u0bzqesyLMSemX3LZgsslQkO/MMDOvKKi5JM5qQNkFjzvgilcMFNCUZN7sl29zpEi4vR6vu8LawEQu3MKNJSqNKxYp9J2wk2MBmSXc5UGpkY0Z/apFkqOV505tu2c+9ccmMbLrtiNT//mC7BnorXcwuOiXaP4y9dc6duNsqDNFwfVMvRLKWkyqs7xPLVWj5SA2HE+ftYO6HEa+qBABHQO1x+2L/wv3HMSQHcYOuCxv6ImxyaNGAMmBFtKuLmFRWYT56p28RBjQUw/PeuW/cczdHtN0pYlF8BmZL/7MxfhF47sbf3gDuBZ8yO4cu8ELtkdnlYP8C4XjqHHBPRCoOVrRe8dQ8+Cw7ts+SVYMWonK+0b91RJC7WAWN/CYJd3vPQw/u4Xr+7YrssrDvInRQlJd/4VeMllvRbSzwHPzfS4Q+LmEvrWDwJafvOEkL0APgpgHgAFcAul9G8CjyEA/gbATQCqAN5AKf1h55fbXRzeNYrd43mX1XRL32Wvm6S9MoucKktb6vioce1zz5V1TBU1twMeY+YrgbaiccgpssvQtxLQAeBlV+zGy67ojW+XEIIvvOXa2L/nFNkdQxc3rYgh2MO70jDdxmP9hMmShvmxXMi6eK6iu0ErOPsU6N4s3XYQ1a6YjXhLc00UndYBzabdFjfocAG8a5Ex9H5xq7SLNFelCeB/UkovAXANgLcQQoLdoF4K4ELnvzcB+IeOrrJHIIS4LH0kp3RNDmAMJq73OOAx9K3o50DAtljVMVnSXCbutgEINP6PXZPqSS5ZhkQPAljXvbh5ogyFwJSduDYB/YDDu8ZCkstqgKEHfegsf9QtuTELojT0mm6lSogCng/9bLkBw6KJDP34cjm2SnSQ0DJaUEpPMbZNKd0EcAzAnsDDXgngo9TG9wFMEEK2v2yqDbCA3k2Gwi6WJIbOGPBW9HPAlmskYgf0c063ufGC6vOir1R0jOXDBStRa2K2vq0y9H5DyenpUWmYkCUS2wI1r25fUjQrDu8axWNLZbfTJqW2dZUlv6MZevZe6N1CMcrlEjGtKA7s5rvoDHNJ0tBPrNYGXj8HMmrohJD9AJ4N4M7An/YAOMH9exHhoD8QeMHBGeQUqSuNuRg8hp4Q0DnJZatQZQm6E9AnS+GE2NlyI5UFK6fIbjXtsAV0VjFYrpsoafFbel5yoZS1CejTgL57FLrVxHHHwVHV7cZqbCc2XdKwVjN8M1K7WYORFZ6G7u2IaoaVut84uyEwfZwNHuHBjp3VpAPvcAEyBHRCyAiAzwD4HUppfG/O5Nd4EyHkKCHk6PLycjsv0XUUNBk/e9UeXLW3ey0wPQ29teTSiYCuyZKdFOX6QfMJsZVyclGRtyYJ7NrfisulH+EG9IaVWL3KB/S6YQ8F7pUPPSsu2W2fw8dO2ZcrY+NTRe8coNSbdgS01wu9W8i7LhevY2I9i+TCEp7ODS3KecPbU9N40Psdqa5KQogKO5j/C6X0sxEPOQmAtywsOL/zgVJ6C6X0CKX0yOxsch/v7cSfv/oK/OnPX9G113ddLikkl04wYVWRUNMtbNRNt0yc326fLTdiB1v41hSYtD5MKLkaupF4o+U19KjhFv2EA7MlaLIUDujspu7synjZhTH0fth1RGroGSQXZgl+fLmMgipH3qR423CrKtFBQMto4ThYPgzgGKX0vTEP+yKAXyE2rgGwTik91cF1DhXcUXhJkovSOclFkyW31/OUMzF+esRLiK20KPv31iRzPw8ZQ3fG0NlJzmT3EdN0q3rvGnO1A1WWcOH8CB4MBvQRT3IB/I3aNupGVw0BWaDKBLJEQhp62j7truSyXMbuiXykjCY5hX7AcDD0NGfitQBeD+A+Qsg9zu/+AMD5AEAp/SCAr8C2LD4G27b4a51f6vCAtdBNarnKXCTqFlrnMqgKcXvH8A6H1aoB03G/tCoqAvxBvN1eLv0KNoZus5E8S5YfXNzvDB0ALtk9hjseXgIQllzYbo1n6Bu19qpEuwFCiNNCl9PQdQuzKUvumTTz1EoVzz8wHfu4kbw94GQYNPSWAZ1S+m0AiVGF2g0j3tKpRQ07mOSStK3tJENXZckN6JMOQ58q5bBa1bFS0UEpMJNSQ/d+7t8g1g7YGLpK3oy0tzHkFJlj6OEB0f2Gi3eP4d/uXsTSZj3M0EfCAX29ZnS9v04WBMfQ1Y1stkXATngGB1vwsI9ffKfFQcJw0awBwahbKdo7yeVsYAj0tJMQYw6AVBr6EEsubAzdpiM5xKGgyWiENPT+CYBBXOxUxz74zAZWKjpUmbg7xEiG3qVJXe0iWJmbSUPnAn9UURED+z4GvUoU2IFDovsBLGAk9QBhkstWC4sA/03BY+j2xfzIGbvwJJWG3qGGYf0IdlGfLevJGroiQ7ea+Nj3n3IDYb9q6ADcdgfHTm3afVxKmqsla4qE0bwSkFwMX4Os7YY9dYhzuRjNzBo6EG6by4Md70GvEgVEQN8WsGCa5HXvVGFR8DV4lwsAPOIw9HQ+dHtNmiy1HFwwaChxfuQkhn71vgmMF1T8r8/f7/6uHzzbcRgvqtgzUcCxUxuo6pZ7/BmmS/4GXe32Qu8W8prsNuQCHIaeUXIBgPMSGPpITnH/G3QM/icYQOybLuGW1z8HP/mseOtmpzV0wG4fy35mAf1Rh6Gn09Bl39qGCbwfOap1LsOLLpzFPe96MRZXa7j/5DpqhpVqkMN24uLddm/00bwS2onZ9lWv/L/daUXdQkGV3JF/VpNCN5vpK0V5hh5RVMRwePeYT6cfZIiAvk34mUt3Jf7dLSzqQPBkRUB8r5ZpV3IpQ5FIqovY9cYPmcMF8MtfrTRxQgj2ThX7SppIwiW7x3D7Q0uYH8vjiDPsg2GqpGFx1S6NN60mKrrVVzuOgiq7+R+mpWetFAWiy/4Z3v7iZ21hhf2F4bsyhwSqTEBIZ7RqTQ4H9Enn5/WakWr2I+AF8mHTzwFvDB3g6enDgot3j6FJgVPrdUwFZpLa9lU7YG72UR8XhqKmuOy5lmG4BWDvTFWZYCyv9HXiupMYvitzSEAIQU6ROpoU5QO6Kkuu3zqNBx3gJJch67QI+L3kw3bxX8z1gZ8KuJmmSjmcq+iglOJzP7KLu/upHXBe9eaCsv+nTYoCdvBP0s+HDSKg9zEKqtwReYPJNqGEmBPI0+jngLfVHUYNfZRj6P1cKNQOzp8qutWQUyPhpKhhUXzv+Ar+9NZj+G8Xz+GGi+e2Y5mRKGjehCgmDU0U052vgM3wkzzow4bhoiJDhr/6hStxwczIll+HuVyCF/NUScMTZystJxUxDHNSlA/ifHAfBkgSweHdY7j7qdXQsWa7trd+/EeYLuXwF6++cksDVToN27ZoB/Sv3HcKeVXCCw7GV30G8drn7sWB2VK3ltd3EAG9j3H94fmOvI6roRejL+b0kgtj6MPFYAG/s2XYGDpgO13ufmo1tEtjN/m1qo5P/MY1bm6lX1BQ7UpRw2ri1vtP4YaL5zNJYv/PECU802D4qJZACFEaOuA5XdIUFQFeUnQYXS5sDB2QbFscVFx2nt1KN9ivZMHRl992w7PwEwn9TrYLeUcq+ubDyzhb1vGKHo0tHFQM35krEEKUbRHwnC4zKcr+gc6NxetXjOQUrFaTS/8HFT939R5Mj+RwcNYv4V04P4qvvf2ncLBPZQnW3vZfj57ASE7BdRf1j77fjxjOK1PAB8bQg9vpzAx9iH3ogO1ukUh6W9wgIafIePEl0RLeobmRvtLNebDioNsfWsKLL5nP5HDZiRjOK1PAB82REuISYq2GQzN4SdHhvKhGcrZfuV+D204EC+Bmk+IVVwq5pRVEQN8BiGPo1x+ew5t/6iAu25Nu3F5uiG2LAIamn8cwge2WxgsqXniof6ec9QvE2bsD8Iorz0NBk0Pl/RNFDe946eHUr9PJsXj9iJG8COj9Bia53HjprqGbY9sNiLN3B2D/TAlvfNGBLb+OmxQd0gvr//7Jg+5MTYH+wJ6JAhSJ4NVHFrZ7KQMBEdAFUmOYfegA8PwMBSsCvcGB2RHc/+6XiGRoSgwn1RLoCoZdQxfoT4hgnh6CoQukRlFT8Hs3XoSXtGj9KyAgsD0QAV0gE/7HdYe2ewkCAgIxEHtnAQEBgSFBy4BOCPknQsgSIeT+mL9fRwhZJ4Tc4/z3rs4vU0BAQECgFdJILv8fgA8A+GjCY75FKX15R1YkICAgINAWWjJ0Sul/ATjXg7UICAgICGwBndLQn08I+TEh5FZCyKVxDyKEvIkQcpQQcnR5eblDby0gICAgAHQmoP8QwD5K6ZUA3g/g83EPpJTeQik9Qik9Mjsr+jIICAgIdBJbDuiU0g1Kadn5+SsAVELIzJZXJiAgICCQCVsO6ISQXcTpN0oIeZ7zmitbfV0BAQEBgWwglNLkBxDyCQDXAZgBcAbAHwFQAYBS+kFCyFsB/CYAE0ANwNsppd9t+caELAN4qs11zwA42+ZzBxk78XPvxM8M7MzPvRM/M5D9c++jlEZq1i0Dej+CEHKUUnpku9fRa+zEz70TPzOwMz/3TvzMQGc/t6gUFRAQEBgSiIAuICAgMCQY1IB+y3YvYJuwEz/3TvzMwM783DvxMwMd/NwDqaELCAgICIQxqAxdQEBAQCCAgQvohJAbCSEPE0IeI4S8Y7vX0w0Q8v+3cz4hVpVhGP89jGlp0IwtpGYEJxqKIUglYqII0RZpkS1aFEEuhDZBFkIUrVoGUSmEG6U0wqRJanAR1OS2qawwcyQ1JUfGdOE/2qj0tPg+4TJ2SPEeD/e77w8O57zfOXDe9z7nPtzznu8eLZS0W9J+Sb9JWpfH50v6WtLBvO5rOtc6kNQj6WdJu3I8KGkia75D0uymc2wnknoljUo6IGlS0kPdoLWkV/P1vU/Sdkk3l6j1f72xtkpfJTbm+vdKWnot5+ooQ5fUA3wArASGgeckDTebVS1cAtbbHgZGgJdyna8D47aHgPEcl8g6YLIlfht4z/bdwGlgbSNZ1ccG4Cvb9wL3k2ovWmtJ/cDLwAO27wN6gGcpU+uPgMdnjFXpuxIYysuLwKZrOVFHGTrwIHDI9h+2LwCfAqsbzqnt2J62/VPePk/6gveTat2aD9sKPN1MhvUhaQB4AticYwHLgdF8SFF1S7oNeBTYAmD7gu0zdIHWpNd33yJpFjAXmKZArSveWFul72pgmxPfAb2S7rjac3WaofcDx1riqTxWLJIWAUuACWCB7em86wSwoKG06uR94DXgnxzfDpyxfSnHpWk+CJwCPsxtps2S5lG41raPA+8Af5KM/Cywh7K1bqVK3+vyuE4z9K5C0q3A58Arts+17nOanlTUFCVJTwInbe9pOpcbyCxgKbDJ9hLgb2a0VwrVuo/0a3QQuBOYx5Vtia6gnfp2mqEfBxa2xAN5rDgk3UQy809s78zDf12+/crrk03lVxMPA09JOkpqpy0n9Zd78205lKf5FDBleyLHoySDL13rx4Ajtk/ZvgjsJOlfstatVOl7XR7XaYb+AzCUn4TPJj1EGWs4p7aT+8ZbgEnb77bsGgPW5O01wJc3Orc6sf2G7QHbi0jafmv7eWA38Ew+rKi6bZ8Ajkm6Jw+tAPZTuNakVsuIpLn5er9cd7Faz6BK3zHghTzbZQQ429Ka+X9sd9QCrAJ+Bw4DbzadT001PkK6BdsL/JKXVaR+8jhwEPgGmN90rjV+BsuAXXn7LuB74BDwGTCn6fzaXOti4Mes9xdAXzdoDbwFHAD2AR8Dc0rUGthOek5wkXRHtrZKX0CkmXyHgV9Js4Cu+lzxT9EgCIJC6LSWSxAEQVBBGHoQBEEhhKEHQRAUQhh6EARBIYShB0EQFEIYehAEQSGEoQdBEBRCGHoQBEEh/AvUTGskG0EllgAAAABJRU5ErkJggg==\n", 336 | "text/plain": [ 337 | "
" 338 | ] 339 | }, 340 | "metadata": { 341 | "needs_background": "light" 342 | } 343 | } 344 | ] 345 | }, 346 | { 347 | "cell_type": "code", 348 | "metadata": { 349 | "id": "Tlda0rBjX0vN" 350 | }, 351 | "source": [ 352 | "" 353 | ], 354 | "execution_count": null, 355 | "outputs": [] 356 | } 357 | ] 358 | } -------------------------------------------------------------------------------- /nlp/bert_classify.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | # bert文本分类baseline模型 3 | # model: bert 4 | # date: 2021.10.10 10:01 5 | 6 | import os 7 | import numpy as np 8 | import pandas as pd 9 | import torch 10 | import torch.nn as nn 11 | import torch.utils.data as Data 12 | import torch.optim as optim 13 | import transformers 14 | from transformers import AutoModel, AutoTokenizer 15 | import matplotlib.pyplot as plt 16 | 17 | train_curve = [] 18 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 19 | 20 | batch_size = 2 21 | epoches = 100 22 | model = "bert-base-chinese" 23 | hidden_size = 768 24 | n_class = 2 25 | maxlen = 8 26 | 27 | # data 28 | sentences = ["我喜欢打篮球", "这个相机很好看", "今天玩的特别开心", "我不喜欢你", "太糟糕了", "真是件令人伤心的事情"] 29 | labels = [1, 1, 1, 0, 0, 0] # 1积极, 0消极. 30 | 31 | # word_list = ' '.join(sentences).split() 32 | # word_list = list(set(word_list)) 33 | # word_dict = {w: i for i, w in enumerate(word_list)} 34 | # num_dict = {i: w for w, i in word_dict.items()} 35 | # vocab_size = len(word_list) 36 | 37 | class MyDataset(Data.Dataset): 38 | def __init__(self, sentences, labels=None, with_labels=True,): 39 | self.tokenizer = AutoTokenizer.from_pretrained(model) 40 | self.with_labels = with_labels 41 | self.sentences = sentences 42 | self.labels = labels 43 | def __len__(self): 44 | return len(sentences) 45 | 46 | def __getitem__(self, index): 47 | # Selecting sentence1 and sentence2 at the specified index in the data frame 48 | sent = self.sentences[index] 49 | 50 | # Tokenize the pair of sentences to get token ids, attention masks and token type ids 51 | encoded_pair = self.tokenizer(sent, 52 | padding='max_length', # Pad to max_length 53 | truncation=True, # Truncate to max_length 54 | max_length=maxlen, 55 | return_tensors='pt') # Return torch.Tensor objects 56 | 57 | token_ids = encoded_pair['input_ids'].squeeze(0) # tensor of token ids 58 | attn_masks = encoded_pair['attention_mask'].squeeze(0) # binary tensor with "0" for padded values and "1" for the other values 59 | token_type_ids = encoded_pair['token_type_ids'].squeeze(0) # binary tensor with "0" for the 1st sentence tokens & "1" for the 2nd sentence tokens 60 | 61 | if self.with_labels: # True if the dataset has labels 62 | label = self.labels[index] 63 | return token_ids, attn_masks, token_type_ids, label 64 | else: 65 | return token_ids, attn_masks, token_type_ids 66 | 67 | train = Data.DataLoader(dataset=MyDataset(sentences, labels), batch_size=batch_size, shuffle=True, num_workers=1) 68 | 69 | # model 70 | class BertClassify(nn.Module): 71 | def __init__(self): 72 | super(BertClassify, self).__init__() 73 | self.bert = AutoModel.from_pretrained(model, output_hidden_states=True, return_dict=True) 74 | self.linear = nn.Linear(hidden_size, n_class) 75 | self.dropout = nn.Dropout(0.5) 76 | 77 | def forward(self, X): 78 | input_ids, attention_mask, token_type_ids = X[0], X[1], X[2] 79 | outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids) # 返回一个output字典 80 | # 用最后一层cls向量做分类 81 | # outputs.pooler_output: [bs, hidden_size] 82 | logits = self.linear(self.dropout(outputs.pooler_output)) 83 | 84 | return logits 85 | 86 | bc = BertClassify().to(device) 87 | 88 | optimizer = optim.Adam(bc.parameters(), lr=1e-3, weight_decay=1e-2) 89 | loss_fn = nn.CrossEntropyLoss() 90 | 91 | # train 92 | sum_loss = 0 93 | total_step = len(train) 94 | for epoch in range(epoches): 95 | for i, batch in enumerate(train): 96 | optimizer.zero_grad() 97 | batch = tuple(p.to(device) for p in batch) 98 | pred = bc([batch[0], batch[1], batch[2]]) 99 | loss = loss_fn(pred, batch[3]) 100 | sum_loss += loss.item() 101 | 102 | loss.backward() 103 | optimizer.step() 104 | if epoch % 10 == 0: 105 | print('[{}|{}] step:{}/{} loss:{:.4f}'.format(epoch+1, epoches, i+1, total_step, loss.item())) 106 | train_curve.append(sum_loss) 107 | sum_loss = 0 108 | 109 | # test 110 | bc.eval() 111 | with torch.no_grad(): 112 | test_text = ['我不喜欢打篮球'] 113 | test = MyDataset(test_text, labels=None, with_labels=False) 114 | x = test.__getitem__(0) 115 | x = tuple(p.unsqueeze(0).to(device) for p in x) 116 | pred = bc([x[0], x[1], x[2]]) 117 | pred = pred.data.max(dim=1, keepdim=True)[1] 118 | if pred[0][0] == 0: 119 | print('消极') 120 | else: 121 | print('积极') 122 | 123 | pd.DataFrame(train_curve).plot() # loss曲线 124 | 125 | -------------------------------------------------------------------------------- /nlp/focal_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class FocalLoss(nn.Module): 5 | """Multi-class Focal loss implementation""" 6 | def __init__(self, gamma=2, weight=None, reduction='mean', ignore_index=-100): 7 | super(FocalLoss, self).__init__() 8 | self.gamma = gamma 9 | self.weight = weight 10 | self.ignore_index = ignore_index 11 | self.reduction = reduction 12 | 13 | def forward(self, input, target): 14 | """ 15 | input: [N, C] 16 | target: [N, ] 17 | """ 18 | log_pt = torch.log_softmax(input, dim=1) 19 | pt = torch.exp(log_pt) 20 | log_pt = (1 - pt) ** self.gamma * log_pt 21 | loss = torch.nn.functional.nll_loss(log_pt, target, self.weight, reduction=self.reduction, ignore_index=self.ignore_index) 22 | return loss --------------------------------------------------------------------------------