├── .gitignore ├── 9781789614381_ColorImages.pdf ├── Chapter01 └── mnist_pytorch.ipynb ├── Chapter02 ├── image_captioning_pytorch.ipynb └── sample.jpg ├── Chapter03 ├── DenseNetBlock.ipynb ├── GoogLeNet.ipynb ├── ResNetBlock.ipynb ├── imagenet1000_clsidx_to_labels.txt ├── lenet.ipynb ├── transfer_learning_alexnet.ipynb └── vgg13_pretrained_run_inference.ipynb ├── Chapter04 ├── lstm.ipynb └── rnn.ipynb ├── Chapter05 ├── out_of_the_box_transformers.ipynb ├── rand_wire_nn.ipynb ├── randwirenn.svg ├── randwirenn[representational_purpose_only].png └── transformer.ipynb ├── Chapter06 ├── music_generation.ipynb ├── text_generation.ipynb └── text_generation_out_of_the_box.ipynb ├── Chapter07 ├── images │ ├── content.jpg │ └── style.jpg └── neural_style_transfer.ipynb ├── Chapter08 ├── dcgan.ipynb └── pix2pix_architecture.ipynb ├── Chapter09 └── pong.ipynb ├── Chapter10 ├── Dockerfile ├── convnet.onnx ├── convnet.pb ├── convnet.pth ├── convnet.py ├── convnet_handler.py ├── cpp_convnet │ ├── CMakeLists.txt │ └── cpp_convnet.cpp ├── digit_image.jpg ├── example.py ├── make_request.py ├── mnist_pytorch.ipynb ├── model_scripting.ipynb ├── model_store │ └── convnet.mar ├── model_tracing.ipynb ├── onnx.ipynb ├── requirements.txt ├── run_inference.ipynb ├── scripted_convnet.pt ├── server.py └── traced_convnet.pt ├── Chapter11 ├── convnet_distributed.py ├── convnet_distributed_cuda.py └── convnet_undistributed.py ├── Chapter12 ├── automl-pytorch.ipynb └── optuna_pytorch.ipynb ├── Chapter13 ├── captum_interpretability.ipynb └── pytorch_interpretability.ipynb ├── Chapter14 ├── fastai.ipynb ├── poutyne.ipynb └── pytorch_lightning.ipynb ├── LICENSE ├── environment.yml └── environment_gpu.yml /.gitignore: -------------------------------------------------------------------------------- 1 | */.ipynb_checkpoints/*.* 2 | **/data_dir/* 3 | **/models_dir/* 4 | **/*data* 5 | */model_checkpoint 6 | */result_plots 7 | 8 | *.pt 9 | *.pth 10 | 11 | Chapter04/aclImdb 12 | Chapter05/cached_graph_obj 13 | Chapter05/image2* 14 | Chapter08/images_mnist 15 | -------------------------------------------------------------------------------- /9781789614381_ColorImages.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/9781789614381_ColorImages.pdf -------------------------------------------------------------------------------- /Chapter02/sample.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter02/sample.jpg -------------------------------------------------------------------------------- /Chapter03/DenseNetBlock.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch.nn as nn" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "class DenseBlock(nn.Module):\n", 19 | " def __init__(self, input_num_planes, rate_inc):\n", 20 | " super(DenseBlock, self).__init__()\n", 21 | " self.batch_norm1 = nn.BatchNorm2d(input_num_planes)\n", 22 | " self.conv_layer1 = nn.Conv2d(in_channels=input_num_planes, out_channels=4*rate_inc, kernel_size=1, bias=False)\n", 23 | " self.batch_norm2 = nn.BatchNorm2d(4*rate_inc)\n", 24 | " self.conv_layer2 = nn.Conv2d(in_channels=4*rate_inc, out_channels=rate_inc, kernel_size=3, padding=1, bias=False)\n", 25 | " def forward(self, inp):\n", 26 | " op = self.conv_layer1(F.relu(self.batch_norm1(inp)))\n", 27 | " op = self.conv_layer2(F.relu(self.batch_norm2(op)))\n", 28 | " op = torch.cat([op,inp], 1)\n", 29 | " return op\n", 30 | "\n", 31 | "class TransBlock(nn.Module):\n", 32 | " def __init__(self, input_num_planes, output_num_planes):\n", 33 | " super(TransBlock, self).__init__()\n", 34 | " self.batch_norm = nn.BatchNorm2d(input_num_planes)\n", 35 | " self.conv_layer = nn.Conv2d(in_channels=input_num_planes, out_channels=output_num_planes, kernel_size=1, bias=False)\n", 36 | " def forward(self, inp):\n", 37 | " op = self.conv_layer(F.relu(self.batch_norm(inp)))\n", 38 | " op = F.avg_pool2d(op, 2)\n", 39 | " return op" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": null, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [] 48 | } 49 | ], 50 | "metadata": { 51 | "kernelspec": { 52 | "display_name": "Python 3", 53 | "language": "python", 54 | "name": "python3" 55 | }, 56 | "language_info": { 57 | "codemirror_mode": { 58 | "name": "ipython", 59 | "version": 3 60 | }, 61 | "file_extension": ".py", 62 | "mimetype": "text/x-python", 63 | "name": "python", 64 | "nbconvert_exporter": "python", 65 | "pygments_lexer": "ipython3", 66 | "version": "3.7.6" 67 | } 68 | }, 69 | "nbformat": 4, 70 | "nbformat_minor": 4 71 | } 72 | -------------------------------------------------------------------------------- /Chapter03/GoogLeNet.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch.nn as nn" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "class InceptionModule(nn.Module):\n", 19 | " def __init__(self, input_planes, n_channels1x1, n_channels3x3red, n_channels3x3, n_channels5x5red, n_channels5x5, pooling_planes):\n", 20 | " super(InceptionModule, self).__init__()\n", 21 | " # 1x1 convolution branch\n", 22 | " self.block1 = nn.Sequential(\n", 23 | " nn.Conv2d(input_planes, n_channels1x1, kernel_size=1),\n", 24 | " nn.BatchNorm2d(n_channels1x1),\n", 25 | " nn.ReLU(True),\n", 26 | " )\n", 27 | " \n", 28 | " # 1x1 convolution -> 3x3 convolution branch\n", 29 | " self.block2 = nn.Sequential(\n", 30 | " nn.Conv2d(input_planes, n_channels3x3red, kernel_size=1),\n", 31 | " nn.BatchNorm2d(n_channels3x3red),\n", 32 | " nn.ReLU(True),\n", 33 | " nn.Conv2d(n_channels3x3red, n_channels3x3, kernel_size=3, padding=1),\n", 34 | " nn.BatchNorm2d(n_channels3x3),\n", 35 | " nn.ReLU(True),\n", 36 | " )\n", 37 | " \n", 38 | " # 1x1 conv -> 5x5 conv branch\n", 39 | " self.block3 = nn.Sequential(\n", 40 | " nn.Conv2d(input_planes, n_channels5x5red, kernel_size=1),\n", 41 | " nn.BatchNorm2d(n_channels5x5red),\n", 42 | " nn.ReLU(True),\n", 43 | " nn.Conv2d(n_channels5x5red, n_channels5x5, kernel_size=3, padding=1),\n", 44 | " nn.BatchNorm2d(n_channels5x5),\n", 45 | " nn.ReLU(True),\n", 46 | " nn.Conv2d(n_channels5x5, n_channels5x5, kernel_size=3, padding=1),\n", 47 | " nn.BatchNorm2d(n_channels5x5),\n", 48 | " nn.ReLU(True),\n", 49 | " )\n", 50 | " \n", 51 | " # 3x3 pool -> 1x1 conv branch\n", 52 | " self.block4 = nn.Sequential(\n", 53 | " nn.MaxPool2d(3, stride=1, padding=1),\n", 54 | " nn.Conv2d(input_planes, pooling_planes, kernel_size=1),\n", 55 | " nn.BatchNorm2d(pooling_planes),\n", 56 | " nn.ReLU(True),\n", 57 | " )\n", 58 | " \n", 59 | " def forward(self, ip):\n", 60 | " op1 = self.block1(ip)\n", 61 | " op2 = self.block2(ip)\n", 62 | " op3 = self.block3(ip)\n", 63 | " op4 = self.block4(ip)\n", 64 | " return torch.cat([op1,op2,op3,op4], 1)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 3, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "class GoogLeNet(nn.Module):\n", 74 | " def __init__(self):\n", 75 | " super(GoogLeNet, self).__init__()\n", 76 | " self.stem = nn.Sequential(\n", 77 | " nn.Conv2d(3, 192, kernel_size=3, padding=1),\n", 78 | " nn.BatchNorm2d(192),\n", 79 | " nn.ReLU(True),\n", 80 | " )\n", 81 | " \n", 82 | " self.im1 = InceptionModule(192, 64, 96, 128, 16, 32, 32)\n", 83 | " self.im2 = InceptionModule(256, 128, 128, 192, 32, 96, 64)\n", 84 | " \n", 85 | " self.max_pool = nn.MaxPool2d(3, stride=2, padding=1)\n", 86 | " \n", 87 | " self.im3 = InceptionModule(480, 192, 96, 208, 16, 48, 64)\n", 88 | " self.im4 = InceptionModule(512, 160, 112, 224, 24, 64, 64)\n", 89 | " self.im5 = InceptionModule(512, 128, 128, 256, 24, 64, 64)\n", 90 | " self.im6 = InceptionModule(512, 112, 144, 288, 32, 64, 64)\n", 91 | " self.im7 = InceptionModule(528, 256, 160, 320, 32, 128, 128)\n", 92 | " \n", 93 | " self.im8 = InceptionModule(832, 256, 160, 320, 32, 128, 128)\n", 94 | " self.im9 = InceptionModule(832, 384, 192, 384, 48, 128, 128)\n", 95 | " \n", 96 | " self.average_pool = nn.AvgPool2d(7, stride=1)\n", 97 | " self.fc = nn.Linear(4096, 1000)\n", 98 | " \n", 99 | " def forward(self, ip):\n", 100 | " op = self.stem(ip)\n", 101 | " out = self.im1(op)\n", 102 | " out = self.im2(op)\n", 103 | " op = self.maxpool(op)\n", 104 | " op = self.a4(op)\n", 105 | " op = self.b4(op)\n", 106 | " op = self.c4(op)\n", 107 | " op = self.d4(op)\n", 108 | " op = self.e4(op)\n", 109 | " op = self.max_pool(op)\n", 110 | " op = self.a5(op)\n", 111 | " op = self.b5(op)\n", 112 | " op = self.avgerage_pool(op)\n", 113 | " op = op.view(op.size(0), -1)\n", 114 | " op = self.fc(op)\n", 115 | " return op" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [] 124 | } 125 | ], 126 | "metadata": { 127 | "kernelspec": { 128 | "display_name": "Python 3", 129 | "language": "python", 130 | "name": "python3" 131 | }, 132 | "language_info": { 133 | "codemirror_mode": { 134 | "name": "ipython", 135 | "version": 3 136 | }, 137 | "file_extension": ".py", 138 | "mimetype": "text/x-python", 139 | "name": "python", 140 | "nbconvert_exporter": "python", 141 | "pygments_lexer": "ipython3", 142 | "version": "3.7.6" 143 | } 144 | }, 145 | "nbformat": 4, 146 | "nbformat_minor": 4 147 | } 148 | -------------------------------------------------------------------------------- /Chapter03/ResNetBlock.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch.nn as nn" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "class BasicBlock(nn.Module):\n", 19 | " multiplier=1\n", 20 | " def __init__(self, input_num_planes, num_planes, strd=1):\n", 21 | " super(BasicBlock, self).__init__()\n", 22 | " self.conv_layer1 = nn.Conv2d(in_channels=input_num_planes, out_channels=num_planes, kernel_size=3, stride=stride, padding=1, bias=False)\n", 23 | " self.batch_norm1 = nn.BatchNorm2d(num_planes)\n", 24 | " self.conv_layer2 = nn.Conv2d(in_channels=num_planes, out_channels=num_planes, kernel_size=3, stride=1, padding=1, bias=False)\n", 25 | " self.batch_norm2 = nn.BatchNorm2d(num_planes)\n", 26 | " \n", 27 | " self.res_connnection = nn.Sequential()\n", 28 | " if strd > 1 or input_num_planes != self.multiplier*num_planes:\n", 29 | " self.res_connnection = nn.Sequential(\n", 30 | " nn.Conv2d(in_channels=input_num_planes, out_channels=self.multiplier*num_planes, kernel_size=1, stride=strd, bias=False),\n", 31 | " nn.BatchNorm2d(self.multiplier*num_planes)\n", 32 | " )\n", 33 | " def forward(self, inp):\n", 34 | " op = F.relu(self.batch_norm1(self.conv_layer1(inp)))\n", 35 | " op = self.batch_norm2(self.conv_layer2(op))\n", 36 | " op += self.res_connnection(inp)\n", 37 | " op = F.relu(op)\n", 38 | " return op" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [] 47 | } 48 | ], 49 | "metadata": { 50 | "kernelspec": { 51 | "display_name": "Python 3", 52 | "language": "python", 53 | "name": "python3" 54 | }, 55 | "language_info": { 56 | "codemirror_mode": { 57 | "name": "ipython", 58 | "version": 3 59 | }, 60 | "file_extension": ".py", 61 | "mimetype": "text/x-python", 62 | "name": "python", 63 | "nbconvert_exporter": "python", 64 | "pygments_lexer": "ipython3", 65 | "version": "3.7.6" 66 | } 67 | }, 68 | "nbformat": 4, 69 | "nbformat_minor": 4 70 | } 71 | -------------------------------------------------------------------------------- /Chapter04/lstm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "text/plain": [ 11 | "" 12 | ] 13 | }, 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "output_type": "execute_result" 17 | } 18 | ], 19 | "source": [ 20 | "import os\n", 21 | "import time\n", 22 | "import numpy as np\n", 23 | "from tqdm import tqdm\n", 24 | "from string import punctuation\n", 25 | "from collections import Counter\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "\n", 28 | "import torch\n", 29 | "import torch.nn as nn\n", 30 | "import torch.optim as optim\n", 31 | "from torch.utils.data import DataLoader, TensorDataset\n", 32 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", 33 | "torch.manual_seed(123)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 2, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "import random\n", 43 | "from torchtext import (data, datasets)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 3, 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "name": "stdout", 53 | "output_type": "stream", 54 | "text": [ 55 | "downloading aclImdb_v1.tar.gz\n" 56 | ] 57 | }, 58 | { 59 | "name": "stderr", 60 | "output_type": "stream", 61 | "text": [ 62 | ".data\\imdb\\aclImdb_v1.tar.gz: 100%|███████████████████████████████████████████████| 84.1M/84.1M [00:10<00:00, 7.88MB/s]\n" 63 | ] 64 | } 65 | ], 66 | "source": [ 67 | "TEXT_FIELD = data.Field(tokenize = data.get_tokenizer(\"basic_english\"), include_lengths = True)\n", 68 | "LABEL_FIELD = data.LabelField(dtype = torch.float)\n", 69 | "\n", 70 | "train_dataset, test_dataset = datasets.IMDB.splits(TEXT_FIELD, LABEL_FIELD)\n", 71 | "train_dataset, valid_dataset = train_dataset.split(random_state = random.seed(123))" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 4, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "MAX_VOCABULARY_SIZE = 25000\n", 81 | "\n", 82 | "TEXT_FIELD.build_vocab(train_dataset, \n", 83 | " max_size = MAX_VOCABULARY_SIZE)\n", 84 | "\n", 85 | "LABEL_FIELD.build_vocab(train_dataset)" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": 5, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "B_SIZE = 64\n", 95 | "\n", 96 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", 97 | "\n", 98 | "train_data_iterator, valid_data_iterator, test_data_iterator = data.BucketIterator.splits(\n", 99 | " (train_dataset, valid_dataset, test_dataset), \n", 100 | " batch_size = B_SIZE,\n", 101 | " sort_within_batch = True,\n", 102 | " device = device)" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 6, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "## If you are training using GPUs, we need to use the following function for the pack_padded_sequence method to work \n", 112 | "## (reference : https://discuss.pytorch.org/t/error-with-lengths-in-pack-padded-sequence/35517/3)\n", 113 | "if torch.cuda.is_available():\n", 114 | " torch.set_default_tensor_type(torch.cuda.FloatTensor)\n", 115 | "from torch.nn.utils.rnn import pack_padded_sequence, PackedSequence\n", 116 | "\n", 117 | "def cuda_pack_padded_sequence(input, lengths, batch_first=False, enforce_sorted=True):\n", 118 | " lengths = torch.as_tensor(lengths, dtype=torch.int64)\n", 119 | " lengths = lengths.cpu()\n", 120 | " if enforce_sorted:\n", 121 | " sorted_indices = None\n", 122 | " else:\n", 123 | " lengths, sorted_indices = torch.sort(lengths, descending=True)\n", 124 | " sorted_indices = sorted_indices.to(input.device)\n", 125 | " batch_dim = 0 if batch_first else 1\n", 126 | " input = input.index_select(batch_dim, sorted_indices)\n", 127 | "\n", 128 | " data, batch_sizes = \\\n", 129 | " torch._C._VariableFunctions._pack_padded_sequence(input, lengths, batch_first)\n", 130 | " return PackedSequence(data, batch_sizes, sorted_indices)" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 7, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "name": "stderr", 140 | "output_type": "stream", 141 | "text": [ 142 | "C:\\Users\\sk8er\\anaconda3\\envs\\mpytc\\lib\\site-packages\\torch\\nn\\modules\\rnn.py:50: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n", 143 | " \"num_layers={}\".format(dropout, num_layers))\n" 144 | ] 145 | } 146 | ], 147 | "source": [ 148 | "class LSTM(nn.Module):\n", 149 | " def __init__(self, vocabulary_size, embedding_dimension, hidden_dimension, output_dimension, dropout, pad_index):\n", 150 | " super().__init__()\n", 151 | " self.embedding_layer = nn.Embedding(vocabulary_size, embedding_dimension, padding_idx = pad_index)\n", 152 | " self.lstm_layer = nn.LSTM(embedding_dimension, \n", 153 | " hidden_dimension, \n", 154 | " num_layers=1, \n", 155 | " bidirectional=True, \n", 156 | " dropout=dropout)\n", 157 | " self.fc_layer = nn.Linear(hidden_dimension * 2, output_dimension)\n", 158 | " self.dropout_layer = nn.Dropout(dropout)\n", 159 | " \n", 160 | " def forward(self, sequence, sequence_lengths=None):\n", 161 | " if sequence_lengths is None:\n", 162 | " sequence_lengths = torch.LongTensor([len(sequence)])\n", 163 | " \n", 164 | " # sequence := (sequence_length, batch_size)\n", 165 | " embedded_output = self.dropout_layer(self.embedding_layer(sequence))\n", 166 | " \n", 167 | " \n", 168 | " # embedded_output := (sequence_length, batch_size, embedding_dimension)\n", 169 | " if torch.cuda.is_available():\n", 170 | " packed_embedded_output = cuda_pack_padded_sequence(embedded_output, sequence_lengths)\n", 171 | " else:\n", 172 | " packed_embedded_output = nn.utils.rnn.pack_padded_sequence(embedded_output, sequence_lengths)\n", 173 | " \n", 174 | " packed_output, (hidden_state, cell_state) = self.lstm_layer(packed_embedded_output)\n", 175 | " # hidden_state := (num_layers * num_directions, batch_size, hidden_dimension)\n", 176 | " # cell_state := (num_layers * num_directions, batch_size, hidden_dimension)\n", 177 | " \n", 178 | " op, op_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n", 179 | " # op := (sequence_length, batch_size, hidden_dimension * num_directions)\n", 180 | " \n", 181 | " hidden_output = torch.cat((hidden_state[-2,:,:], hidden_state[-1,:,:]), dim = 1) \n", 182 | " # hidden_output := (batch_size, hidden_dimension * num_directions)\n", 183 | " \n", 184 | " return self.fc_layer(hidden_output)\n", 185 | "\n", 186 | " \n", 187 | "INPUT_DIMENSION = len(TEXT_FIELD.vocab)\n", 188 | "EMBEDDING_DIMENSION = 100\n", 189 | "HIDDEN_DIMENSION = 32\n", 190 | "OUTPUT_DIMENSION = 1\n", 191 | "DROPOUT = 0.5\n", 192 | "PAD_INDEX = TEXT_FIELD.vocab.stoi[TEXT_FIELD.pad_token]\n", 193 | "\n", 194 | "lstm_model = LSTM(INPUT_DIMENSION, \n", 195 | " EMBEDDING_DIMENSION, \n", 196 | " HIDDEN_DIMENSION, \n", 197 | " OUTPUT_DIMENSION, \n", 198 | " DROPOUT, \n", 199 | " PAD_INDEX)" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": 8, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "UNK_INDEX = TEXT_FIELD.vocab.stoi[TEXT_FIELD.unk_token]\n", 209 | "\n", 210 | "lstm_model.embedding_layer.weight.data[UNK_INDEX] = torch.zeros(EMBEDDING_DIMENSION)\n", 211 | "lstm_model.embedding_layer.weight.data[PAD_INDEX] = torch.zeros(EMBEDDING_DIMENSION)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 9, 217 | "metadata": {}, 218 | "outputs": [], 219 | "source": [ 220 | "optim = torch.optim.Adam(lstm_model.parameters())\n", 221 | "loss_func = nn.BCEWithLogitsLoss()\n", 222 | "\n", 223 | "lstm_model = lstm_model.to(device)\n", 224 | "loss_func = loss_func.to(device)" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 10, 230 | "metadata": {}, 231 | "outputs": [], 232 | "source": [ 233 | "def accuracy_metric(predictions, ground_truth):\n", 234 | " \"\"\"\n", 235 | " Returns 0-1 accuracy for the given set of predictions and ground truth\n", 236 | " \"\"\"\n", 237 | " # round predictions to either 0 or 1\n", 238 | " rounded_predictions = torch.round(torch.sigmoid(predictions))\n", 239 | " success = (rounded_predictions == ground_truth).float() #convert into float for division \n", 240 | " accuracy = success.sum() / len(success)\n", 241 | " return accuracy" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": 11, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "def train(model, data_iterator, optim, loss_func):\n", 251 | " loss = 0\n", 252 | " accuracy = 0\n", 253 | " model.train()\n", 254 | " \n", 255 | " for curr_batch in data_iterator:\n", 256 | " optim.zero_grad()\n", 257 | " sequence, sequence_lengths = curr_batch.text\n", 258 | " preds = lstm_model(sequence, sequence_lengths).squeeze(1)\n", 259 | " \n", 260 | " loss_curr = loss_func(preds, curr_batch.label)\n", 261 | " accuracy_curr = accuracy_metric(preds, curr_batch.label)\n", 262 | " \n", 263 | " loss_curr.backward()\n", 264 | " optim.step()\n", 265 | " \n", 266 | " loss += loss_curr.item()\n", 267 | " accuracy += accuracy_curr.item()\n", 268 | " \n", 269 | " return loss/len(data_iterator), accuracy/len(data_iterator)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 12, 275 | "metadata": {}, 276 | "outputs": [], 277 | "source": [ 278 | "def validate(model, data_iterator, loss_func):\n", 279 | " loss = 0\n", 280 | " accuracy = 0\n", 281 | " model.eval()\n", 282 | " \n", 283 | " with torch.no_grad():\n", 284 | " for curr_batch in data_iterator:\n", 285 | " sequence, sequence_lengths = curr_batch.text\n", 286 | " preds = model(sequence, sequence_lengths).squeeze(1)\n", 287 | " \n", 288 | " loss_curr = loss_func(preds, curr_batch.label)\n", 289 | " accuracy_curr = accuracy_metric(preds, curr_batch.label)\n", 290 | "\n", 291 | " loss += loss_curr.item()\n", 292 | " accuracy += accuracy_curr.item()\n", 293 | " \n", 294 | " return loss/len(data_iterator), accuracy/len(data_iterator)" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 13, 300 | "metadata": {}, 301 | "outputs": [ 302 | { 303 | "name": "stdout", 304 | "output_type": "stream", 305 | "text": [ 306 | "epoch number: 1 | time elapsed: 10.499292135238647s\n", 307 | "training loss: 0.688 | training accuracy: 54.09%\n", 308 | "validation loss: 0.674 | validation accuracy: 58.18%\n", 309 | "\n", 310 | "epoch number: 2 | time elapsed: 10.434584379196167s\n", 311 | "training loss: 0.663 | training accuracy: 60.19%\n", 312 | "validation loss: 0.611 | validation accuracy: 66.78%\n", 313 | "\n", 314 | "epoch number: 3 | time elapsed: 10.492015361785889s\n", 315 | "training loss: 0.591 | training accuracy: 68.11%\n", 316 | "validation loss: 0.620 | validation accuracy: 71.48%\n", 317 | "\n", 318 | "epoch number: 4 | time elapsed: 10.2067391872406s\n", 319 | "training loss: 0.529 | training accuracy: 73.84%\n", 320 | "validation loss: 0.588 | validation accuracy: 74.06%\n", 321 | "\n", 322 | "epoch number: 5 | time elapsed: 10.220768213272095s\n", 323 | "training loss: 0.471 | training accuracy: 77.72%\n", 324 | "validation loss: 0.655 | validation accuracy: 73.51%\n", 325 | "\n", 326 | "epoch number: 6 | time elapsed: 10.350415468215942s\n", 327 | "training loss: 0.426 | training accuracy: 80.57%\n", 328 | "validation loss: 0.562 | validation accuracy: 78.74%\n", 329 | "\n", 330 | "epoch number: 7 | time elapsed: 10.357213735580444s\n", 331 | "training loss: 0.391 | training accuracy: 82.64%\n", 332 | "validation loss: 0.637 | validation accuracy: 76.60%\n", 333 | "\n", 334 | "epoch number: 8 | time elapsed: 10.276661396026611s\n", 335 | "training loss: 0.377 | training accuracy: 83.39%\n", 336 | "validation loss: 0.543 | validation accuracy: 80.62%\n", 337 | "\n", 338 | "epoch number: 9 | time elapsed: 10.23417329788208s\n", 339 | "training loss: 0.353 | training accuracy: 84.77%\n", 340 | "validation loss: 0.506 | validation accuracy: 78.40%\n", 341 | "\n", 342 | "epoch number: 10 | time elapsed: 10.402759075164795s\n", 343 | "training loss: 0.325 | training accuracy: 86.44%\n", 344 | "validation loss: 0.557 | validation accuracy: 79.22%\n", 345 | "\n" 346 | ] 347 | } 348 | ], 349 | "source": [ 350 | "num_epochs = 10\n", 351 | "best_validation_loss = float('inf')\n", 352 | "\n", 353 | "for ep in range(num_epochs):\n", 354 | "\n", 355 | " time_start = time.time()\n", 356 | " \n", 357 | " training_loss, train_accuracy = train(lstm_model, train_data_iterator, optim, loss_func)\n", 358 | " validation_loss, validation_accuracy = validate(lstm_model, valid_data_iterator, loss_func)\n", 359 | " \n", 360 | " time_end = time.time()\n", 361 | " time_delta = time_end - time_start \n", 362 | " \n", 363 | " if validation_loss < best_validation_loss:\n", 364 | " best_validation_loss = validation_loss\n", 365 | " torch.save(lstm_model.state_dict(), 'lstm_model.pt')\n", 366 | " \n", 367 | " print(f'epoch number: {ep+1} | time elapsed: {time_delta}s')\n", 368 | " print(f'training loss: {training_loss:.3f} | training accuracy: {train_accuracy*100:.2f}%')\n", 369 | " print(f'validation loss: {validation_loss:.3f} | validation accuracy: {validation_accuracy*100:.2f}%')\n", 370 | " print()" 371 | ] 372 | }, 373 | { 374 | "cell_type": "code", 375 | "execution_count": 14, 376 | "metadata": {}, 377 | "outputs": [ 378 | { 379 | "name": "stdout", 380 | "output_type": "stream", 381 | "text": [ 382 | "test loss: 0.544 | test accuracy: 77.04%\n" 383 | ] 384 | } 385 | ], 386 | "source": [ 387 | "#lstm_model.load_state_dict(torch.load('../../mastering_pytorch_packt/04_deep_recurrent_net_architectures/lstm_model.pt'))\n", 388 | "lstm_model.load_state_dict(torch.load('lstm_model.pt'))\n", 389 | "\n", 390 | "test_loss, test_accuracy = validate(lstm_model, test_data_iterator, loss_func)\n", 391 | "\n", 392 | "print(f'test loss: {test_loss:.3f} | test accuracy: {test_accuracy*100:.2f}%')" 393 | ] 394 | }, 395 | { 396 | "cell_type": "code", 397 | "execution_count": 15, 398 | "metadata": {}, 399 | "outputs": [], 400 | "source": [ 401 | "def sentiment_inference(model, sentence):\n", 402 | " model.eval()\n", 403 | " \n", 404 | " # text transformations\n", 405 | " tokenized = data.get_tokenizer(\"basic_english\")(sentence)\n", 406 | " tokenized = [TEXT_FIELD.vocab.stoi[t] for t in tokenized]\n", 407 | " \n", 408 | " # model inference\n", 409 | " model_input = torch.LongTensor(tokenized).to(device)\n", 410 | " model_input = model_input.unsqueeze(1)\n", 411 | " \n", 412 | " pred = torch.sigmoid(model(model_input))\n", 413 | " \n", 414 | " return pred.item()" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": 16, 420 | "metadata": { 421 | "scrolled": true 422 | }, 423 | "outputs": [ 424 | { 425 | "name": "stdout", 426 | "output_type": "stream", 427 | "text": [ 428 | "0.07853574305772781\n", 429 | "0.01614055596292019\n", 430 | "0.325915664434433\n", 431 | "0.8385397791862488\n", 432 | "0.942153811454773\n" 433 | ] 434 | } 435 | ], 436 | "source": [ 437 | "print(sentiment_inference(lstm_model, \"This film is horrible\"))\n", 438 | "print(sentiment_inference(lstm_model, \"Director tried too hard but this film is bad\"))\n", 439 | "print(sentiment_inference(lstm_model, \"Decent movie, although could be shorter\"))\n", 440 | "print(sentiment_inference(lstm_model, \"This film will be houseful for weeks\"))\n", 441 | "print(sentiment_inference(lstm_model, \"I loved the movie, every part of it\"))" 442 | ] 443 | }, 444 | { 445 | "cell_type": "code", 446 | "execution_count": null, 447 | "metadata": {}, 448 | "outputs": [], 449 | "source": [] 450 | } 451 | ], 452 | "metadata": { 453 | "kernelspec": { 454 | "display_name": "Python 3 (ipykernel)", 455 | "language": "python", 456 | "name": "python3" 457 | }, 458 | "language_info": { 459 | "codemirror_mode": { 460 | "name": "ipython", 461 | "version": 3 462 | }, 463 | "file_extension": ".py", 464 | "mimetype": "text/x-python", 465 | "name": "python", 466 | "nbconvert_exporter": "python", 467 | "pygments_lexer": "ipython3", 468 | "version": "3.7.10" 469 | } 470 | }, 471 | "nbformat": 4, 472 | "nbformat_minor": 4 473 | } 474 | -------------------------------------------------------------------------------- /Chapter04/rnn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "text/plain": [ 11 | "" 12 | ] 13 | }, 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "output_type": "execute_result" 17 | } 18 | ], 19 | "source": [ 20 | "import os\n", 21 | "import time\n", 22 | "import numpy as np\n", 23 | "from tqdm import tqdm\n", 24 | "from string import punctuation\n", 25 | "from collections import Counter\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "\n", 28 | "import torch\n", 29 | "import torch.nn as nn\n", 30 | "import torch.optim as optim\n", 31 | "from torch.utils.data import DataLoader, TensorDataset\n", 32 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", 33 | "torch.manual_seed(123)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "## Dataset Instructions:\n", 41 | "\n", 42 | "- You need to download the dataset from here: https://ai.stanford.edu/~amaas/data/sentiment/\n", 43 | "- Then, you need to unzip the downloaded zipped file in the present working directory\n", 44 | "- That should result in a new folder named aclImdb under the present working directory" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 4, 50 | "metadata": {}, 51 | "outputs": [ 52 | { 53 | "name": "stderr", 54 | "output_type": "stream", 55 | "text": [ 56 | "100%|██████████| 12500/12500 [00:00<00:00, 13570.44it/s]\n", 57 | "100%|██████████| 12500/12500 [00:00<00:00, 14500.84it/s]" 58 | ] 59 | }, 60 | { 61 | "name": "stdout", 62 | "output_type": "stream", 63 | "text": [ 64 | "Number of reviews : 25000\n" 65 | ] 66 | }, 67 | { 68 | "name": "stderr", 69 | "output_type": "stream", 70 | "text": [ 71 | "\n" 72 | ] 73 | } 74 | ], 75 | "source": [ 76 | "# read sentiments and reviews data from the text files\n", 77 | "review_list = []\n", 78 | "label_list = []\n", 79 | "for label in ['pos', 'neg']:\n", 80 | " for fname in tqdm(os.listdir(f'./aclImdb/train/{label}/')):\n", 81 | " if 'txt' not in fname:\n", 82 | " continue\n", 83 | " with open(os.path.join(f'./aclImdb/train/{label}/', fname), encoding=\"utf8\") as f:\n", 84 | " review_list += [f.read()]\n", 85 | " label_list += [label]\n", 86 | "print ('Number of reviews :', len(review_list))" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 3, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "name": "stderr", 96 | "output_type": "stream", 97 | "text": [ 98 | "100%|██████████| 25000/25000 [00:01<00:00, 13231.57it/s]\n" 99 | ] 100 | }, 101 | { 102 | "name": "stdout", 103 | "output_type": "stream", 104 | "text": [ 105 | "[('the', 334691), ('and', 162228), ('a', 161940), ('of', 145326), ('to', 135042), ('is', 106855), ('in', 93028), ('it', 77099), ('i', 75719), ('this', 75190)]\n" 106 | ] 107 | } 108 | ], 109 | "source": [ 110 | "# pre-processing review text\n", 111 | "review_list = [review.lower() for review in review_list]\n", 112 | "review_list = [''.join([letter for letter in review if letter not in punctuation]) for review in tqdm(review_list)]\n", 113 | "\n", 114 | "# accumulate all review texts together\n", 115 | "reviews_blob = ' '.join(review_list)\n", 116 | "\n", 117 | "# generate list of all words of all reviews\n", 118 | "review_words = reviews_blob.split()\n", 119 | "\n", 120 | "# get the word counts\n", 121 | "count_words = Counter(review_words)\n", 122 | "\n", 123 | "# sort words as per counts (decreasing order)\n", 124 | "total_review_words = len(review_words)\n", 125 | "sorted_review_words = count_words.most_common(total_review_words)\n", 126 | "\n", 127 | "print(sorted_review_words[:10])" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 4, 133 | "metadata": {}, 134 | "outputs": [ 135 | { 136 | "name": "stdout", 137 | "output_type": "stream", 138 | "text": [ 139 | "[('the', 1), ('and', 2), ('a', 3), ('of', 4), ('to', 5), ('is', 6), ('in', 7), ('it', 8), ('i', 9), ('this', 10)]\n" 140 | ] 141 | } 142 | ], 143 | "source": [ 144 | "# create word to integer (token) dictionary in order to encode text as numbers\n", 145 | "vocab_to_token = {word:idx+1 for idx, (word, count) in enumerate(sorted_review_words)}\n", 146 | "print(list(vocab_to_token.items())[:10])" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 5, 152 | "metadata": {}, 153 | "outputs": [ 154 | { 155 | "name": "stdout", 156 | "output_type": "stream", 157 | "text": [ 158 | "for a movie that gets no respect there sure are a lot of memorable quotes listed for this gem imagine a movie where joe piscopo is actually funny maureen stapleton is a scene stealer the moroni character is an absolute scream watch for alan the skipper hale jr as a police sgt\n", 159 | "\n", 160 | "[15, 3, 17, 11, 201, 56, 1165, 47, 242, 23, 3, 168, 4, 891, 4325, 3513, 15, 10, 1514, 822, 3, 17, 112, 884, 14623, 6, 155, 161, 7307, 15816, 6, 3, 134, 20049, 1, 32064, 108, 6, 33, 1492, 1943, 103, 15, 1550, 1, 18993, 9055, 1809, 14, 3, 549, 6906]\n" 161 | ] 162 | } 163 | ], 164 | "source": [ 165 | "reviews_tokenized = []\n", 166 | "for review in review_list:\n", 167 | " word_to_token = [vocab_to_token[word] for word in review.split()]\n", 168 | " reviews_tokenized.append(word_to_token)\n", 169 | "print(review_list[0])\n", 170 | "print()\n", 171 | "print (reviews_tokenized[0])" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 6, 177 | "metadata": {}, 178 | "outputs": [], 179 | "source": [ 180 | "# encode sentiments as 0 or 1\n", 181 | "encoded_label_list = [1 if label =='pos' else 0 for label in label_list]\n", 182 | "\n", 183 | "reviews_len = [len(review) for review in reviews_tokenized]\n", 184 | "\n", 185 | "reviews_tokenized = [reviews_tokenized[i] for i, l in enumerate(reviews_len) if l>0 ]\n", 186 | "encoded_label_list = np.array([encoded_label_list[i] for i, l in enumerate(reviews_len) if l> 0 ], dtype='float32')" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 7, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "data": { 196 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAATmUlEQVR4nO3df6xf9X3f8edrdkFVmggT7iwXw+xkTiUSbQ5YBGlJlI0FDJlqMlXUaCpOiuJEAanRNq1mmQRKi0S60mhoGRVZrJgphbBShtU4Iw6KiiYNwiVxwSYhXIgRtoztYhaypaKFvPfH93M//ca5176+3+t7bd/nQzr6nu/7fM45n889vn75/Ph+napCkiSAv7fQHZAknToMBUlSZyhIkjpDQZLUGQqSpG7pQndgts4777xatWrVQndDkk4rTz755F9V1dh0y0/bUFi1ahXj4+ML3Q1JOq0kefFYy718JEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJ3XFDIcnWJIeS7B6qfS3JrjbtTbKr1Vcl+euhZX88tM4lSZ5OMpHkziRp9XOT7EzyXHtddjIGKkk6vpl8ovkrwH8G7pksVNVvTs4nuQP48VD756tq7RTbuQv4BPA4sANYD3wD2AI8UlW3J9nS3v/uiQ3jxKza8vWTuflp7b39IwuyX0maqeOeKVTVo8CRqZa1f+1fC9x7rG0kWQG8raoeq8F/9XYPcE1bvAHY1ua3DdUlSfNs1HsKHwAOVtVzQ7XVSb6X5C+SfKDVzgf2DbXZ12oAy6vqQJt/GVg+3c6SbE4ynmT88OHDI3ZdknS0UUPhOn7+LOEAcGFVvRf418CfJHnbTDfWziKm/U+jq+ruqlpXVevGxqb9kj9J0izN+ltSkywF/iVwyWStql4HXm/zTyZ5HngXsB9YObT6ylYDOJhkRVUdaJeZDs22T5Kk0YxypvDPgR9UVb8slGQsyZI2/w5gDfBCuzz0WpLL2n2I64GH2mrbgU1tftNQXZI0z2bySOq9wP8Gfi3JviQ3tEUb+cUbzB8EnmqPqP4p8KmqmrxJ/WngvwITwPMMnjwCuB34cJLnGATN7SOMR5I0guNePqqq66apf2yK2gPAA9O0HwfeM0X9FeDy4/VDknTy+YlmSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpO64oZBka5JDSXYP1W5Nsj/JrjZdPbTs5iQTSZ5NcuVQfX2rTSTZMlRfneTxVv9akrPmcoCSpJmbyZnCV4D1U9S/UFVr27QDIMlFwEbg3W2d/5JkSZIlwBeBq4CLgOtaW4DPt239Q+BV4IZRBiRJmr3jhkJVPQocmeH2NgD3VdXrVfUjYAK4tE0TVfVCVf0NcB+wIUmAfwb8aVt/G3DNCY5BkjRHRrmncFOSp9rlpWWtdj7w0lCbfa02Xf3twP+pqjeOqk8pyeYk40nGDx8+PELXJUlTmW0o3AW8E1gLHADumLMeHUNV3V1V66pq3djY2HzsUpIWlaWzWamqDk7OJ/kS8Oft7X7ggqGmK1uNaeqvAOckWdrOFobbS5Lm2azOFJKsGHr7UWDyyaTtwMYkZydZDawBvgM8AaxpTxqdxeBm9PaqKuDbwG+09TcBD82mT5Kk0R33TCHJvcCHgPOS7ANuAT6UZC1QwF7gkwBVtSfJ/cAzwBvAjVX1ZtvOTcDDwBJga1Xtabv4XeC+JL8PfA/48pyNTpJ0Qo4bClV13RTlaf/irqrbgNumqO8AdkxRf4HB00mSpAXmJ5olSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSuuOGQpKtSQ4l2T1U+49JfpDkqSQPJjmn1Vcl+esku9r0x0PrXJLk6SQTSe5MklY/N8nOJM+112UnY6CSpOObyZnCV4D1R9V2Au+pqn8E/BC4eWjZ81W1tk2fGqrfBXwCWNOmyW1uAR6pqjXAI+29JGkBHDcUqupR4MhRtW9W1Rvt7WPAymNtI8kK4G1V9VhVFXAPcE1bvAHY1ua3DdUlSfNsLu4p/DbwjaH3q5N8L8lfJPlAq50P7Btqs6/VAJZX1YE2/zKwfA76JEmahaWjrJzks8AbwFdb6QBwYVW9kuQS4H8kefdMt1dVlaSOsb/NwGaACy+8cPYdlyRNadZnCkk+BvwL4F+1S0JU1etV9UqbfxJ4HngXsJ+fv8S0stUADrbLS5OXmQ5Nt8+quruq1lXVurGxsdl2XZI0jVmFQpL1wL8Dfr2qfjpUH0uypM2/g8EN5Rfa5aHXklzWnjq6HniorbYd2NTmNw3VJUnz7LiXj5LcC3wIOC/JPuAWBk8bnQ3sbE+WPtaeNPog8Lkkfwv8DPhUVU3epP40gyeZfpnBPYjJ+xC3A/cnuQF4Ebh2TkYmSTphxw2FqrpuivKXp2n7APDANMvGgfdMUX8FuPx4/ZAknXx+olmS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSd2MQiHJ1iSHkuweqp2bZGeS59rrslZPkjuTTCR5KsnFQ+tsau2fS7JpqH5JkqfbOncmyVwOUpI0MzM9U/gKsP6o2hbgkapaAzzS3gNcBaxp02bgLhiECHAL8D7gUuCWySBpbT4xtN7R+5IkzYMZhUJVPQocOaq8AdjW5rcB1wzV76mBx4BzkqwArgR2VtWRqnoV2Amsb8veVlWPVVUB9wxtS5I0j0a5p7C8qg60+ZeB5W3+fOCloXb7Wu1Y9X1T1H9Bks1JxpOMHz58eISuS5KmMic3mtu/8GsutnWc/dxdVeuqat3Y2NjJ3p0kLTqjhMLBdumH9nqo1fcDFwy1W9lqx6qvnKIuSZpno4TCdmDyCaJNwEND9evbU0iXAT9ul5keBq5IsqzdYL4CeLgtey3JZe2po+uHtiVJmkdLZ9Ioyb3Ah4Dzkuxj8BTR7cD9SW4AXgSubc13AFcDE8BPgY8DVNWRJL8HPNHafa6qJm9ef5rBE06/DHyjTZKkeTajUKiq66ZZdPkUbQu4cZrtbAW2TlEfB94zk75Ikk4eP9EsSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1M06FJL8WpJdQ9NrST6T5NYk+4fqVw+tc3OSiSTPJrlyqL6+1SaSbBl1UJKk2Vk62xWr6llgLUCSJcB+4EHg48AXquoPh9snuQjYCLwb+FXgW0ne1RZ/EfgwsA94Isn2qnpmtn2TJM3OrEPhKJcDz1fVi0mma7MBuK+qXgd+lGQCuLQtm6iqFwCS3NfaGgqSNM/m6p7CRuDeofc3JXkqydYky1rtfOCloTb7Wm26+i9IsjnJeJLxw4cPz1HXJUmTRg6FJGcBvw7891a6C3gng0tLB4A7Rt3HpKq6u6rWVdW6sbGxudqsJKmZi8tHVwHfraqDAJOvAEm+BPx5e7sfuGBovZWtxjHqkqR5NBeXj65j6NJRkhVDyz4K7G7z24GNSc5OshpYA3wHeAJYk2R1O+vY2NpKkubZSGcKSd7C4KmhTw6V/yDJWqCAvZPLqmpPkvsZ3EB+A7ixqt5s27kJeBhYAmytqj2j9EuSNDsjhUJV/T/g7UfVfusY7W8DbpuivgPYMUpfJEmj8xPNkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSd3SUTeQZC/wE+BN4I2qWpfkXOBrwCpgL3BtVb2aJMB/Aq4Gfgp8rKq+27azCfgPbbO/X1XbRu3bqWbVlq8v2L733v6RBdu3pNPHXJ0p/NOqWltV69r7LcAjVbUGeKS9B7gKWNOmzcBdAC1EbgHeB1wK3JJk2Rz1TZI0Qyfr8tEGYPJf+tuAa4bq99TAY8A5SVYAVwI7q+pIVb0K7ATWn6S+SZKmMRehUMA3kzyZZHOrLa+qA23+ZWB5mz8feGlo3X2tNl395yTZnGQ8yfjhw4fnoOuSpGEj31MA3l9V+5P8fWBnkh8ML6yqSlJzsB+q6m7gboB169bNyTYlSX9n5DOFqtrfXg8BDzK4J3CwXRaivR5qzfcDFwytvrLVpqtLkubRSKGQ5C1J3jo5D1wB7Aa2A5tas03AQ21+O3B9Bi4DftwuMz0MXJFkWbvBfEWrSZLm0aiXj5YDDw6eNGUp8CdV9T+TPAHcn+QG4EXg2tZ+B4PHUScYPJL6cYCqOpLk94AnWrvPVdWREfsmSTpBI4VCVb0A/OMp6q8Al09RL+DGaba1Fdg6Sn8kSaPxE82SpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJ3axDIckFSb6d5Jkke5L8TqvfmmR/kl1tunponZuTTCR5NsmVQ/X1rTaRZMtoQ5IkzdbSEdZ9A/g3VfXdJG8Fnkyysy37QlX94XDjJBcBG4F3A78KfCvJu9riLwIfBvYBTyTZXlXPjNA3SdIszDoUquoAcKDN/yTJ94Hzj7HKBuC+qnod+FGSCeDStmyiql4ASHJfa2soSNI8m5N7CklWAe8FHm+lm5I8lWRrkmWtdj7w0tBq+1pturokaZ6NHApJfgV4APhMVb0G3AW8E1jL4EzijlH3MbSvzUnGk4wfPnx4rjYrSWpGCoUkv8QgEL5aVX8GUFUHq+rNqvoZ8CX+7hLRfuCCodVXttp09V9QVXdX1bqqWjc2NjZK1yVJUxjl6aMAXwa+X1V/NFRfMdTso8DuNr8d2Jjk7CSrgTXAd4AngDVJVic5i8HN6O2z7ZckafZGefronwC/BTydZFer/XvguiRrgQL2Ap8EqKo9Se5ncAP5DeDGqnoTIMlNwMPAEmBrVe0ZoV+SpFka5emj/wVkikU7jrHObcBtU9R3HGs9SdL88BPNkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUjfKJZp1GVm35+oLsd+/tH1mQ/UqaHc8UJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2faNZJtVCfpAY/TS3NhmcKkqTOUJAkdadMKCRZn+TZJBNJtix0fyRpMTolQiHJEuCLwFXARcB1SS5a2F5J0uJzqtxovhSYqKoXAJLcB2wAnlnQXum05teFSyfuVAmF84GXht7vA953dKMkm4HN7e3/TfLsLPZ1HvBXs1jvdOe450k+P597m5LHenE50XH/g2MtPFVCYUaq6m7g7lG2kWS8qtbNUZdOG4578ViMYwbHPVfbOyXuKQD7gQuG3q9sNUnSPDpVQuEJYE2S1UnOAjYC2xe4T5K06JwSl4+q6o0kNwEPA0uArVW15yTtbqTLT6cxx714LMYxg+OeE6mqudyeJOk0dqpcPpIknQIMBUlSt2hC4Uz/Go0ke5M8nWRXkvFWOzfJziTPtddlrZ4kd7afxVNJLl7Y3s9ckq1JDiXZPVQ74XEm2dTaP5dk00KM5URMM+5bk+xvx3xXkquHlt3cxv1skiuH6qfV70GSC5J8O8kzSfYk+Z1WP2OP+THGPD/Hu6rO+InBzevngXcAZwF/CVy00P2a4zHuBc47qvYHwJY2vwX4fJu/GvgGEOAy4PGF7v8JjPODwMXA7tmOEzgXeKG9LmvzyxZ6bLMY963Av52i7UXtz/jZwOr2Z3/J6fh7AKwALm7zbwV+2MZ3xh7zY4x5Xo73YjlT6F+jUVV/A0x+jcaZbgOwrc1vA64Zqt9TA48B5yRZsRAdPFFV9Shw5KjyiY7zSmBnVR2pqleBncD6k9/72Ztm3NPZANxXVa9X1Y+ACQa/A6fd70FVHaiq77b5nwDfZ/ANCGfsMT/GmKczp8d7sYTCVF+jcawf8umogG8mebJ9HQjA8qo60OZfBpa3+TPt53Gi4zyTxn9Tu0yydfISCmfouJOsAt4LPM4iOeZHjRnm4XgvllBYDN5fVRcz+KbZG5N8cHhhDc4zz/jnjxfLOJu7gHcCa4EDwB0L252TJ8mvAA8An6mq14aXnanHfIoxz8vxXiyhcMZ/jUZV7W+vh4AHGZw6Hpy8LNReD7XmZ9rP40THeUaMv6oOVtWbVfUz4EsMjjmcYeNO8ksM/nL8alX9WSuf0cd8qjHP1/FeLKFwRn+NRpK3JHnr5DxwBbCbwRgnn7LYBDzU5rcD17cnNS4Dfjx0Kn46OtFxPgxckWRZOwW/otVOK0fdB/oog2MOg3FvTHJ2ktXAGuA7nIa/B0kCfBn4flX90dCiM/aYTzfmeTveC32nfb4mBk8l/JDB3fjPLnR/5nhs72DwZMFfAnsmxwe8HXgEeA74FnBuq4fBf2r0PPA0sG6hx3ACY72Xwanz3zK4RnrDbMYJ/DaDG3ITwMcXelyzHPd/a+N6qv2yrxhq/9k27meBq4bqp9XvAfB+BpeGngJ2tenqM/mYH2PM83K8/ZoLSVK3WC4fSZJmwFCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJK6/w/7yGjFzSz/rwAAAABJRU5ErkJggg==\n", 197 | "text/plain": [ 198 | "
" 199 | ] 200 | }, 201 | "metadata": { 202 | "needs_background": "light" 203 | }, 204 | "output_type": "display_data" 205 | } 206 | ], 207 | "source": [ 208 | "def pad_sequence(reviews_tokenized, sequence_length):\n", 209 | " ''' returns the tokenized review sequences padded with 0's or truncated to the sequence_length.\n", 210 | " '''\n", 211 | " padded_reviews = np.zeros((len(reviews_tokenized), sequence_length), dtype = int)\n", 212 | " \n", 213 | " for idx, review in enumerate(reviews_tokenized):\n", 214 | " review_len = len(review)\n", 215 | " \n", 216 | " if review_len <= sequence_length:\n", 217 | " zeroes = list(np.zeros(sequence_length-review_len))\n", 218 | " new_sequence = zeroes+review\n", 219 | " elif review_len > sequence_length:\n", 220 | " new_sequence = review[0:sequence_length]\n", 221 | " \n", 222 | " padded_reviews[idx,:] = np.array(new_sequence)\n", 223 | " \n", 224 | " return padded_reviews\n", 225 | "\n", 226 | "sequence_length = 512\n", 227 | "padded_reviews = pad_sequence(reviews_tokenized=reviews_tokenized, sequence_length=sequence_length)\n", 228 | "\n", 229 | "plt.hist(reviews_len);" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": 8, 235 | "metadata": {}, 236 | "outputs": [], 237 | "source": [ 238 | "train_val_split = 0.75\n", 239 | "train_X = padded_reviews[:int(train_val_split*len(padded_reviews))]\n", 240 | "train_y = encoded_label_list[:int(train_val_split*len(padded_reviews))]\n", 241 | "validation_X = padded_reviews[int(train_val_split*len(padded_reviews)):]\n", 242 | "validation_y = encoded_label_list[int(train_val_split*len(padded_reviews)):]" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [ 251 | "## If while training, you get a runtime error that says: \"RuntimeError: Expected tensor for argument #1 'indices' to have scalar type Long\".\n", 252 | "## simply uncomment run the following lines of code additionally\n", 253 | "# train_X = train_X.astype('int64')\n", 254 | "# train_y = train_y.astype('int64')\n", 255 | "# validation_X = validation_X.astype('int64')\n", 256 | "# validation_y = validation_y.astype('int64')" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 9, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "# generate torch datasets\n", 266 | "train_dataset = TensorDataset(torch.from_numpy(train_X).to(device), torch.from_numpy(train_y).to(device))\n", 267 | "validation_dataset = TensorDataset(torch.from_numpy(validation_X).to(device), torch.from_numpy(validation_y).to(device))\n", 268 | "\n", 269 | "\n", 270 | "batch_size = 32\n", 271 | "# torch dataloaders (shuffle data)\n", 272 | "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", 273 | "validation_dataloader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=True)" 274 | ] 275 | }, 276 | { 277 | "cell_type": "code", 278 | "execution_count": 10, 279 | "metadata": {}, 280 | "outputs": [ 281 | { 282 | "name": "stdout", 283 | "output_type": "stream", 284 | "text": [ 285 | "Example Input size: torch.Size([32, 512])\n", 286 | "Example Input:\n", 287 | " tensor([[ 0, 0, 0, ..., 1, 875, 520],\n", 288 | " [ 0, 0, 0, ..., 482, 800, 1794],\n", 289 | " [ 0, 0, 0, ..., 3, 1285, 70251],\n", 290 | " ...,\n", 291 | " [ 0, 0, 0, ..., 4, 1, 1374],\n", 292 | " [ 0, 0, 0, ..., 2, 8268, 17117],\n", 293 | " [ 0, 0, 0, ..., 6429, 271, 116]])\n", 294 | "\n", 295 | "Example Output size: torch.Size([32])\n", 296 | "Example Output:\n", 297 | " tensor([1., 0., 1., 0., 0., 1., 0., 1., 1., 0., 1., 1., 0., 0., 0., 1., 1., 1.,\n", 298 | " 1., 1., 1., 0., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1.])\n" 299 | ] 300 | } 301 | ], 302 | "source": [ 303 | "# get a batch of train data\n", 304 | "train_data_iter = iter(train_dataloader)\n", 305 | "X_example, y_example = train_data_iter.next()\n", 306 | "print('Example Input size: ', X_example.size()) # batch_size, seq_length\n", 307 | "print('Example Input:\\n', X_example)\n", 308 | "print()\n", 309 | "print('Example Output size: ', y_example.size()) # batch_size\n", 310 | "print('Example Output:\\n', y_example)" 311 | ] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": 11, 316 | "metadata": {}, 317 | "outputs": [], 318 | "source": [ 319 | "class RNN(nn.Module):\n", 320 | " def __init__(self, input_dimension, embedding_dimension, hidden_dimension, output_dimension):\n", 321 | " super().__init__()\n", 322 | " self.embedding_layer = nn.Embedding(input_dimension, embedding_dimension) \n", 323 | " self.rnn_layer = nn.RNN(embedding_dimension, hidden_dimension, num_layers=1)\n", 324 | " self.fc_layer = nn.Linear(hidden_dimension, output_dimension)\n", 325 | " \n", 326 | " def forward(self, sequence):\n", 327 | " # sequence shape = (sequence_length, batch_size)\n", 328 | " embedding = self.embedding_layer(sequence) \n", 329 | " # embedding shape = [sequence_length, batch_size, embedding_dimension]\n", 330 | " output, hidden_state = self.rnn_layer(embedding)\n", 331 | " # output shape = [sequence_length, batch_size, hidden_dimension]\n", 332 | " # hidden_state shape = [1, batch_size, hidden_dimension]\n", 333 | " final_output = self.fc_layer(hidden_state[-1,:,:].squeeze(0)) \n", 334 | " return final_output\n", 335 | " \n", 336 | "input_dimension = len(vocab_to_token)+1 # +1 to account for padding\n", 337 | "embedding_dimension = 100\n", 338 | "hidden_dimension = 32\n", 339 | "output_dimension = 1\n", 340 | "\n", 341 | "rnn_model = RNN(input_dimension, embedding_dimension, hidden_dimension, output_dimension)\n", 342 | "\n", 343 | "optim = torch.optim.Adam(rnn_model.parameters())\n", 344 | "loss_func = nn.BCEWithLogitsLoss()\n", 345 | "\n", 346 | "rnn_model = rnn_model.to(device)\n", 347 | "loss_func = loss_func.to(device)" 348 | ] 349 | }, 350 | { 351 | "cell_type": "code", 352 | "execution_count": 12, 353 | "metadata": {}, 354 | "outputs": [], 355 | "source": [ 356 | "def accuracy_metric(predictions, ground_truth):\n", 357 | " \"\"\"\n", 358 | " Returns 0-1 accuracy for the given set of predictions and ground truth\n", 359 | " \"\"\"\n", 360 | " # round predictions to either 0 or 1\n", 361 | " rounded_predictions = torch.round(torch.sigmoid(predictions))\n", 362 | " success = (rounded_predictions == ground_truth).float() #convert into float for division \n", 363 | " accuracy = success.sum() / len(success)\n", 364 | " return accuracy" 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": 13, 370 | "metadata": {}, 371 | "outputs": [], 372 | "source": [ 373 | "def train(model, dataloader, optim, loss_func):\n", 374 | " loss = 0\n", 375 | " accuracy = 0\n", 376 | " model.train()\n", 377 | " \n", 378 | " for sequence, sentiment in dataloader:\n", 379 | " optim.zero_grad() \n", 380 | " preds = model(sequence.T).squeeze()\n", 381 | " \n", 382 | " loss_curr = loss_func(preds, sentiment)\n", 383 | " accuracy_curr = accuracy_metric(preds, sentiment)\n", 384 | " \n", 385 | " loss_curr.backward()\n", 386 | " optim.step()\n", 387 | " \n", 388 | " loss += loss_curr.item()\n", 389 | " accuracy += accuracy_curr.item()\n", 390 | " \n", 391 | " return loss/len(dataloader), accuracy/len(dataloader)" 392 | ] 393 | }, 394 | { 395 | "cell_type": "code", 396 | "execution_count": 14, 397 | "metadata": {}, 398 | "outputs": [], 399 | "source": [ 400 | "def validate(model, dataloader, loss_func):\n", 401 | " loss = 0\n", 402 | " accuracy = 0\n", 403 | " model.eval()\n", 404 | " \n", 405 | " with torch.no_grad():\n", 406 | " for sequence, sentiment in dataloader:\n", 407 | " \n", 408 | " preds = model(sequence.T).squeeze()\n", 409 | " \n", 410 | " loss_curr = loss_func(preds, sentiment) \n", 411 | " accuracy_curr = accuracy_metric(preds, sentiment)\n", 412 | "\n", 413 | " loss += loss_curr.item()\n", 414 | " accuracy += accuracy_curr.item()\n", 415 | " \n", 416 | " return loss/len(dataloader), accuracy/len(dataloader)" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": 15, 422 | "metadata": { 423 | "scrolled": false 424 | }, 425 | "outputs": [ 426 | { 427 | "name": "stdout", 428 | "output_type": "stream", 429 | "text": [ 430 | "epoch number: 1 | time elapsed: 136.13723397254944s\n", 431 | "training loss: 0.627 | training accuracy: 66.23%\n", 432 | "validation loss: 1.048 | validation accuracy: 19.65%\n", 433 | "\n", 434 | "epoch number: 2 | time elapsed: 150.36637210845947s\n", 435 | "training loss: 0.533 | training accuracy: 73.80%\n", 436 | "validation loss: 0.858 | validation accuracy: 54.43%\n", 437 | "\n", 438 | "epoch number: 3 | time elapsed: 186.54570603370667s\n", 439 | "training loss: 0.438 | training accuracy: 80.39%\n", 440 | "validation loss: 0.551 | validation accuracy: 78.56%\n", 441 | "\n", 442 | "epoch number: 4 | time elapsed: 178.38556694984436s\n", 443 | "training loss: 0.383 | training accuracy: 83.26%\n", 444 | "validation loss: 0.915 | validation accuracy: 57.72%\n", 445 | "\n", 446 | "epoch number: 5 | time elapsed: 184.10346102714539s\n", 447 | "training loss: 0.334 | training accuracy: 86.46%\n", 448 | "validation loss: 1.001 | validation accuracy: 56.94%\n", 449 | "\n", 450 | "epoch number: 6 | time elapsed: 180.75322008132935s\n", 451 | "training loss: 0.299 | training accuracy: 88.33%\n", 452 | "validation loss: 1.164 | validation accuracy: 54.97%\n", 453 | "\n", 454 | "epoch number: 7 | time elapsed: 189.0122902393341s\n", 455 | "training loss: 0.266 | training accuracy: 89.62%\n", 456 | "validation loss: 1.133 | validation accuracy: 60.51%\n", 457 | "\n", 458 | "epoch number: 8 | time elapsed: 199.3309519290924s\n", 459 | "training loss: 0.198 | training accuracy: 92.92%\n", 460 | "validation loss: 0.971 | validation accuracy: 66.19%\n", 461 | "\n", 462 | "epoch number: 9 | time elapsed: 185.76586294174194s\n", 463 | "training loss: 0.315 | training accuracy: 87.93%\n", 464 | "validation loss: 0.950 | validation accuracy: 62.34%\n", 465 | "\n", 466 | "epoch number: 10 | time elapsed: 188.91670608520508s\n", 467 | "training loss: 0.193 | training accuracy: 93.08%\n", 468 | "validation loss: 1.042 | validation accuracy: 62.71%\n", 469 | "\n" 470 | ] 471 | } 472 | ], 473 | "source": [ 474 | "num_epochs = 10\n", 475 | "best_validation_loss = float('inf')\n", 476 | "\n", 477 | "for ep in range(num_epochs):\n", 478 | "\n", 479 | " time_start = time.time()\n", 480 | " \n", 481 | " training_loss, train_accuracy = train(rnn_model, train_dataloader, optim, loss_func)\n", 482 | " validation_loss, validation_accuracy = validate(rnn_model, validation_dataloader, loss_func)\n", 483 | " \n", 484 | " time_end = time.time()\n", 485 | " time_delta = time_end - time_start \n", 486 | " \n", 487 | " if validation_loss < best_validation_loss:\n", 488 | " best_validation_loss = validation_loss\n", 489 | " torch.save(rnn_model.state_dict(), 'rnn_model.pt')\n", 490 | " \n", 491 | " print(f'epoch number: {ep+1} | time elapsed: {time_delta}s')\n", 492 | " print(f'training loss: {training_loss:.3f} | training accuracy: {train_accuracy*100:.2f}%')\n", 493 | " print(f'validation loss: {validation_loss:.3f} | validation accuracy: {validation_accuracy*100:.2f}%')\n", 494 | " print()" 495 | ] 496 | }, 497 | { 498 | "cell_type": "code", 499 | "execution_count": 16, 500 | "metadata": {}, 501 | "outputs": [], 502 | "source": [ 503 | "def sentiment_inference(model, sentence):\n", 504 | " model.eval()\n", 505 | " \n", 506 | " # text transformations\n", 507 | " sentence = sentence.lower()\n", 508 | " sentence = ''.join([c for c in sentence if c not in punctuation])\n", 509 | " tokenized = [vocab_to_token.get(token, 0) for token in sentence.split()]\n", 510 | " tokenized = np.pad(tokenized, (512-len(tokenized), 0), 'constant')\n", 511 | " \n", 512 | " # model inference\n", 513 | " model_input = torch.LongTensor(tokenized).to(device)\n", 514 | " model_input = model_input.unsqueeze(1)\n", 515 | " pred = torch.sigmoid(model(model_input))\n", 516 | " \n", 517 | " return pred.item()" 518 | ] 519 | }, 520 | { 521 | "cell_type": "code", 522 | "execution_count": 20, 523 | "metadata": {}, 524 | "outputs": [ 525 | { 526 | "name": "stdout", 527 | "output_type": "stream", 528 | "text": [ 529 | "0.05216024070978165\n", 530 | "0.17682921886444092\n", 531 | "0.7510029077529907\n", 532 | "0.9689022898674011\n", 533 | "0.9829260110855103\n" 534 | ] 535 | } 536 | ], 537 | "source": [ 538 | "print(sentiment_inference(rnn_model, \"This film is horrible\"))\n", 539 | "print(sentiment_inference(rnn_model, \"Director tried too hard but this film is bad\"))\n", 540 | "print(sentiment_inference(rnn_model, \"Decent movie, although could be shorter\"))\n", 541 | "print(sentiment_inference(rnn_model, \"This film will be houseful for weeks\"))\n", 542 | "print(sentiment_inference(rnn_model, \"I loved the movie, every part of it\"))" 543 | ] 544 | } 545 | ], 546 | "metadata": { 547 | "kernelspec": { 548 | "display_name": "Python 3", 549 | "language": "python", 550 | "name": "python3" 551 | }, 552 | "language_info": { 553 | "codemirror_mode": { 554 | "name": "ipython", 555 | "version": 3 556 | }, 557 | "file_extension": ".py", 558 | "mimetype": "text/x-python", 559 | "name": "python", 560 | "nbconvert_exporter": "python", 561 | "pygments_lexer": "ipython3", 562 | "version": "3.7.6" 563 | } 564 | }, 565 | "nbformat": 4, 566 | "nbformat_minor": 4 567 | } 568 | -------------------------------------------------------------------------------- /Chapter05/out_of_the_box_transformers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "application/vnd.jupyter.widget-view+json": { 11 | "model_id": "582e1762180841f2ab79f467230da2db", 12 | "version_major": 2, 13 | "version_minor": 0 14 | }, 15 | "text/plain": [ 16 | "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…" 17 | ] 18 | }, 19 | "metadata": {}, 20 | "output_type": "display_data" 21 | }, 22 | { 23 | "name": "stdout", 24 | "output_type": "stream", 25 | "text": [ 26 | "\n" 27 | ] 28 | }, 29 | { 30 | "data": { 31 | "application/vnd.jupyter.widget-view+json": { 32 | "model_id": "4bdb8b75fe53479c9092c991a6dc0fd2", 33 | "version_major": 2, 34 | "version_minor": 0 35 | }, 36 | "text/plain": [ 37 | "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…" 38 | ] 39 | }, 40 | "metadata": {}, 41 | "output_type": "display_data" 42 | }, 43 | { 44 | "name": "stdout", 45 | "output_type": "stream", 46 | "text": [ 47 | "\n" 48 | ] 49 | }, 50 | { 51 | "name": "stderr", 52 | "output_type": "stream", 53 | "text": [ 54 | "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n", 55 | "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n", 56 | "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", 57 | "Some weights of BertForMaskedLM were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['cls.predictions.decoder.bias']\n", 58 | "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" 59 | ] 60 | }, 61 | { 62 | "data": { 63 | "application/vnd.jupyter.widget-view+json": { 64 | "model_id": "270a2ef3c12d48808b4cb60feb7a107e", 65 | "version_major": 2, 66 | "version_minor": 0 67 | }, 68 | "text/plain": [ 69 | "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…" 70 | ] 71 | }, 72 | "metadata": {}, 73 | "output_type": "display_data" 74 | }, 75 | { 76 | "name": "stdout", 77 | "output_type": "stream", 78 | "text": [ 79 | "\n" 80 | ] 81 | } 82 | ], 83 | "source": [ 84 | "import torch\n", 85 | "from transformers import BertForMaskedLM, BertTokenizer\n", 86 | " \n", 87 | "bert_model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n", 88 | "token_gen = BertTokenizer.from_pretrained('bert-base-uncased')\n", 89 | "\n", 90 | "\n", 91 | "ip_sequence = token_gen(\"I love PyTorch !\", return_tensors=\"pt\")[\"input_ids\"]\n", 92 | " \n", 93 | "op = bert_model(ip_sequence, labels=ip_sequence)\n", 94 | "total_loss, raw_preds = op[:2]\n" 95 | ] 96 | } 97 | ], 98 | "metadata": { 99 | "kernelspec": { 100 | "display_name": "Python 3", 101 | "language": "python", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "codemirror_mode": { 106 | "name": "ipython", 107 | "version": 3 108 | }, 109 | "file_extension": ".py", 110 | "mimetype": "text/x-python", 111 | "name": "python", 112 | "nbconvert_exporter": "python", 113 | "pygments_lexer": "ipython3", 114 | "version": "3.7.6" 115 | } 116 | }, 117 | "nbformat": 4, 118 | "nbformat_minor": 4 119 | } 120 | -------------------------------------------------------------------------------- /Chapter05/randwirenn[representational_purpose_only].png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter05/randwirenn[representational_purpose_only].png -------------------------------------------------------------------------------- /Chapter05/transformer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import math\n", 10 | "import time\n", 11 | "\n", 12 | "import torch\n", 13 | "import torch.nn as nn\n", 14 | "import torch.nn.functional as F\n", 15 | "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n", 16 | "\n", 17 | "import torchtext\n", 18 | "from torchtext.data.utils import get_tokenizer" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "class Transformer(nn.Module):\n", 28 | " def __init__(self, num_token, num_inputs, num_heads, num_hidden, num_layers, dropout=0.3):\n", 29 | " super(Transformer, self).__init__()\n", 30 | " self.model_name = 'transformer'\n", 31 | " self.mask_source = None\n", 32 | " self.position_enc = PosEnc(num_inputs, dropout)\n", 33 | " layers_enc = TransformerEncoderLayer(num_inputs, num_heads, num_hidden, dropout)\n", 34 | " self.enc_transformer = TransformerEncoder(layers_enc, num_layers)\n", 35 | " self.enc = nn.Embedding(num_token, num_inputs)\n", 36 | " self.num_inputs = num_inputs\n", 37 | " self.dec = nn.Linear(num_inputs, num_token)\n", 38 | " self.init_params()\n", 39 | "\n", 40 | " def _gen_sqr_nxt_mask(self, size):\n", 41 | " msk = (torch.triu(torch.ones(size, size)) == 1).transpose(0, 1)\n", 42 | " msk = msk.float().masked_fill(msk == 0, float('-inf'))\n", 43 | " msk = msk.masked_fill(msk == 1, float(0.0))\n", 44 | " return msk\n", 45 | "\n", 46 | " def init_params(self):\n", 47 | " initial_rng = 0.12\n", 48 | " self.enc.weight.data.uniform_(-initial_rng, initial_rng)\n", 49 | " self.dec.bias.data.zero_()\n", 50 | " self.dec.weight.data.uniform_(-initial_rng, initial_rng)\n", 51 | "\n", 52 | " def forward(self, source):\n", 53 | " if self.mask_source is None or self.mask_source.size(0) != len(source):\n", 54 | " dvc = source.device\n", 55 | " msk = self._gen_sqr_nxt_mask(len(source)).to(dvc)\n", 56 | " self.mask_source = msk\n", 57 | "\n", 58 | " source = self.enc(source) * math.sqrt(self.num_inputs)\n", 59 | " source = self.position_enc(source)\n", 60 | " op = self.enc_transformer(source, self.mask_source)\n", 61 | " op = self.dec(op)\n", 62 | " return op" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 3, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "class PosEnc(nn.Module):\n", 72 | " def __init__(self, d_m, dropout=0.2, size_limit=5000):\n", 73 | " # d_m is same as the dimension of the embeddings\n", 74 | " super(PosEnc, self).__init__()\n", 75 | " self.dropout = nn.Dropout(dropout)\n", 76 | " p_enc = torch.zeros(size_limit, d_m)\n", 77 | " pos = torch.arange(0, size_limit, dtype=torch.float).unsqueeze(1)\n", 78 | " divider = torch.exp(torch.arange(0, d_m, 2).float() * (-math.log(10000.0) / d_m))\n", 79 | " # divider is the list of radians, multiplied by position indices of words, and fed to the sinusoidal and cosinusoidal function\n", 80 | " p_enc[:, 0::2] = torch.sin(pos * divider)\n", 81 | " p_enc[:, 1::2] = torch.cos(pos * divider)\n", 82 | " p_enc = p_enc.unsqueeze(0).transpose(0, 1)\n", 83 | " self.register_buffer('p_enc', p_enc)\n", 84 | "\n", 85 | " def forward(self, x):\n", 86 | " return self.dropout(x + self.p_enc[:x.size(0), :])" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 4, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "TEXT = torchtext.data.Field(tokenize=get_tokenizer(\"basic_english\"), lower=True, eos_token='', init_token='')\n", 96 | "training_text, validation_text, testing_text = torchtext.datasets.WikiText2.splits(TEXT)\n", 97 | "TEXT.build_vocab(training_text)\n", 98 | "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", 99 | "\n", 100 | "def gen_batches(text_dataset, batch_size):\n", 101 | " text_dataset = TEXT.numericalize([text_dataset.examples[0].text])\n", 102 | " # divide text dataset into parts of size equal to batch_size\n", 103 | " num_batches = text_dataset.size(0) // batch_size\n", 104 | " # remove data points that lie outside batches (remainders)\n", 105 | " text_dataset = text_dataset.narrow(0, 0, num_batches * batch_size)\n", 106 | " # distribute dataset across batches evenly\n", 107 | " text_dataset = text_dataset.view(batch_size, -1).t().contiguous()\n", 108 | " return text_dataset.to(device)\n", 109 | "\n", 110 | "training_batch_size = 32\n", 111 | "evaluation_batch_size = 16\n", 112 | "\n", 113 | "training_data = gen_batches(training_text, training_batch_size)\n", 114 | "validation_data = gen_batches(validation_text, evaluation_batch_size)\n", 115 | "testing_data = gen_batches(testing_text, evaluation_batch_size)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 5, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "max_seq_len = 64\n", 125 | "def return_batch(src, k):\n", 126 | " sequence_length = min(max_seq_len, len(src) - 1 - k)\n", 127 | " sequence_data = src[k:k+sequence_length]\n", 128 | " sequence_label = src[k+1:k+1+sequence_length].view(-1)\n", 129 | " return sequence_data, sequence_label" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 6, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "num_tokens = len(TEXT.vocab.stoi) # vocabulary size\n", 139 | "embedding_size = 256 # dimension of embedding layer\n", 140 | "num_hidden_params = 256 # transformer encoder's hidden (feed forward) layer dimension\n", 141 | "num_layers = 2 # num of transformer encoder layers within transformer encoder\n", 142 | "num_heads = 2 # num of heads in (multi head) attention models\n", 143 | "dropout = 0.25 # value (fraction) of dropout\n", 144 | "loss_func = nn.CrossEntropyLoss()\n", 145 | "lrate = 4.0 # learning rate\n", 146 | "transformer_model = Transformer(num_tokens, embedding_size, num_heads, num_hidden_params, num_layers, \n", 147 | " dropout).to(device)\n", 148 | "optim_module = torch.optim.SGD(transformer_model.parameters(), lr=lrate)\n", 149 | "sched_module = torch.optim.lr_scheduler.StepLR(optim_module, 1.0, gamma=0.88)" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 7, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "def train_model():\n", 159 | " transformer_model.train()\n", 160 | " loss_total = 0.\n", 161 | " time_start = time.time()\n", 162 | " num_tokens = len(TEXT.vocab.stoi)\n", 163 | " for b, i in enumerate(range(0, training_data.size(0) - 1, max_seq_len)):\n", 164 | " train_data_batch, train_label_batch = return_batch(training_data, i)\n", 165 | " optim_module.zero_grad()\n", 166 | " op = transformer_model(train_data_batch)\n", 167 | " loss_curr = loss_func(op.view(-1, num_tokens), train_label_batch)\n", 168 | " loss_curr.backward()\n", 169 | " torch.nn.utils.clip_grad_norm_(transformer_model.parameters(), 0.6)\n", 170 | " optim_module.step()\n", 171 | "\n", 172 | " loss_total += loss_curr.item()\n", 173 | " interval = 100\n", 174 | " if b % interval == 0 and b > 0:\n", 175 | " loss_interval = loss_total / interval\n", 176 | " time_delta = time.time() - time_start\n", 177 | " print(f\"epoch {ep}, {b}/{len(training_data)//max_seq_len} batches, training loss {loss_interval:.2f}, training perplexity {math.exp(loss_interval):.2f}\")\n", 178 | " loss_total = 0\n", 179 | " time_start = time.time()\n", 180 | "\n", 181 | "def eval_model(eval_model_obj, eval_data_source):\n", 182 | " eval_model_obj.eval() \n", 183 | " loss_total = 0.\n", 184 | " num_tokens = len(TEXT.vocab.stoi)\n", 185 | " with torch.no_grad():\n", 186 | " for j in range(0, eval_data_source.size(0) - 1, max_seq_len):\n", 187 | " eval_data, eval_label = return_batch(eval_data_source, j)\n", 188 | " op = eval_model_obj(eval_data)\n", 189 | " op_flat = op.view(-1, num_tokens)\n", 190 | " loss_total += len(eval_data) * loss_func(op_flat, eval_label).item()\n", 191 | " return loss_total / (len(eval_data_source) - 1)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 8, 197 | "metadata": {}, 198 | "outputs": [ 199 | { 200 | "name": "stdout", 201 | "output_type": "stream", 202 | "text": [ 203 | "epoch 1, 100/1018 batches, training loss 8.50, training perplexity 4901.66\n", 204 | "epoch 1, 200/1018 batches, training loss 7.16, training perplexity 1286.24\n", 205 | "epoch 1, 300/1018 batches, training loss 6.76, training perplexity 865.43\n", 206 | "epoch 1, 400/1018 batches, training loss 6.55, training perplexity 702.21\n", 207 | "epoch 1, 500/1018 batches, training loss 6.45, training perplexity 631.90\n", 208 | "epoch 1, 600/1018 batches, training loss 6.31, training perplexity 548.01\n", 209 | "epoch 1, 700/1018 batches, training loss 6.25, training perplexity 516.28\n", 210 | "epoch 1, 800/1018 batches, training loss 6.11, training perplexity 450.42\n", 211 | "epoch 1, 900/1018 batches, training loss 6.09, training perplexity 441.72\n", 212 | "epoch 1, 1000/1018 batches, training loss 6.08, training perplexity 436.78\n", 213 | "\n", 214 | "epoch 1, validation loss 5.82, validation perplexity 336.19\n", 215 | "\n", 216 | "epoch 2, 100/1018 batches, training loss 5.98, training perplexity 394.64\n", 217 | "epoch 2, 200/1018 batches, training loss 5.90, training perplexity 364.08\n", 218 | "epoch 2, 300/1018 batches, training loss 5.82, training perplexity 337.72\n", 219 | "epoch 2, 400/1018 batches, training loss 5.78, training perplexity 324.68\n", 220 | "epoch 2, 500/1018 batches, training loss 5.82, training perplexity 335.50\n", 221 | "epoch 2, 600/1018 batches, training loss 5.77, training perplexity 319.43\n", 222 | "epoch 2, 700/1018 batches, training loss 5.78, training perplexity 322.60\n", 223 | "epoch 2, 800/1018 batches, training loss 5.65, training perplexity 283.28\n", 224 | "epoch 2, 900/1018 batches, training loss 5.67, training perplexity 291.07\n", 225 | "epoch 2, 1000/1018 batches, training loss 5.71, training perplexity 300.54\n", 226 | "\n", 227 | "epoch 2, validation loss 5.53, validation perplexity 251.09\n", 228 | "\n", 229 | "epoch 3, 100/1018 batches, training loss 5.67, training perplexity 288.79\n", 230 | "epoch 3, 200/1018 batches, training loss 5.59, training perplexity 268.81\n", 231 | "epoch 3, 300/1018 batches, training loss 5.55, training perplexity 257.23\n", 232 | "epoch 3, 400/1018 batches, training loss 5.52, training perplexity 249.65\n", 233 | "epoch 3, 500/1018 batches, training loss 5.55, training perplexity 257.02\n", 234 | "epoch 3, 600/1018 batches, training loss 5.52, training perplexity 249.50\n", 235 | "epoch 3, 700/1018 batches, training loss 5.53, training perplexity 252.90\n", 236 | "epoch 3, 800/1018 batches, training loss 5.39, training perplexity 219.61\n", 237 | "epoch 3, 900/1018 batches, training loss 5.44, training perplexity 230.41\n", 238 | "epoch 3, 1000/1018 batches, training loss 5.49, training perplexity 241.15\n", 239 | "\n", 240 | "epoch 3, validation loss 5.37, validation perplexity 215.04\n", 241 | "\n", 242 | "epoch 4, 100/1018 batches, training loss 5.46, training perplexity 235.42\n", 243 | "epoch 4, 200/1018 batches, training loss 5.40, training perplexity 220.81\n", 244 | "epoch 4, 300/1018 batches, training loss 5.36, training perplexity 213.61\n", 245 | "epoch 4, 400/1018 batches, training loss 5.34, training perplexity 208.66\n", 246 | "epoch 4, 500/1018 batches, training loss 5.37, training perplexity 213.88\n", 247 | "epoch 4, 600/1018 batches, training loss 5.35, training perplexity 210.60\n", 248 | "epoch 4, 700/1018 batches, training loss 5.36, training perplexity 213.75\n", 249 | "epoch 4, 800/1018 batches, training loss 5.21, training perplexity 183.18\n", 250 | "epoch 4, 900/1018 batches, training loss 5.26, training perplexity 193.41\n", 251 | "epoch 4, 1000/1018 batches, training loss 5.32, training perplexity 205.22\n", 252 | "\n", 253 | "epoch 4, validation loss 5.31, validation perplexity 202.42\n", 254 | "\n", 255 | "epoch 5, 100/1018 batches, training loss 5.31, training perplexity 202.77\n", 256 | "epoch 5, 200/1018 batches, training loss 5.25, training perplexity 189.64\n", 257 | "epoch 5, 300/1018 batches, training loss 5.22, training perplexity 184.80\n", 258 | "epoch 5, 400/1018 batches, training loss 5.20, training perplexity 181.18\n", 259 | "epoch 5, 500/1018 batches, training loss 5.22, training perplexity 185.54\n", 260 | "epoch 5, 600/1018 batches, training loss 5.21, training perplexity 182.95\n", 261 | "epoch 5, 700/1018 batches, training loss 5.22, training perplexity 185.69\n", 262 | "epoch 5, 800/1018 batches, training loss 5.07, training perplexity 158.79\n", 263 | "epoch 5, 900/1018 batches, training loss 5.13, training perplexity 169.36\n", 264 | "epoch 5, 1000/1018 batches, training loss 5.19, training perplexity 179.63\n", 265 | "\n", 266 | "epoch 5, validation loss 5.23, validation perplexity 186.53\n", 267 | "\n" 268 | ] 269 | } 270 | ], 271 | "source": [ 272 | "min_validation_loss = float(\"inf\")\n", 273 | "eps = 5\n", 274 | "best_model_so_far = None\n", 275 | "\n", 276 | "for ep in range(1, eps + 1):\n", 277 | " ep_time_start = time.time()\n", 278 | " train_model()\n", 279 | " validation_loss = eval_model(transformer_model, validation_data)\n", 280 | " print()\n", 281 | " print(f\"epoch {ep:}, validation loss {validation_loss:.2f}, validation perplexity {math.exp(validation_loss):.2f}\")\n", 282 | " print()\n", 283 | "\n", 284 | " if validation_loss < min_validation_loss:\n", 285 | " min_validation_loss = validation_loss\n", 286 | " best_model_so_far = transformer_model\n", 287 | "\n", 288 | " sched_module.step()" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 9, 294 | "metadata": {}, 295 | "outputs": [ 296 | { 297 | "name": "stdout", 298 | "output_type": "stream", 299 | "text": [ 300 | "testing loss 5.14, testing perplexity 171.47\n" 301 | ] 302 | } 303 | ], 304 | "source": [ 305 | "testing_loss = eval_model(best_model_so_far, testing_data)\n", 306 | "print(f\"testing loss {testing_loss:.2f}, testing perplexity {math.exp(testing_loss):.2f}\")" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": null, 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [] 315 | } 316 | ], 317 | "metadata": { 318 | "kernelspec": { 319 | "display_name": "Python 3", 320 | "language": "python", 321 | "name": "python3" 322 | }, 323 | "language_info": { 324 | "codemirror_mode": { 325 | "name": "ipython", 326 | "version": 3 327 | }, 328 | "file_extension": ".py", 329 | "mimetype": "text/x-python", 330 | "name": "python", 331 | "nbconvert_exporter": "python", 332 | "pygments_lexer": "ipython3", 333 | "version": "3.7.6" 334 | } 335 | }, 336 | "nbformat": 4, 337 | "nbformat_minor": 4 338 | } 339 | -------------------------------------------------------------------------------- /Chapter06/text_generation_out_of_the_box.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 2, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n", 17 | "import torch" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 3, 23 | "metadata": { 24 | "scrolled": true 25 | }, 26 | "outputs": [], 27 | "source": [ 28 | "torch.manual_seed(799)\n", 29 | "tkz = GPT2Tokenizer.from_pretrained(\"gpt2\")\n", 30 | "mdl = GPT2LMHeadModel.from_pretrained('gpt2')\n", 31 | "ln = 10\n", 32 | "cue = \"It will\"\n", 33 | "gen = tkz.encode(cue)\n", 34 | "ctx = torch.tensor([gen])" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 4, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "name": "stdout", 44 | "output_type": "stream", 45 | "text": [ 46 | "It will be interesting to see how the new system works out\n" 47 | ] 48 | } 49 | ], 50 | "source": [ 51 | "prv=None\n", 52 | "for i in range(ln):\n", 53 | " op, prv = mdl(ctx, past=prv)\n", 54 | " tkn = torch.argmax(op[..., -1, :])\n", 55 | "\n", 56 | " gen += [tkn.tolist()]\n", 57 | " ctx = tkn.unsqueeze(0)\n", 58 | "\n", 59 | "seq = tkz.decode(gen)\n", 60 | "\n", 61 | "print(seq)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 5, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "name": "stdout", 71 | "output_type": "stream", 72 | "text": [ 73 | "It will be interesting to see how the new system\n" 74 | ] 75 | } 76 | ], 77 | "source": [ 78 | "ip_ids = tkz.encode(cue, return_tensors='pt')\n", 79 | "op_greedy = mdl.generate(ip_ids, max_length=ln)\n", 80 | "seq = tkz.decode(op_greedy[0], skip_special_tokens=True)\n", 81 | "print(seq)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 6, 87 | "metadata": {}, 88 | "outputs": [ 89 | { 90 | "name": "stdout", 91 | "output_type": "stream", 92 | "text": [ 93 | "It will be interesting to\n", 94 | "It will be a long\n", 95 | "It will be a great\n" 96 | ] 97 | } 98 | ], 99 | "source": [ 100 | "op_beam = mdl.generate(\n", 101 | " ip_ids, \n", 102 | " max_length=5, \n", 103 | " num_beams=3, \n", 104 | " num_return_sequences=3, \n", 105 | ")\n", 106 | "\n", 107 | "for op_beam_cur in op_beam:\n", 108 | " print(tkz.decode(op_beam_cur, skip_special_tokens=True))" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 7, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "name": "stdout", 118 | "output_type": "stream", 119 | "text": [ 120 | "It will also be a\n", 121 | "It will be a long\n", 122 | "It will also be interesting\n" 123 | ] 124 | } 125 | ], 126 | "source": [ 127 | "for i in range(3):\n", 128 | " torch.manual_seed(i)\n", 129 | " op = mdl.generate(\n", 130 | " ip_ids, \n", 131 | " do_sample=True, \n", 132 | " max_length=5, \n", 133 | " top_k=2\n", 134 | " )\n", 135 | "\n", 136 | " seq = tkz.decode(op[0], skip_special_tokens=True)\n", 137 | " print(seq)" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 8, 143 | "metadata": {}, 144 | "outputs": [ 145 | { 146 | "name": "stdout", 147 | "output_type": "stream", 148 | "text": [ 149 | "It will be interesting to\n", 150 | "It will be interesting to\n", 151 | "It will be interesting to\n" 152 | ] 153 | } 154 | ], 155 | "source": [ 156 | "for i in range(3):\n", 157 | " torch.manual_seed(i)\n", 158 | " op_greedy = mdl.generate(ip_ids, max_length=5)\n", 159 | " seq = tkz.decode(op_greedy[0], skip_special_tokens=True)\n", 160 | " print(seq)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 9, 166 | "metadata": {}, 167 | "outputs": [ 168 | { 169 | "name": "stdout", 170 | "output_type": "stream", 171 | "text": [ 172 | "It will require work in\n", 173 | "It will be an interesting\n", 174 | "It will likely be important\n" 175 | ] 176 | } 177 | ], 178 | "source": [ 179 | "for i in range(3):\n", 180 | " torch.manual_seed(i)\n", 181 | " op = mdl.generate(\n", 182 | " ip_ids, \n", 183 | " do_sample=True, \n", 184 | " max_length=5, \n", 185 | " top_p=0.75, \n", 186 | " top_k=0\n", 187 | " )\n", 188 | "\n", 189 | " seq = tkz.decode(op[0], skip_special_tokens=True)\n", 190 | " print(seq)" 191 | ] 192 | } 193 | ], 194 | "metadata": { 195 | "kernelspec": { 196 | "display_name": "Python 3", 197 | "language": "python", 198 | "name": "python3" 199 | }, 200 | "language_info": { 201 | "codemirror_mode": { 202 | "name": "ipython", 203 | "version": 3 204 | }, 205 | "file_extension": ".py", 206 | "mimetype": "text/x-python", 207 | "name": "python", 208 | "nbconvert_exporter": "python", 209 | "pygments_lexer": "ipython3", 210 | "version": "3.7.6" 211 | } 212 | }, 213 | "nbformat": 4, 214 | "nbformat_minor": 4 215 | } 216 | -------------------------------------------------------------------------------- /Chapter07/images/content.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter07/images/content.jpg -------------------------------------------------------------------------------- /Chapter07/images/style.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter07/images/style.jpg -------------------------------------------------------------------------------- /Chapter08/dcgan.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import numpy as np\n", 11 | "\n", 12 | "import torch\n", 13 | "import torch.nn as nn\n", 14 | "import torch.nn.functional as F\n", 15 | "from torch.utils.data import DataLoader\n", 16 | "from torch.autograd import Variable\n", 17 | "\n", 18 | "\n", 19 | "import torchvision.transforms as transforms\n", 20 | "from torchvision.utils import save_image\n", 21 | "from torchvision import datasets" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "num_eps=10\n", 31 | "bsize=32\n", 32 | "lrate=0.001\n", 33 | "lat_dimension=64\n", 34 | "image_sz=64\n", 35 | "chnls=1\n", 36 | "logging_intv=200" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "class GANGenerator(nn.Module):\n", 46 | " def __init__(self):\n", 47 | " super(GANGenerator, self).__init__()\n", 48 | " self.inp_sz = image_sz // 4\n", 49 | " self.lin = nn.Linear(lat_dimension, 128 * self.inp_sz ** 2)\n", 50 | " self.bn1 = nn.BatchNorm2d(128)\n", 51 | " self.up1 = nn.Upsample(scale_factor=2)\n", 52 | " self.cn1 = nn.Conv2d(128, 128, 3, stride=1, padding=1)\n", 53 | " self.bn2 = nn.BatchNorm2d(128, 0.8)\n", 54 | " self.rl1 = nn.LeakyReLU(0.2, inplace=True)\n", 55 | " self.up2 = nn.Upsample(scale_factor=2)\n", 56 | " self.cn2 = nn.Conv2d(128, 64, 3, stride=1, padding=1)\n", 57 | " self.bn3 = nn.BatchNorm2d(64, 0.8)\n", 58 | " self.rl2 = nn.LeakyReLU(0.2, inplace=True)\n", 59 | " self.cn3 = nn.Conv2d(64, chnls, 3, stride=1, padding=1)\n", 60 | " self.act = nn.Tanh()\n", 61 | "\n", 62 | " def forward(self, x):\n", 63 | " x = self.lin(x)\n", 64 | " x = x.view(x.shape[0], 128, self.inp_sz, self.inp_sz)\n", 65 | " x = self.bn1(x)\n", 66 | " x = self.up1(x)\n", 67 | " x = self.cn1(x)\n", 68 | " x = self.bn2(x)\n", 69 | " x = self.rl1(x)\n", 70 | " x = self.up2(x)\n", 71 | " x = self.cn2(x)\n", 72 | " x = self.bn3(x)\n", 73 | " x = self.rl2(x)\n", 74 | " x = self.cn3(x)\n", 75 | " out = self.act(x)\n", 76 | " return out" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "class GANDiscriminator(nn.Module):\n", 86 | " def __init__(self):\n", 87 | " super(GANDiscriminator, self).__init__()\n", 88 | "\n", 89 | " def disc_module(ip_chnls, op_chnls, bnorm=True):\n", 90 | " mod = [nn.Conv2d(ip_chnls, op_chnls, 3, 2, 1), \n", 91 | " nn.LeakyReLU(0.2, inplace=True), \n", 92 | " nn.Dropout2d(0.25)]\n", 93 | " if bnorm:\n", 94 | " mod += [nn.BatchNorm2d(op_chnls, 0.8)]\n", 95 | " return mod\n", 96 | "\n", 97 | " self.disc_model = nn.Sequential(\n", 98 | " *disc_module(chnls, 16, bnorm=False),\n", 99 | " *disc_module(16, 32),\n", 100 | " *disc_module(32, 64),\n", 101 | " *disc_module(64, 128),\n", 102 | " )\n", 103 | "\n", 104 | " # width and height of the down-sized image\n", 105 | " ds_size = image_sz // 2 ** 4\n", 106 | " self.adverse_lyr = nn.Sequential(nn.Linear(128 * ds_size ** 2, 1), nn.Sigmoid())\n", 107 | "\n", 108 | " def forward(self, x):\n", 109 | " x = self.disc_model(x)\n", 110 | " x = x.view(x.shape[0], -1)\n", 111 | " out = self.adverse_lyr(x)\n", 112 | " return out" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [ 121 | "# instantiate the discriminator and generator models\n", 122 | "gen = GANGenerator()\n", 123 | "disc = GANDiscriminator()\n", 124 | "\n", 125 | "# define the loss metric\n", 126 | "adv_loss_func = torch.nn.BCELoss()" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "# define the dataset and corresponding dataloader\n", 136 | "dloader = torch.utils.data.DataLoader(\n", 137 | " datasets.MNIST(\n", 138 | " \"./data/mnist/\",\n", 139 | " download=True,\n", 140 | " transform=transforms.Compose(\n", 141 | " [transforms.Resize((image_sz, image_sz)), \n", 142 | " transforms.ToTensor(), transforms.Normalize([0.5], [0.5])]\n", 143 | " ),\n", 144 | " ),\n", 145 | " batch_size=bsize,\n", 146 | " shuffle=True,\n", 147 | ")\n", 148 | "\n", 149 | "# define the optimization schedule for both G and D\n", 150 | "opt_gen = torch.optim.Adam(gen.parameters(), lr=lrate)\n", 151 | "opt_disc = torch.optim.Adam(disc.parameters(), lr=lrate)" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "os.makedirs(\"./images_mnist\", exist_ok=True)\n", 161 | "\n", 162 | "for ep in range(num_eps):\n", 163 | " for idx, (images, _) in enumerate(dloader):\n", 164 | "\n", 165 | " # generate grounnd truths for real and fake images\n", 166 | " good_img = Variable(torch.FloatTensor(images.shape[0], 1).fill_(1.0), requires_grad=False)\n", 167 | " bad_img = Variable(torch.FloatTensor(images.shape[0], 1).fill_(0.0), requires_grad=False)\n", 168 | "\n", 169 | " # get a real image\n", 170 | " actual_images = Variable(images.type(torch.FloatTensor))\n", 171 | "\n", 172 | " # train the generator model\n", 173 | " opt_gen.zero_grad()\n", 174 | "\n", 175 | " # generate a batch of images based on random noise as input\n", 176 | " noise = Variable(torch.FloatTensor(np.random.normal(0, 1, (images.shape[0], lat_dimension))))\n", 177 | " gen_images = gen(noise)\n", 178 | "\n", 179 | " # generator model optimization - how well can it fool the discriminator\n", 180 | " generator_loss = adv_loss_func(disc(gen_images), good_img)\n", 181 | " generator_loss.backward()\n", 182 | " opt_gen.step()\n", 183 | "\n", 184 | " # train the discriminator model\n", 185 | " opt_disc.zero_grad()\n", 186 | "\n", 187 | " # calculate discriminator loss as average of mistakes(losses) in confusing real images as fake and vice versa\n", 188 | " actual_image_loss = adv_loss_func(disc(actual_images), good_img)\n", 189 | " fake_image_loss = adv_loss_func(disc(gen_images.detach()), bad_img)\n", 190 | " discriminator_loss = (actual_image_loss + fake_image_loss) / 2\n", 191 | "\n", 192 | " # discriminator model optimization\n", 193 | " discriminator_loss.backward()\n", 194 | " opt_disc.step()\n", 195 | "\n", 196 | " batches_completed = ep * len(dloader) + idx\n", 197 | " if batches_completed % logging_intv == 0:\n", 198 | " print(f\"epoch number {ep} | batch number {idx} | generator loss = {generator_loss.item()} | discriminator loss = {discriminator_loss.item()}\")\n", 199 | " save_image(gen_images.data[:25], f\"images_mnist/{batches_completed}.png\", nrow=5, normalize=True)" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [] 208 | } 209 | ], 210 | "metadata": { 211 | "kernelspec": { 212 | "display_name": "Python 3", 213 | "language": "python", 214 | "name": "python3" 215 | }, 216 | "language_info": { 217 | "codemirror_mode": { 218 | "name": "ipython", 219 | "version": 3 220 | }, 221 | "file_extension": ".py", 222 | "mimetype": "text/x-python", 223 | "name": "python", 224 | "nbconvert_exporter": "python", 225 | "pygments_lexer": "ipython3", 226 | "version": "3.7.6" 227 | } 228 | }, 229 | "nbformat": 4, 230 | "nbformat_minor": 4 231 | } 232 | -------------------------------------------------------------------------------- /Chapter08/pix2pix_architecture.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "import torch.nn as nn" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "class UNetGenerator(nn.Module):\n", 20 | " def __init__(self, chnls_in=3, chnls_op=3):\n", 21 | " super(UNetGenerator, self).__init__()\n", 22 | " self.down_conv_layer_1 = DownConvBlock(chnls_in, 64, norm=False)\n", 23 | " self.down_conv_layer_2 = DownConvBlock(64, 128)\n", 24 | " self.down_conv_layer_3 = DownConvBlock(128, 256)\n", 25 | " self.down_conv_layer_4 = DownConvBlock(256, 512, dropout=0.5)\n", 26 | " self.down_conv_layer_5 = DownConvBlock(512, 512, dropout=0.5)\n", 27 | " self.down_conv_layer_6 = DownConvBlock(512, 512, dropout=0.5)\n", 28 | " self.down_conv_layer_7 = DownConvBlock(512, 512, dropout=0.5)\n", 29 | " self.down_conv_layer_8 = DownConvBlock(512, 512, norm=False, dropout=0.5)\n", 30 | " self.up_conv_layer_1 = UpConvBlock(512, 512, dropout=0.5)\n", 31 | " self.up_conv_layer_2 = UpConvBlock(1024, 512, dropout=0.5)\n", 32 | " self.up_conv_layer_3 = UpConvBlock(1024, 512, dropout=0.5)\n", 33 | " self.up_conv_layer_4 = UpConvBlock(1024, 512, dropout=0.5)\n", 34 | " self.up_conv_layer_5 = UpConvBlock(1024, 256)\n", 35 | " self.up_conv_layer_6 = UpConvBlock(512, 128)\n", 36 | " self.up_conv_layer_7 = UpConvBlock(256, 64)\n", 37 | " self.upsample_layer = nn.Upsample(scale_factor=2)\n", 38 | " self.zero_pad = nn.ZeroPad2d((1, 0, 1, 0))\n", 39 | " self.conv_layer_1 = nn.Conv2d(128, chnls_op, 4, padding=1)\n", 40 | " self.activation = nn.Tanh()\n", 41 | " def forward(self, x):\n", 42 | " enc1 = self.down_conv_layer_1(x)\n", 43 | " enc2 = self.down_conv_layer_2(enc1)\n", 44 | " enc3 = self.down_conv_layer_3(enc2)\n", 45 | " enc4 = self.down_conv_layer_4(enc3)\n", 46 | " enc5 = self.down_conv_layer_5(enc4)\n", 47 | " enc6 = self.down_conv_layer_6(enc5)\n", 48 | " enc7 = self.down_conv_layer_7(enc6)\n", 49 | " enc8 = self.down_conv_layer_8(enc7)\n", 50 | " dec1 = self.up_conv_layer_1(enc8, enc7)\n", 51 | " dec2 = self.up_conv_layer_2(dec1, enc6)\n", 52 | " dec3 = self.up_conv_layer_3(dec2, enc5)\n", 53 | " dec4 = self.up_conv_layer_4(dec3, enc4)\n", 54 | " dec5 = self.up_conv_layer_5(dec4, enc3)\n", 55 | " dec6 = self.up_conv_layer_6(dec5, enc2)\n", 56 | " dec7 = self.up_conv_layer_7(dec6, enc1)\n", 57 | " final = self.upsample_layer(dec7)\n", 58 | " final = self.zero_pad(final)\n", 59 | " final = self.conv_layer_1(final)\n", 60 | " return self.activation(final)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 3, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "class UpConvBlock(nn.Module):\n", 70 | " def __init__(self, ip_sz, op_sz, dropout=0.0):\n", 71 | " super(UpConvBlock, self).__init__()\n", 72 | " self.layers = [\n", 73 | " nn.ConvTranspose2d(ip_sz, op_sz, 4, 2, 1),\n", 74 | " nn.InstanceNorm2d(op_sz),\n", 75 | " nn.ReLU(),\n", 76 | " ]\n", 77 | " if dropout:\n", 78 | " self.layers += [nn.Dropout(dropout)]\n", 79 | " def forward(self, x, enc_ip):\n", 80 | " x = nn.Sequential(*(self.layers))(x)\n", 81 | " op = torch.cat((x, enc_ip), 1)\n", 82 | " return op" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 4, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "class DownConvBlock(nn.Module):\n", 92 | " def __init__(self, ip_sz, op_sz, norm=True, dropout=0.0):\n", 93 | " super(DownConvBlock, self).__init__()\n", 94 | " self.layers = [nn.Conv2d(ip_sz, op_sz, 4, 2, 1)]\n", 95 | " if norm:\n", 96 | " self.layers.append(nn.InstanceNorm2d(op_sz))\n", 97 | " self.layers += [nn.LeakyReLU(0.2)]\n", 98 | " if dropout:\n", 99 | " self.layers += [nn.Dropout(dropout)]\n", 100 | " def forward(self, x):\n", 101 | " op = nn.Sequential(*(self.layers))(x)\n", 102 | " return op" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 5, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "class Pix2PixDiscriminator(nn.Module):\n", 112 | " def __init__(self, chnls_in=3):\n", 113 | " super(Pix2PixDiscriminator, self).__init__()\n", 114 | " def disc_conv_block(chnls_in, chnls_op, norm=1):\n", 115 | " layers = [nn.Conv2d(chnls_in, chnls_op, 4, stride=2, padding=1)]\n", 116 | " if normalization:\n", 117 | " layers.append(nn.InstanceNorm2d(chnls_op))\n", 118 | " layers.append(nn.LeakyReLU(0.2, inplace=True))\n", 119 | " return layers\n", 120 | " self.lyr1 = disc_conv_block(chnls_in * 2, 64, norm=0)\n", 121 | " self.lyr2 = disc_conv_block(64, 128)\n", 122 | " self.lyr3 = disc_conv_block(128, 256)\n", 123 | " self.lyr4 = disc_conv_block(256, 512)\n", 124 | " def forward(self, real_image, translated_image):\n", 125 | " ip = torch.cat((real_image, translated_image), 1)\n", 126 | " op = self.lyr1(ip)\n", 127 | " op = self.lyr2(op)\n", 128 | " op = self.lyr3(op)\n", 129 | " op = self.lyr4(op)\n", 130 | " op = nn.ZeroPad2d((1, 0, 1, 0))(op)\n", 131 | " op = nn.Conv2d(512, 1, 4, padding=1)(op)\n", 132 | " return op" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [] 141 | } 142 | ], 143 | "metadata": { 144 | "kernelspec": { 145 | "display_name": "Python 3", 146 | "language": "python", 147 | "name": "python3" 148 | }, 149 | "language_info": { 150 | "codemirror_mode": { 151 | "name": "ipython", 152 | "version": 3 153 | }, 154 | "file_extension": ".py", 155 | "mimetype": "text/x-python", 156 | "name": "python", 157 | "nbconvert_exporter": "python", 158 | "pygments_lexer": "ipython3", 159 | "version": "3.7.6" 160 | } 161 | }, 162 | "nbformat": 4, 163 | "nbformat_minor": 4 164 | } 165 | -------------------------------------------------------------------------------- /Chapter10/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | 3 | RUN apt-get -q update && apt-get -q install -y wget 4 | 5 | COPY ./server.py ./ 6 | COPY ./requirements.txt ./ 7 | 8 | RUN wget -q https://raw.githubusercontent.com/wikibook/mpytc/main/Chapter10/convnet.pth 9 | RUN wget -q https://github.com/wikibook/mpytc/raw/main/Chapter10/digit_image.jpg 10 | 11 | RUN pip install --no-cache-dir -r requirements.txt 12 | 13 | 14 | USER root 15 | ENTRYPOINT ["python", "server.py"] 16 | -------------------------------------------------------------------------------- /Chapter10/convnet.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter10/convnet.onnx -------------------------------------------------------------------------------- /Chapter10/convnet.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter10/convnet.pb -------------------------------------------------------------------------------- /Chapter10/convnet.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter10/convnet.pth -------------------------------------------------------------------------------- /Chapter10/convnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class ConvNet(nn.Module): 6 | def __init__(self): 7 | super(ConvNet, self).__init__() 8 | self.cn1 = nn.Conv2d(1, 16, 3, 1) 9 | self.cn2 = nn.Conv2d(16, 32, 3, 1) 10 | self.dp1 = nn.Dropout2d(0.10) 11 | self.dp2 = nn.Dropout2d(0.25) 12 | self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32 13 | self.fc2 = nn.Linear(64, 10) 14 | 15 | def forward(self, x): 16 | x = self.cn1(x) 17 | x = F.relu(x) 18 | x = self.cn2(x) 19 | x = F.relu(x) 20 | x = F.max_pool2d(x, 2) 21 | x = self.dp1(x) 22 | x = torch.flatten(x, 1) 23 | x = self.fc1(x) 24 | x = F.relu(x) 25 | x = self.dp2(x) 26 | x = self.fc2(x) 27 | op = F.log_softmax(x, dim=1) 28 | return op -------------------------------------------------------------------------------- /Chapter10/convnet_handler.py: -------------------------------------------------------------------------------- 1 | from torchvision import transforms 2 | from ts.torch_handler.image_classifier import ImageClassifier 3 | 4 | class ConvNetClassifier(ImageClassifier): 5 | """ 6 | Extends the ImageClassifier handler meant to handle image data for image classification. We have adpated it to convert color (RGB) images to grayscale images resized to 28x28 pixels as well as normalizing pixel values. The postprocess method ensures to return the digit with the highest prediction probability. 7 | """ 8 | 9 | image_processing = transforms.Compose([ 10 | transforms.Grayscale(), 11 | transforms.Resize((28, 28)), 12 | transforms.ToTensor(), 13 | transforms.Normalize((0.1302,), (0.3069,)) 14 | ]) 15 | 16 | def postprocess(self, output): 17 | return output.argmax(1).tolist() -------------------------------------------------------------------------------- /Chapter10/cpp_convnet/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 2 | project(cpp_convnet) 3 | find_package(Torch REQUIRED) 4 | find_package(OpenCV REQUIRED) 5 | 6 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") 7 | add_executable(cpp_convnet cpp_convnet.cpp) 8 | target_link_libraries(cpp_convnet pthread) 9 | target_link_libraries(cpp_convnet ${TORCH_LIBRARIES} ${OpenCV_LIBS}) 10 | set_property(TARGET cpp_convnet PROPERTY CXX_STANDARD 14) 11 | -------------------------------------------------------------------------------- /Chapter10/cpp_convnet/cpp_convnet.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace cv; 9 | using namespace std; 10 | 11 | int main(int argc, char **argv) { 12 | Mat img = imread(argv[2], IMREAD_GRAYSCALE); 13 | resize(img, img, Size(28, 28)); 14 | auto input_ = torch::from_blob(img.data, { img.rows, img.cols, img.channels() }, at::kByte); 15 | auto input = input_.permute({2,0,1}).unsqueeze_(0).reshape({1, 1, img.rows, img.cols}).toType(c10::kFloat).div(255); 16 | input = (input - 0.1302) / 0.3069; 17 | auto module = torch::jit::load(argv[1]); 18 | std::vector inputs; 19 | inputs.push_back(input); 20 | auto output_ = module.forward(inputs).toTensor(); 21 | auto output = output_.argmax(1); 22 | cout << output << '\n'; 23 | return 0; 24 | } -------------------------------------------------------------------------------- /Chapter10/digit_image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter10/digit_image.jpg -------------------------------------------------------------------------------- /Chapter10/example.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | app = Flask(__name__) 3 | 4 | @app.route('/') 5 | def hello_world(): 6 | return 'Hello, World!' 7 | 8 | if __name__ == '__main__': 9 | app.run(host='localhost', port=8890) 10 | 11 | -------------------------------------------------------------------------------- /Chapter10/make_request.py: -------------------------------------------------------------------------------- 1 | import io 2 | import json 3 | import requests 4 | from PIL import Image 5 | 6 | from torchvision import transforms 7 | 8 | 9 | image = Image.open("./digit_image.jpg") 10 | 11 | def image_to_tensor(image): 12 | gray_image = transforms.functional.to_grayscale(image) 13 | resized_image = transforms.functional.resize(gray_image, (28, 28)) 14 | input_image_tensor = transforms.functional.to_tensor(resized_image) 15 | input_image_tensor_norm = transforms.functional.normalize(input_image_tensor, (0.1302,), (0.3069,)) 16 | return input_image_tensor_norm 17 | 18 | image_tensor = image_to_tensor(image) 19 | 20 | dimensions = io.StringIO(json.dumps({'dims': list(image_tensor.shape)})) 21 | data = io.BytesIO(bytearray(image_tensor.numpy())) 22 | 23 | r = requests.post('http://localhost:8890/test', 24 | files={'metadata': dimensions, 'data' : data}) 25 | 26 | response = json.loads(r.content) 27 | 28 | print("Predicted digit :", response) -------------------------------------------------------------------------------- /Chapter10/model_scripting.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "import torch.nn as nn\n", 11 | "import torch.nn.functional as F\n", 12 | "import torch.optim as optim\n", 13 | "from torch.utils.data import DataLoader\n", 14 | "from torchvision import datasets, transforms\n", 15 | "\n", 16 | "import numpy as np\n", 17 | "from PIL import Image" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "class ConvNet(nn.Module):\n", 27 | " def __init__(self):\n", 28 | " super(ConvNet, self).__init__()\n", 29 | " self.cn1 = nn.Conv2d(1, 16, 3, 1)\n", 30 | " self.cn2 = nn.Conv2d(16, 32, 3, 1)\n", 31 | " self.dp1 = nn.Dropout2d(0.10)\n", 32 | " self.dp2 = nn.Dropout2d(0.25)\n", 33 | " self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32\n", 34 | " self.fc2 = nn.Linear(64, 10)\n", 35 | " \n", 36 | " def forward(self, x):\n", 37 | " x = self.cn1(x)\n", 38 | " x = F.relu(x)\n", 39 | " x = self.cn2(x)\n", 40 | " x = F.relu(x)\n", 41 | " x = F.max_pool2d(x, 2)\n", 42 | " x = self.dp1(x)\n", 43 | " x = torch.flatten(x, 1)\n", 44 | " x = self.fc1(x)\n", 45 | " x = F.relu(x)\n", 46 | " x = self.dp2(x)\n", 47 | " x = self.fc2(x)\n", 48 | " op = F.log_softmax(x, dim=1)\n", 49 | " return op\n", 50 | " \n", 51 | "model = ConvNet()" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 3, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "data": { 61 | "text/plain": [ 62 | "" 63 | ] 64 | }, 65 | "execution_count": 3, 66 | "metadata": {}, 67 | "output_type": "execute_result" 68 | } 69 | ], 70 | "source": [ 71 | "PATH_TO_MODEL = \"./convnet.pth\"\n", 72 | "model.load_state_dict(torch.load(PATH_TO_MODEL, map_location=\"cpu\"))" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 4, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "model.eval()\n", 82 | "for p in model.parameters():\n", 83 | " p.requires_grad_(False)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 5, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "scripted_model = torch.jit.script(model)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 6, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "data": { 102 | "text/plain": [ 103 | "graph(%self : __torch__.ConvNet,\n", 104 | " %x.1 : Tensor):\n", 105 | " %51 : Function = prim::Constant[name=\"log_softmax\"]()\n", 106 | " %49 : int = prim::Constant[value=3]()\n", 107 | " %40 : Function = prim::Constant[name=\"relu\"]()\n", 108 | " %33 : int = prim::Constant[value=-1]()\n", 109 | " %26 : Function = prim::Constant[name=\"_max_pool2d\"]()\n", 110 | " %20 : int = prim::Constant[value=0]()\n", 111 | " %19 : None = prim::Constant()\n", 112 | " %14 : Function = prim::Constant[name=\"relu\"]()\n", 113 | " %7 : Function = prim::Constant[name=\"relu\"]()\n", 114 | " %6 : bool = prim::Constant[value=0]()\n", 115 | " %17 : int = prim::Constant[value=2]() # :16:28\n", 116 | " %32 : int = prim::Constant[value=1]() # :18:29\n", 117 | " %2 : __torch__.torch.nn.modules.conv.Conv2d = prim::GetAttr[name=\"cn1\"](%self)\n", 118 | " %x.3 : Tensor = prim::CallMethod[name=\"forward\"](%2, %x.1) # :12:12\n", 119 | " %x.5 : Tensor = prim::CallFunction(%7, %x.3, %6) # :13:12\n", 120 | " %9 : __torch__.torch.nn.modules.conv.___torch_mangle_1.Conv2d = prim::GetAttr[name=\"cn2\"](%self)\n", 121 | " %x.7 : Tensor = prim::CallMethod[name=\"forward\"](%9, %x.5) # :14:12\n", 122 | " %x.9 : Tensor = prim::CallFunction(%14, %x.7, %6) # :15:12\n", 123 | " %18 : int[] = prim::ListConstruct(%17, %17)\n", 124 | " %21 : int[] = prim::ListConstruct(%20, %20)\n", 125 | " %23 : int[] = prim::ListConstruct(%32, %32)\n", 126 | " %x.11 : Tensor = prim::CallFunction(%26, %x.9, %18, %19, %21, %23, %6, %6) # :16:12\n", 127 | " %28 : __torch__.torch.nn.modules.dropout.Dropout2d = prim::GetAttr[name=\"dp1\"](%self)\n", 128 | " %x.13 : Tensor = prim::CallMethod[name=\"forward\"](%28, %x.11) # :17:12\n", 129 | " %x.15 : Tensor = aten::flatten(%x.13, %32, %33) # :18:12\n", 130 | " %35 : __torch__.torch.nn.modules.linear.Linear = prim::GetAttr[name=\"fc1\"](%self)\n", 131 | " %x.17 : Tensor = prim::CallMethod[name=\"forward\"](%35, %x.15) # :19:12\n", 132 | " %x.19 : Tensor = prim::CallFunction(%40, %x.17, %6) # :20:12\n", 133 | " %42 : __torch__.torch.nn.modules.dropout.___torch_mangle_2.Dropout2d = prim::GetAttr[name=\"dp2\"](%self)\n", 134 | " %x.21 : Tensor = prim::CallMethod[name=\"forward\"](%42, %x.19) # :21:12\n", 135 | " %45 : __torch__.torch.nn.modules.linear.___torch_mangle_3.Linear = prim::GetAttr[name=\"fc2\"](%self)\n", 136 | " %x.23 : Tensor = prim::CallMethod[name=\"forward\"](%45, %x.21) # :22:12\n", 137 | " %op.1 : Tensor = prim::CallFunction(%51, %x.23, %32, %49, %19) # :23:13\n", 138 | " return (%op.1)" 139 | ] 140 | }, 141 | "execution_count": 6, 142 | "metadata": {}, 143 | "output_type": "execute_result" 144 | } 145 | ], 146 | "source": [ 147 | "scripted_model.graph" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 7, 153 | "metadata": {}, 154 | "outputs": [ 155 | { 156 | "name": "stdout", 157 | "output_type": "stream", 158 | "text": [ 159 | "def forward(self,\n", 160 | " x: Tensor) -> Tensor:\n", 161 | " _0 = __torch__.torch.nn.functional.___torch_mangle_12.relu\n", 162 | " _1 = __torch__.torch.nn.functional._max_pool2d\n", 163 | " _2 = __torch__.torch.nn.functional.___torch_mangle_13.relu\n", 164 | " _3 = __torch__.torch.nn.functional.log_softmax\n", 165 | " x0 = (self.cn1).forward(x, )\n", 166 | " x1 = __torch__.torch.nn.functional.relu(x0, False, )\n", 167 | " x2 = (self.cn2).forward(x1, )\n", 168 | " x3 = _0(x2, False, )\n", 169 | " x4 = _1(x3, [2, 2], None, [0, 0], [1, 1], False, False, )\n", 170 | " x5 = (self.dp1).forward(x4, )\n", 171 | " x6 = torch.flatten(x5, 1, -1)\n", 172 | " x7 = (self.fc1).forward(x6, )\n", 173 | " x8 = _2(x7, False, )\n", 174 | " x9 = (self.dp2).forward(x8, )\n", 175 | " x10 = (self.fc2).forward(x9, )\n", 176 | " return _3(x10, 1, 3, None, )\n", 177 | "\n" 178 | ] 179 | } 180 | ], 181 | "source": [ 182 | "print(scripted_model.code)" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 8, 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "torch.jit.save(scripted_model, 'scripted_convnet.pt')" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 9, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "loaded_scripted_model = torch.jit.load('scripted_convnet.pt')" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 10, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "image = Image.open(\"./digit_image.jpg\")" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": 11, 215 | "metadata": {}, 216 | "outputs": [], 217 | "source": [ 218 | "def image_to_tensor(image):\n", 219 | " gray_image = transforms.functional.to_grayscale(image)\n", 220 | " resized_image = transforms.functional.resize(gray_image, (28, 28))\n", 221 | " input_image_tensor = transforms.functional.to_tensor(resized_image)\n", 222 | " input_image_tensor_norm = transforms.functional.normalize(input_image_tensor, (0.1302,), (0.3069,))\n", 223 | " return input_image_tensor_norm" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 12, 229 | "metadata": {}, 230 | "outputs": [], 231 | "source": [ 232 | "input_tensor = image_to_tensor(image)" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": 13, 238 | "metadata": {}, 239 | "outputs": [ 240 | { 241 | "data": { 242 | "text/plain": [ 243 | "tensor([[-9.3505e+00, -1.2089e+01, -2.2391e-03, -8.9248e+00, -9.8197e+00,\n", 244 | " -1.3350e+01, -9.0460e+00, -1.4492e+01, -6.3023e+00, -1.2283e+01]])" 245 | ] 246 | }, 247 | "execution_count": 13, 248 | "metadata": {}, 249 | "output_type": "execute_result" 250 | } 251 | ], 252 | "source": [ 253 | "loaded_scripted_model(input_tensor.unsqueeze(0))" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": 14, 259 | "metadata": {}, 260 | "outputs": [ 261 | { 262 | "data": { 263 | "text/plain": [ 264 | "tensor([[-9.3505e+00, -1.2089e+01, -2.2391e-03, -8.9248e+00, -9.8197e+00,\n", 265 | " -1.3350e+01, -9.0460e+00, -1.4492e+01, -6.3023e+00, -1.2283e+01]])" 266 | ] 267 | }, 268 | "execution_count": 14, 269 | "metadata": {}, 270 | "output_type": "execute_result" 271 | } 272 | ], 273 | "source": [ 274 | "model(input_tensor.unsqueeze(0))" 275 | ] 276 | } 277 | ], 278 | "metadata": { 279 | "kernelspec": { 280 | "display_name": "Python 3", 281 | "language": "python", 282 | "name": "python3" 283 | }, 284 | "language_info": { 285 | "codemirror_mode": { 286 | "name": "ipython", 287 | "version": 3 288 | }, 289 | "file_extension": ".py", 290 | "mimetype": "text/x-python", 291 | "name": "python", 292 | "nbconvert_exporter": "python", 293 | "pygments_lexer": "ipython3", 294 | "version": "3.7.6" 295 | } 296 | }, 297 | "nbformat": 4, 298 | "nbformat_minor": 4 299 | } 300 | -------------------------------------------------------------------------------- /Chapter10/model_store/convnet.mar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter10/model_store/convnet.mar -------------------------------------------------------------------------------- /Chapter10/model_tracing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "import torch.nn as nn\n", 11 | "import torch.nn.functional as F\n", 12 | "import torch.optim as optim\n", 13 | "from torch.utils.data import DataLoader\n", 14 | "from torchvision import datasets, transforms\n", 15 | "\n", 16 | "import numpy as np\n", 17 | "from PIL import Image" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "class ConvNet(nn.Module):\n", 27 | " def __init__(self):\n", 28 | " super(ConvNet, self).__init__()\n", 29 | " self.cn1 = nn.Conv2d(1, 16, 3, 1)\n", 30 | " self.cn2 = nn.Conv2d(16, 32, 3, 1)\n", 31 | " self.dp1 = nn.Dropout2d(0.10)\n", 32 | " self.dp2 = nn.Dropout2d(0.25)\n", 33 | " self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32\n", 34 | " self.fc2 = nn.Linear(64, 10)\n", 35 | " \n", 36 | " def forward(self, x):\n", 37 | " x = self.cn1(x)\n", 38 | " x = F.relu(x)\n", 39 | " x = self.cn2(x)\n", 40 | " x = F.relu(x)\n", 41 | " x = F.max_pool2d(x, 2)\n", 42 | " x = self.dp1(x)\n", 43 | " x = torch.flatten(x, 1)\n", 44 | " x = self.fc1(x)\n", 45 | " x = F.relu(x)\n", 46 | " x = self.dp2(x)\n", 47 | " x = self.fc2(x)\n", 48 | " op = F.log_softmax(x, dim=1)\n", 49 | " return op\n", 50 | " \n", 51 | "model = ConvNet()" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 3, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "data": { 61 | "text/plain": [ 62 | "" 63 | ] 64 | }, 65 | "execution_count": 3, 66 | "metadata": {}, 67 | "output_type": "execute_result" 68 | } 69 | ], 70 | "source": [ 71 | "PATH_TO_MODEL = \"./convnet.pth\"\n", 72 | "model.load_state_dict(torch.load(PATH_TO_MODEL, map_location=\"cpu\"))" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 4, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "model.eval()\n", 82 | "for p in model.parameters():\n", 83 | " p.requires_grad_(False)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 5, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "demo_input = torch.ones(1, 1, 28, 28)\n", 93 | "traced_model = torch.jit.trace(model, demo_input)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 6, 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "data": { 103 | "text/plain": [ 104 | "graph(%self.1 : __torch__.torch.nn.modules.module.___torch_mangle_6.Module,\n", 105 | " %input.1 : Float(1, 1, 28, 28)):\n", 106 | " %113 : __torch__.torch.nn.modules.module.___torch_mangle_5.Module = prim::GetAttr[name=\"fc2\"](%self.1)\n", 107 | " %110 : __torch__.torch.nn.modules.module.___torch_mangle_3.Module = prim::GetAttr[name=\"dp2\"](%self.1)\n", 108 | " %109 : __torch__.torch.nn.modules.module.___torch_mangle_4.Module = prim::GetAttr[name=\"fc1\"](%self.1)\n", 109 | " %106 : __torch__.torch.nn.modules.module.___torch_mangle_2.Module = prim::GetAttr[name=\"dp1\"](%self.1)\n", 110 | " %105 : __torch__.torch.nn.modules.module.___torch_mangle_1.Module = prim::GetAttr[name=\"cn2\"](%self.1)\n", 111 | " %102 : __torch__.torch.nn.modules.module.Module = prim::GetAttr[name=\"cn1\"](%self.1)\n", 112 | " %120 : Tensor = prim::CallMethod[name=\"forward\"](%102, %input.1)\n", 113 | " %input.3 : Float(1, 16, 26, 26) = aten::relu(%120) # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:914:0\n", 114 | " %121 : Tensor = prim::CallMethod[name=\"forward\"](%105, %input.3)\n", 115 | " %input.5 : Float(1, 32, 24, 24) = aten::relu(%121) # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:914:0\n", 116 | " %61 : int = prim::Constant[value=2]() # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:488:0\n", 117 | " %62 : int = prim::Constant[value=2]() # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:488:0\n", 118 | " %63 : int[] = prim::ListConstruct(%61, %62)\n", 119 | " %64 : int[] = prim::ListConstruct()\n", 120 | " %65 : int = prim::Constant[value=0]() # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:488:0\n", 121 | " %66 : int = prim::Constant[value=0]() # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:488:0\n", 122 | " %67 : int[] = prim::ListConstruct(%65, %66)\n", 123 | " %68 : int = prim::Constant[value=1]() # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:488:0\n", 124 | " %69 : int = prim::Constant[value=1]() # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:488:0\n", 125 | " %70 : int[] = prim::ListConstruct(%68, %69)\n", 126 | " %71 : bool = prim::Constant[value=0]() # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:488:0\n", 127 | " %input.6 : Float(1, 32, 12, 12) = aten::max_pool2d(%input.5, %63, %64, %67, %70, %71) # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:488:0\n", 128 | " %122 : Tensor = prim::CallMethod[name=\"forward\"](%106, %input.6)\n", 129 | " %76 : int = prim::Constant[value=1]() # :18:0\n", 130 | " %77 : int = prim::Constant[value=-1]() # :18:0\n", 131 | " %input.7 : Float(1, 4608) = aten::flatten(%122, %76, %77) # :18:0\n", 132 | " %123 : Tensor = prim::CallMethod[name=\"forward\"](%109, %input.7)\n", 133 | " %input.9 : Float(1, 64) = aten::relu(%123) # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:914:0\n", 134 | " %124 : Tensor = prim::CallMethod[name=\"forward\"](%110, %input.9)\n", 135 | " %125 : Tensor = prim::CallMethod[name=\"forward\"](%113, %124)\n", 136 | " %91 : int = prim::Constant[value=1]() # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:1317:0\n", 137 | " %92 : None = prim::Constant()\n", 138 | " %93 : Float(1, 10) = aten::log_softmax(%125, %91, %92) # /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py:1317:0\n", 139 | " return (%93)" 140 | ] 141 | }, 142 | "execution_count": 6, 143 | "metadata": {}, 144 | "output_type": "execute_result" 145 | } 146 | ], 147 | "source": [ 148 | "traced_model.graph" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 7, 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "name": "stdout", 158 | "output_type": "stream", 159 | "text": [ 160 | "def forward(self,\n", 161 | " input: Tensor) -> Tensor:\n", 162 | " _0 = self.fc2\n", 163 | " _1 = self.dp2\n", 164 | " _2 = self.fc1\n", 165 | " _3 = self.dp1\n", 166 | " _4 = self.cn2\n", 167 | " input0 = torch.relu((self.cn1).forward(input, ))\n", 168 | " input1 = torch.relu((_4).forward(input0, ))\n", 169 | " input2 = torch.max_pool2d(input1, [2, 2], annotate(List[int], []), [0, 0], [1, 1], False)\n", 170 | " input3 = torch.flatten((_3).forward(input2, ), 1, -1)\n", 171 | " input4 = torch.relu((_2).forward(input3, ))\n", 172 | " _5 = (_0).forward((_1).forward(input4, ), )\n", 173 | " return torch.log_softmax(_5, 1, None)\n", 174 | "\n" 175 | ] 176 | } 177 | ], 178 | "source": [ 179 | "print(traced_model.code)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 8, 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [ 188 | "torch.jit.save(traced_model, 'traced_convnet.pt')" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 9, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "loaded_traced_model = torch.jit.load('traced_convnet.pt')" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": 10, 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "image = Image.open(\"./digit_image.jpg\")" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 11, 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "def image_to_tensor(image):\n", 216 | " gray_image = transforms.functional.to_grayscale(image)\n", 217 | " resized_image = transforms.functional.resize(gray_image, (28, 28))\n", 218 | " input_image_tensor = transforms.functional.to_tensor(resized_image)\n", 219 | " input_image_tensor_norm = transforms.functional.normalize(input_image_tensor, (0.1302,), (0.3069,))\n", 220 | " return input_image_tensor_norm" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": 12, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "input_tensor = image_to_tensor(image)" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": 13, 235 | "metadata": {}, 236 | "outputs": [ 237 | { 238 | "data": { 239 | "text/plain": [ 240 | "tensor([[-9.3505e+00, -1.2089e+01, -2.2391e-03, -8.9248e+00, -9.8197e+00,\n", 241 | " -1.3350e+01, -9.0460e+00, -1.4492e+01, -6.3023e+00, -1.2283e+01]])" 242 | ] 243 | }, 244 | "execution_count": 13, 245 | "metadata": {}, 246 | "output_type": "execute_result" 247 | } 248 | ], 249 | "source": [ 250 | "loaded_traced_model(input_tensor.unsqueeze(0))" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 14, 256 | "metadata": {}, 257 | "outputs": [ 258 | { 259 | "data": { 260 | "text/plain": [ 261 | "tensor([[-9.3505e+00, -1.2089e+01, -2.2391e-03, -8.9248e+00, -9.8197e+00,\n", 262 | " -1.3350e+01, -9.0460e+00, -1.4492e+01, -6.3023e+00, -1.2283e+01]])" 263 | ] 264 | }, 265 | "execution_count": 14, 266 | "metadata": {}, 267 | "output_type": "execute_result" 268 | } 269 | ], 270 | "source": [ 271 | "model(input_tensor.unsqueeze(0))" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": null, 277 | "metadata": {}, 278 | "outputs": [], 279 | "source": [] 280 | } 281 | ], 282 | "metadata": { 283 | "kernelspec": { 284 | "display_name": "Python 3", 285 | "language": "python", 286 | "name": "python3" 287 | }, 288 | "language_info": { 289 | "codemirror_mode": { 290 | "name": "ipython", 291 | "version": 3 292 | }, 293 | "file_extension": ".py", 294 | "mimetype": "text/x-python", 295 | "name": "python", 296 | "nbconvert_exporter": "python", 297 | "pygments_lexer": "ipython3", 298 | "version": "3.7.6" 299 | } 300 | }, 301 | "nbformat": 4, 302 | "nbformat_minor": 4 303 | } 304 | -------------------------------------------------------------------------------- /Chapter10/onnx.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "import torch.nn as nn\n", 11 | "import torch.nn.functional as F\n", 12 | "import torch.optim as optim\n", 13 | "from torch.utils.data import DataLoader\n", 14 | "from torchvision import datasets, transforms\n", 15 | "\n", 16 | "import numpy as np\n", 17 | "from PIL import Image\n", 18 | "\n", 19 | "import tensorflow as tf" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "class ConvNet(nn.Module):\n", 29 | " def __init__(self):\n", 30 | " super(ConvNet, self).__init__()\n", 31 | " self.cn1 = nn.Conv2d(1, 16, 3, 1)\n", 32 | " self.cn2 = nn.Conv2d(16, 32, 3, 1)\n", 33 | " self.dp1 = nn.Dropout2d(0.10)\n", 34 | " self.dp2 = nn.Dropout2d(0.25)\n", 35 | " self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32\n", 36 | " self.fc2 = nn.Linear(64, 10)\n", 37 | " \n", 38 | " def forward(self, x):\n", 39 | " x = self.cn1(x)\n", 40 | " x = F.relu(x)\n", 41 | " x = self.cn2(x)\n", 42 | " x = F.relu(x)\n", 43 | " x = F.max_pool2d(x, 2)\n", 44 | " x = self.dp1(x)\n", 45 | " x = torch.flatten(x, 1)\n", 46 | " x = self.fc1(x)\n", 47 | " x = F.relu(x)\n", 48 | " x = self.dp2(x)\n", 49 | " x = self.fc2(x)\n", 50 | " op = F.log_softmax(x, dim=1)\n", 51 | " return op\n", 52 | " \n", 53 | "model = ConvNet()" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/plain": [ 64 | "" 65 | ] 66 | }, 67 | "execution_count": 3, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | } 71 | ], 72 | "source": [ 73 | "PATH_TO_MODEL = \"./convnet.pth\"\n", 74 | "model.load_state_dict(torch.load(PATH_TO_MODEL, map_location=\"cpu\"))" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 6, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "image = Image.open(\"./digit_image.jpg\")" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 7, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "def image_to_tensor(image):\n", 93 | " gray_image = transforms.functional.to_grayscale(image)\n", 94 | " resized_image = transforms.functional.resize(gray_image, (28, 28))\n", 95 | " input_image_tensor = transforms.functional.to_tensor(resized_image)\n", 96 | " input_image_tensor_norm = transforms.functional.normalize(input_image_tensor, (0.1302,), (0.3069,))\n", 97 | " return input_image_tensor_norm" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 8, 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "input_tensor = image_to_tensor(image)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 4, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "model.eval()\n", 116 | "for p in model.parameters():\n", 117 | " p.requires_grad_(False)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 5, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "demo_input = torch.ones(1, 1, 28, 28)\n", 127 | "torch.onnx.export(model, demo_input, \"convnet.onnx\")" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 9, 133 | "metadata": { 134 | "collapsed": true 135 | }, 136 | "outputs": [ 137 | { 138 | "name": "stdout", 139 | "output_type": "stream", 140 | "text": [ 141 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/ceil.py:10: The name tf.ceil is deprecated. Please use tf.math.ceil instead.\n", 142 | "\n", 143 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/depth_to_space.py:12: The name tf.depth_to_space is deprecated. Please use tf.compat.v1.depth_to_space instead.\n", 144 | "\n", 145 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/erf.py:9: The name tf.erf is deprecated. Please use tf.math.erf instead.\n", 146 | "\n", 147 | "WARNING:tensorflow:\n", 148 | "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", 149 | "For more information, please see:\n", 150 | " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", 151 | " * https://github.com/tensorflow/addons\n", 152 | " * https://github.com/tensorflow/io (for I/O related ops)\n", 153 | "If you depend on functionality not listed there, please file an issue.\n", 154 | "\n" 155 | ] 156 | }, 157 | { 158 | "name": "stderr", 159 | "output_type": "stream", 160 | "text": [ 161 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/__init__.py:89: UserWarning: onnx_tf.common.get_outputs_names is deprecated. It will be removed in future release. Use TensorflowGraph.get_outputs_names instead.\n", 162 | " warnings.warn(message)\n", 163 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/dask/dataframe/utils.py:14: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n", 164 | " import pandas.util.testing as tm\n" 165 | ] 166 | }, 167 | { 168 | "name": "stdout", 169 | "output_type": "stream", 170 | "text": [ 171 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/is_nan.py:9: The name tf.is_nan is deprecated. Please use tf.math.is_nan instead.\n", 172 | "\n", 173 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/log.py:10: The name tf.log is deprecated. Please use tf.math.log instead.\n", 174 | "\n", 175 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/random_normal.py:9: The name tf.random_normal is deprecated. Please use tf.random.normal instead.\n", 176 | "\n", 177 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/random_uniform.py:9: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n", 178 | "\n", 179 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/reciprocal.py:10: The name tf.reciprocal is deprecated. Please use tf.math.reciprocal instead.\n", 180 | "\n", 181 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/space_to_depth.py:12: The name tf.space_to_depth is deprecated. Please use tf.compat.v1.space_to_depth instead.\n", 182 | "\n", 183 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/upsample.py:15: The name tf.image.resize_images is deprecated. Please use tf.image.resize instead.\n", 184 | "\n", 185 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend/xor.py:10: The name tf.logical_xor is deprecated. Please use tf.math.logical_xor instead.\n", 186 | "\n" 187 | ] 188 | }, 189 | { 190 | "name": "stderr", 191 | "output_type": "stream", 192 | "text": [ 193 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:37: UserWarning: Unknown op ConstantFill in domain `ai.onnx`.\n", 194 | " handler.ONNX_OP, handler.DOMAIN or \"ai.onnx\"))\n", 195 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of ConvInteger in domain `` with max_inclusive_version=9. Set to 1.\n", 196 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 197 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of DequantizeLinear in domain `` with max_inclusive_version=9. Set to 1.\n", 198 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 199 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of GatherND in domain `` with max_inclusive_version=9. Set to 1.\n", 200 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 201 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:37: UserWarning: Unknown op ImageScaler in domain `ai.onnx`.\n", 202 | " handler.ONNX_OP, handler.DOMAIN or \"ai.onnx\"))\n", 203 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of IsInf in domain `` with max_inclusive_version=9. Set to 1.\n", 204 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 205 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of MatMulInteger in domain `` with max_inclusive_version=9. Set to 1.\n", 206 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 207 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of Mod in domain `` with max_inclusive_version=9. Set to 1.\n", 208 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 209 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of NonMaxSuppression in domain `` with max_inclusive_version=9. Set to 1.\n", 210 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 211 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of QLinearConv in domain `` with max_inclusive_version=9. Set to 1.\n", 212 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 213 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of QLinearMatMul in domain `` with max_inclusive_version=9. Set to 1.\n", 214 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 215 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of QuantizeLinear in domain `` with max_inclusive_version=9. Set to 1.\n", 216 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 217 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of Range in domain `` with max_inclusive_version=9. Set to 1.\n", 218 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 219 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of Resize in domain `` with max_inclusive_version=9. Set to 1.\n", 220 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 221 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of ReverseSequence in domain `` with max_inclusive_version=9. Set to 1.\n", 222 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 223 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of Round in domain `` with max_inclusive_version=9. Set to 1.\n", 224 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 225 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of ScatterElements in domain `` with max_inclusive_version=9. Set to 1.\n", 226 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 227 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of ScatterND in domain `` with max_inclusive_version=9. Set to 1.\n", 228 | " handler.ONNX_OP, handler.DOMAIN, version))\n", 229 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/common/handler_helper.py:34: UserWarning: Fail to get since_version of ThresholdedRelu in domain `` with max_inclusive_version=9. Set to 1.\n", 230 | " handler.ONNX_OP, handler.DOMAIN, version))\n" 231 | ] 232 | }, 233 | { 234 | "name": "stdout", 235 | "output_type": "stream", 236 | "text": [ 237 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/backend.py:123: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n", 238 | "\n", 239 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/onnx_tf/handlers/backend_handler.py:182: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", 240 | "Instructions for updating:\n", 241 | "Use keras.layers.flatten instead.\n", 242 | "WARNING:tensorflow:From /Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/layers/core.py:332: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n", 243 | "Instructions for updating:\n", 244 | "Please use `layer.__call__` method instead.\n" 245 | ] 246 | } 247 | ], 248 | "source": [ 249 | "import onnx\n", 250 | "from onnx_tf.backend import prepare\n", 251 | "\n", 252 | "model_onnx = onnx.load(\"./convnet.onnx\")\n", 253 | "tf_rep = prepare(model_onnx)\n", 254 | "tf_rep.export_graph(\"./convnet.pb\")" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 10, 260 | "metadata": {}, 261 | "outputs": [ 262 | { 263 | "name": "stdout", 264 | "output_type": "stream", 265 | "text": [ 266 | "(,)\n", 267 | "(,)\n", 268 | "(,)\n", 269 | "(,)\n", 270 | "(,)\n", 271 | "(,)\n", 272 | "(,)\n", 273 | "(,)\n", 274 | "(,)\n", 275 | "(,)\n", 276 | "(,)\n", 277 | "(,)\n", 278 | "(,)\n", 279 | "(,)\n", 280 | "(,)\n", 281 | "(,)\n", 282 | "(,)\n", 283 | "(,)\n", 284 | "(,)\n", 285 | "(,)\n", 286 | "(,)\n", 287 | "(,)\n", 288 | "(,)\n", 289 | "(,)\n", 290 | "(,)\n", 291 | "(,)\n", 292 | "(,)\n", 293 | "(,)\n", 294 | "(,)\n", 295 | "(,)\n", 296 | "(,)\n", 297 | "(,)\n", 298 | "(,)\n", 299 | "(,)\n", 300 | "(,)\n", 301 | "(,)\n", 302 | "(,)\n", 303 | "(,)\n", 304 | "(,)\n", 305 | "(,)\n", 306 | "(,)\n", 307 | "(,)\n", 308 | "(,)\n", 309 | "(,)\n", 310 | "(,)\n", 311 | "(,)\n", 312 | "(,)\n", 313 | "(,)\n", 314 | "(,)\n", 315 | "(,)\n", 316 | "(,)\n", 317 | "(,)\n", 318 | "(,)\n", 319 | "(,)\n", 320 | "(,)\n", 321 | "(,)\n", 322 | "(,)\n", 323 | "(,)\n", 324 | "(,)\n", 325 | "(,)\n", 326 | "(,)\n", 327 | "(,)\n", 328 | "(,)\n", 329 | "(,)\n", 330 | "(,)\n", 331 | "(,)\n", 332 | "(,)\n", 333 | "(,)\n", 334 | "(,)\n", 335 | "(,)\n", 336 | "(,)\n", 337 | "(,)\n", 338 | "(,)\n", 339 | "(,)\n" 340 | ] 341 | } 342 | ], 343 | "source": [ 344 | "with tf.gfile.GFile(\"./convnet.pb\", \"rb\") as f:\n", 345 | " graph_definition = tf.GraphDef()\n", 346 | " graph_definition.ParseFromString(f.read())\n", 347 | " \n", 348 | "with tf.Graph().as_default() as model_graph:\n", 349 | " tf.import_graph_def(graph_definition, name=\"\")\n", 350 | " \n", 351 | "for op in model_graph.get_operations():\n", 352 | " print(op.values())" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": 12, 358 | "metadata": {}, 359 | "outputs": [ 360 | { 361 | "name": "stdout", 362 | "output_type": "stream", 363 | "text": [ 364 | "[[-9.35050774e+00 -1.20893326e+01 -2.23922171e-03 -8.92477798e+00\n", 365 | " -9.81972313e+00 -1.33498535e+01 -9.04598618e+00 -1.44924192e+01\n", 366 | " -6.30233145e+00 -1.22827682e+01]]\n" 367 | ] 368 | } 369 | ], 370 | "source": [ 371 | "model_output = model_graph.get_tensor_by_name('18:0')\n", 372 | "model_input = model_graph.get_tensor_by_name('input.1:0')\n", 373 | "\n", 374 | "sess = tf.Session(graph=model_graph)\n", 375 | "output = sess.run(model_output, feed_dict={model_input: input_tensor.unsqueeze(0)})\n", 376 | "print(output)" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": null, 382 | "metadata": {}, 383 | "outputs": [], 384 | "source": [] 385 | } 386 | ], 387 | "metadata": { 388 | "kernelspec": { 389 | "display_name": "Python 3", 390 | "language": "python", 391 | "name": "python3" 392 | }, 393 | "language_info": { 394 | "codemirror_mode": { 395 | "name": "ipython", 396 | "version": 3 397 | }, 398 | "file_extension": ".py", 399 | "mimetype": "text/x-python", 400 | "name": "python", 401 | "nbconvert_exporter": "python", 402 | "pygments_lexer": "ipython3", 403 | "version": "3.7.6" 404 | } 405 | }, 406 | "nbformat": 4, 407 | "nbformat_minor": 4 408 | } 409 | -------------------------------------------------------------------------------- /Chapter10/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.5.0 2 | torchvision==0.6.0 3 | Pillow==6.2.2 4 | Flask==1.1.1 5 | -------------------------------------------------------------------------------- /Chapter10/run_inference.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "import torch.nn as nn\n", 11 | "import torch.nn.functional as F\n", 12 | "import torch.optim as optim\n", 13 | "from torch.utils.data import DataLoader\n", 14 | "from torchvision import datasets, transforms\n", 15 | "\n", 16 | "import numpy as np\n", 17 | "from PIL import Image" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "class ConvNet(nn.Module):\n", 27 | " def __init__(self):\n", 28 | " super(ConvNet, self).__init__()\n", 29 | " self.cn1 = nn.Conv2d(1, 16, 3, 1)\n", 30 | " self.cn2 = nn.Conv2d(16, 32, 3, 1)\n", 31 | " self.dp1 = nn.Dropout2d(0.10)\n", 32 | " self.dp2 = nn.Dropout2d(0.25)\n", 33 | " self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32\n", 34 | " self.fc2 = nn.Linear(64, 10)\n", 35 | " \n", 36 | " def forward(self, x):\n", 37 | " x = self.cn1(x)\n", 38 | " x = F.relu(x)\n", 39 | " x = self.cn2(x)\n", 40 | " x = F.relu(x)\n", 41 | " x = F.max_pool2d(x, 2)\n", 42 | " x = self.dp1(x)\n", 43 | " x = torch.flatten(x, 1)\n", 44 | " x = self.fc1(x)\n", 45 | " x = F.relu(x)\n", 46 | " x = self.dp2(x)\n", 47 | " x = self.fc2(x)\n", 48 | " op = F.log_softmax(x, dim=1)\n", 49 | " return op\n", 50 | " \n", 51 | "model = ConvNet()" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 3, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "data": { 61 | "text/plain": [ 62 | "" 63 | ] 64 | }, 65 | "execution_count": 3, 66 | "metadata": {}, 67 | "output_type": "execute_result" 68 | } 69 | ], 70 | "source": [ 71 | "PATH_TO_MODEL = \"./convnet.pth\"\n", 72 | "model.load_state_dict(torch.load(PATH_TO_MODEL, map_location=\"cpu\"))" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 4, 78 | "metadata": {}, 79 | "outputs": [ 80 | { 81 | "data": { 82 | "text/plain": [ 83 | "ConvNet(\n", 84 | " (cn1): Conv2d(1, 16, kernel_size=(3, 3), stride=(1, 1))\n", 85 | " (cn2): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n", 86 | " (dp1): Dropout2d(p=0.1, inplace=False)\n", 87 | " (dp2): Dropout2d(p=0.25, inplace=False)\n", 88 | " (fc1): Linear(in_features=4608, out_features=64, bias=True)\n", 89 | " (fc2): Linear(in_features=64, out_features=10, bias=True)\n", 90 | ")" 91 | ] 92 | }, 93 | "execution_count": 4, 94 | "metadata": {}, 95 | "output_type": "execute_result" 96 | } 97 | ], 98 | "source": [ 99 | "model.eval()" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 5, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "image = Image.open(\"./digit_image.jpg\")" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 10, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "data": { 118 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAOGUlEQVR4nO3db6xU9Z3H8c93saBCNSBCbqzZ26IPrGukhpgma9S1tkHBIFEbeLBBRS8PEGtsbAk+qGatMbvblUcSaTS9axDEKBaxhrKAtWtCFYmLqEsRg+Fe+aPLAy4miMB3H8xh96r3/M71zJk54/2+X8nNzJzvnDnfTPhwzpzfzPmZuwvAyPc3dTcAoD0IOxAEYQeCIOxAEIQdCOK0dm7MzDj1D7SYu9tQy5vas5vZdDPbaWbvm9niZl4LQGtZ2XF2Mxsl6a+SfiypT9Ibkua6+7uJddizAy3Wij375ZLed/cP3P2YpFWSZjXxegBaqJmwnydp76DHfdmyLzCzHjPbamZbm9gWgCa1/ASduy+XtFziMB6oUzN79n5J5w96/J1sGYAO1EzY35B0oZl918xGS5ojaW01bQGoWunDeHc/bmZ3SVovaZSkJ939nco6A1Cp0kNvpTbGZ3ag5VrypRoA3xyEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4E0dZLSaM1xo8fn1u75pprkuted911yfott9ySrG/ZsqV0ffv27cl1P/vss2R93bp1yTq+iD07EARhB4Ig7EAQhB0IgrADQRB2IAjCDgTB1WVHgGnTpuXWXn/99eS6mzZtamrbReP4rfTSSy8l67fffntu7eOPP666nY7B1WWB4Ag7EARhB4Ig7EAQhB0IgrADQRB2IAjG2UeAiRMn5tZmzJiRXPeFF15I1gcGBpL10aNHJ+tjx47Nrc2cOTO57pgxY5L1RYsWJevd3d25tZtvvjm57vr165P1TpY3zt7UxSvMbI+kAUknJB139/xvdwCoVRVXqvkHd/+kgtcB0EJ8ZgeCaDbsLumPZvammfUM9QQz6zGzrWa2tcltAWhCs4fxV7h7v5lNkrTBzP7b3V8d/AR3Xy5pucQJOqBOTe3Z3b0/uz0oaY2ky6toCkD1SofdzMaa2bdP3Zf0E0k7qmoMQLVKj7Ob2ffU2JtLjY8DT7v7rwvW4TAelUmN4UvS2rVrc2uTJk1KrnvJJZeU6qkTVD7O7u4fSLq0dEcA2oqhNyAIwg4EQdiBIAg7EARhB4LgJ64YsaZOnZpb27ZtW3Ld2267LVnv7e0t01JbcClpIDjCDgRB2IEgCDsQBGEHgiDsQBCEHQiiigtOAh3p8OHDpde99tprk/VOHmfPw54dCIKwA0EQdiAIwg4EQdiBIAg7EARhB4JgnB3fWKedlv7n++CDD+bWiq7j8Morr5RpqaOxZweCIOxAEIQdCIKwA0EQdiAIwg4EQdiBILhuPL6xbrrppmT92Wefza0dPXo0ue6ZZ55ZqqdOUPq68Wb2pJkdNLMdg5ZNMLMNZrYrux1fZbMAqjecw/jfSZr+pWWLJW109wslbcweA+hghWF391clHfrS4lmSTl2Xp1fSjdW2BaBqZb8bP9nd92X390uanPdEM+uR1FNyOwAq0vQPYdzdUyfe3H25pOUSJ+iAOpUdejtgZl2SlN0erK4lAK1QNuxrJc3L7s+T9Ptq2gHQKoWH8Wa2UtLVkiaaWZ+kX0l6RNJqM5sv6UNJP21lk4jpggsuSNafeuqpZP3kyZO5tSVLlpTq6ZusMOzuPjen9KOKewHQQnxdFgiCsANBEHYgCMIOBEHYgSC4lDRqc9ZZZyXra9asSdZHjx6drN933325taVLlybXHYnYswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIyzjwCnn356bu2GG25Irjt//vxkfe/evcn6smXLkvXdu3fn1h5//PHkuhdffHGy/vLLLyfrjz76aLIeDXt2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCKZs7wLnnnpusX3XVVcn6/fffn1u79NJLS/U0XMeOHUvWU+PsF110UXLdojH+onH4I0eOJOsjVekpmwGMDIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7BU455xzkvV77703Wb/jjjuS9aJx+JT+/v5kfeHChcn60aNHk/VnnnkmWT/77LNza59++mly3aLvF2zbti1Zj6r0OLuZPWlmB81sx6BlD5hZv5m9lf1dX2WzAKo3nMP430maPsTyR919avb3h2rbAlC1wrC7+6uSDrWhFwAt1MwJurvMbHt2mD8+70lm1mNmW81saxPbAtCksmFfJmmKpKmS9kn6Td4T3X25u09z92kltwWgAqXC7u4H3P2Eu5+U9FtJl1fbFoCqlQq7mXUNejhb0o685wLoDIXXjTezlZKuljTRzPok/UrS1WY2VZJL2iNpQeta7Azd3d25tddeey25bldXV7JeZP/+/cn6kiVLcmurVq1KrnvixIlkvWge89Q4epGibR8+fLj0a+OrCsPu7nOHWPxEC3oB0EJ8XRYIgrADQRB2IAjCDgRB2IEg+IlrZsyYMcn66tWrc2tF0yJ//vnnyfpDDz2UrG/YsCFZ37JlS7KeMmfOnGT96aefLv3akrRz587c2pQpU5Lr9vX1JetXXnllU+uPVFxKGgiOsANBEHYgCMIOBEHYgSAIOxAEYQeCKPzVWxTTpw91Tc3/lxpLP378eHLdW2+9NVlfuXJlst6Mou8APPFEcz9g3LVrV7I+c+bM3FrROPnDDz+crBf9vDbqOHse9uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7JmiMd+UzZs3J+vNjqOPGjUqWZ8xY0bpbZ9xxhnJ+qZNm5L12bNnJ+sDAwO5td27dyfXXbFiRbI+bty4ZB1fxJ4dCIKwA0EQdiAIwg4EQdiBIAg7EARhB4JgnD0zadKk0uv29/cn60Vj2Zdddlmyfueddybr8+bNy60VzQvQ29ubrN99993JemocvVnHjh1L1g8dOtSybY9EhXt2MzvfzDab2btm9o6Z/SxbPsHMNpjZrux2fOvbBVDWcA7jj0v6ubt/X9IPJS00s+9LWixpo7tfKGlj9hhAhyoMu7vvc/dt2f0BSe9JOk/SLEmnjgF7Jd3Yoh4BVOBrfWY3s25JP5D0F0mT3X1fVtovaXLOOj2SeproEUAFhn023szGSXpO0j3ufnhwzRtngYY8E+Tuy919mrtPa6pTAE0ZVtjN7FtqBH2Fuz+fLT5gZl1ZvUvSwda0CKAKhVM2m5mp8Zn8kLvfM2j5v0j6H3d/xMwWS5rg7r8oeK2OnbJ5wYIFyfpjjz2WW2u8RZ2paGht0aJFyfqRI0eqbAdtkDdl83A+s/+9pH+U9LaZvZUtWyLpEUmrzWy+pA8l/bSCPgG0SGHY3f0/JeXtun5UbTsAWoWvywJBEHYgCMIOBEHYgSAIOxBE4Th7pRvr4HH2Iqmpj5cuXZpct+inmEWXVH7xxReT9Y8++ii3VnSZa4w8eePs7NmBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjG2YERhnF2IDjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCKIw7GZ2vpltNrN3zewdM/tZtvwBM+s3s7eyv+tb3y6AsgovXmFmXZK63H2bmX1b0puSblRjPvYj7v6vw94YF68AWi7v4hXDmZ99n6R92f0BM3tP0nnVtgeg1b7WZ3Yz65b0A0l/yRbdZWbbzexJMxufs06PmW01s63NtQqgGcO+Bp2ZjZP0J0m/dvfnzWyypE8kuaR/UuNQ//aC1+AwHmixvMP4YYXdzL4laZ2k9e7+b0PUuyWtc/e/K3gdwg60WOkLTpqZSXpC0nuDg56duDtltqQdzTYJoHWGczb+Ckl/lvS2pJPZ4iWS5kqaqsZh/B5JC7KTeanXYs8OtFhTh/FVIexA63HdeCA4wg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCFF5ys2CeSPhz0eGK2rBN1am+d2pdEb2VV2dvf5hXa+nv2r2zcbKu7T6utgYRO7a1T+5Lorax29cZhPBAEYQeCqDvsy2vefkqn9tapfUn0VlZbeqv1MzuA9ql7zw6gTQg7EEQtYTez6Wa208zeN7PFdfSQx8z2mNnb2TTUtc5Pl82hd9DMdgxaNsHMNpjZrux2yDn2auqtI6bxTkwzXut7V/f0523/zG5moyT9VdKPJfVJekPSXHd/t62N5DCzPZKmuXvtX8AwsyslHZH076em1jKzf5Z0yN0fyf6jHO/uv+yQ3h7Q15zGu0W95U0zfqtqfO+qnP68jDr27JdLet/dP3D3Y5JWSZpVQx8dz91flXToS4tnSerN7veq8Y+l7XJ66wjuvs/dt2X3BySdmma81vcu0Vdb1BH28yTtHfS4T50137tL+qOZvWlmPXU3M4TJg6bZ2i9pcp3NDKFwGu92+tI04x3z3pWZ/rxZnKD7qivc/TJJ10lamB2udiRvfAbrpLHTZZKmqDEH4D5Jv6mzmWya8eck3ePuhwfX6nzvhuirLe9bHWHvl3T+oMffyZZ1BHfvz24PSlqjxseOTnLg1Ay62e3Bmvv5P+5+wN1PuPtJSb9Vje9dNs34c5JWuPvz2eLa37uh+mrX+1ZH2N+QdKGZfdfMRkuaI2ltDX18hZmNzU6cyMzGSvqJOm8q6rWS5mX350n6fY29fEGnTOOdN824an7vap/+3N3b/ifpejXOyO+WdH8dPeT09T1J/5X9vVN3b5JWqnFY97ka5zbmSzpH0kZJuyT9h6QJHdTbU2pM7b1djWB11dTbFWocom+X9Fb2d33d712ir7a8b3xdFgiCE3RAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EMT/ArW6jsy0t+A4AAAAAElFTkSuQmCC\n", 119 | "text/plain": [ 120 | "" 121 | ] 122 | }, 123 | "execution_count": 10, 124 | "metadata": {}, 125 | "output_type": "execute_result" 126 | } 127 | ], 128 | "source": [ 129 | "image" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 6, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "def image_to_tensor(image):\n", 139 | " gray_image = transforms.functional.to_grayscale(image)\n", 140 | " resized_image = transforms.functional.resize(gray_image, (28, 28))\n", 141 | " input_image_tensor = transforms.functional.to_tensor(resized_image)\n", 142 | " input_image_tensor_norm = transforms.functional.normalize(input_image_tensor, (0.1302,), (0.3069,))\n", 143 | " return input_image_tensor_norm" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 7, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "input_tensor = image_to_tensor(image)" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": 8, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "def run_model(input_tensor):\n", 162 | " model_input = input_tensor.unsqueeze(0)\n", 163 | " with torch.no_grad():\n", 164 | " model_output = model(model_input)[0]\n", 165 | " model_prediction = model_output.detach().numpy().argmax()\n", 166 | " return model_prediction" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 14, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "name": "stdout", 176 | "output_type": "stream", 177 | "text": [ 178 | "2\n", 179 | "\n" 180 | ] 181 | } 182 | ], 183 | "source": [ 184 | "output = run_model(input_tensor)\n", 185 | "print(output)\n", 186 | "print(type(output))" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 22, 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "def debug_model(input_tensor):\n", 196 | " model_input = input_tensor.unsqueeze(0)\n", 197 | " with torch.no_grad():\n", 198 | " model_output = model(model_input)[0]\n", 199 | " model_prediction = model_output.detach().numpy()\n", 200 | " return np.exp(model_prediction)" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 23, 206 | "metadata": { 207 | "scrolled": false 208 | }, 209 | "outputs": [ 210 | { 211 | "name": "stdout", 212 | "output_type": "stream", 213 | "text": [ 214 | "[8.69212745e-05 5.61913612e-06 9.97763395e-01 1.33050999e-04\n", 215 | " 5.43686365e-05 1.59305739e-06 1.17863165e-04 5.08185963e-07\n", 216 | " 1.83202932e-03 4.63086781e-06]\n" 217 | ] 218 | } 219 | ], 220 | "source": [ 221 | "print(debug_model(input_tensor))" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": 25, 227 | "metadata": {}, 228 | "outputs": [], 229 | "source": [ 230 | "def post_process(output):\n", 231 | " return str(output)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 27, 237 | "metadata": {}, 238 | "outputs": [ 239 | { 240 | "name": "stdout", 241 | "output_type": "stream", 242 | "text": [ 243 | "2\n", 244 | "\n" 245 | ] 246 | } 247 | ], 248 | "source": [ 249 | "final_output = post_process(output)\n", 250 | "print(final_output)\n", 251 | "print(type(final_output))" 252 | ] 253 | } 254 | ], 255 | "metadata": { 256 | "kernelspec": { 257 | "display_name": "Python 3", 258 | "language": "python", 259 | "name": "python3" 260 | }, 261 | "language_info": { 262 | "codemirror_mode": { 263 | "name": "ipython", 264 | "version": 3 265 | }, 266 | "file_extension": ".py", 267 | "mimetype": "text/x-python", 268 | "name": "python", 269 | "nbconvert_exporter": "python", 270 | "pygments_lexer": "ipython3", 271 | "version": "3.7.6" 272 | } 273 | }, 274 | "nbformat": 4, 275 | "nbformat_minor": 4 276 | } 277 | -------------------------------------------------------------------------------- /Chapter10/scripted_convnet.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter10/scripted_convnet.pt -------------------------------------------------------------------------------- /Chapter10/server.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import numpy as np 4 | from flask import Flask, request 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | class ConvNet(nn.Module): 11 | def __init__(self): 12 | super(ConvNet, self).__init__() 13 | self.cn1 = nn.Conv2d(1, 16, 3, 1) 14 | self.cn2 = nn.Conv2d(16, 32, 3, 1) 15 | self.dp1 = nn.Dropout2d(0.10) 16 | self.dp2 = nn.Dropout2d(0.25) 17 | self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32 18 | self.fc2 = nn.Linear(64, 10) 19 | 20 | def forward(self, x): 21 | x = self.cn1(x) 22 | x = F.relu(x) 23 | x = self.cn2(x) 24 | x = F.relu(x) 25 | x = F.max_pool2d(x, 2) 26 | x = self.dp1(x) 27 | x = torch.flatten(x, 1) 28 | x = self.fc1(x) 29 | x = F.relu(x) 30 | x = self.dp2(x) 31 | x = self.fc2(x) 32 | op = F.log_softmax(x, dim=1) 33 | return op 34 | 35 | model = ConvNet() 36 | PATH_TO_MODEL = "./convnet.pth" 37 | model.load_state_dict(torch.load(PATH_TO_MODEL, map_location="cpu")) 38 | model.eval() 39 | 40 | def run_model(input_tensor): 41 | model_input = input_tensor.unsqueeze(0) 42 | with torch.no_grad(): 43 | model_output = model(model_input)[0] 44 | model_prediction = model_output.detach().numpy().argmax() 45 | return model_prediction 46 | 47 | def post_process(output): 48 | return str(output) 49 | 50 | app = Flask(__name__) 51 | 52 | @app.route("/test", methods=["POST"]) 53 | def test(): 54 | data = request.files['data'].read() 55 | md = json.load(request.files['metadata']) 56 | input_array = np.frombuffer(data, dtype=np.float32) 57 | input_image_tensor = torch.from_numpy(input_array).view(md["dims"]) 58 | output = run_model(input_image_tensor) 59 | final_output = post_process(output) 60 | return final_output 61 | 62 | if __name__ == '__main__': 63 | app.run(host='0.0.0.0', port=8890) -------------------------------------------------------------------------------- /Chapter10/traced_convnet.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikibook/mpytc/a24655b38bd676ad1c8202771d5071fe7a182b6b/Chapter10/traced_convnet.pt -------------------------------------------------------------------------------- /Chapter11/convnet_distributed.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import torch.optim as optim 5 | from torch.utils.data import DataLoader 6 | from torchvision import datasets, transforms 7 | 8 | import torch.multiprocessing as mp 9 | import torch.distributed as dist 10 | 11 | import os 12 | import time 13 | import argparse 14 | 15 | 16 | class ConvNet(nn.Module): 17 | def __init__(self): 18 | super(ConvNet, self).__init__() 19 | self.cn1 = nn.Conv2d(1, 16, 3, 1) 20 | self.cn2 = nn.Conv2d(16, 32, 3, 1) 21 | self.dp1 = nn.Dropout2d(0.10) 22 | self.dp2 = nn.Dropout2d(0.25) 23 | self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32 24 | self.fc2 = nn.Linear(64, 10) 25 | 26 | def forward(self, x): 27 | x = self.cn1(x) 28 | x = F.relu(x) 29 | x = self.cn2(x) 30 | x = F.relu(x) 31 | x = F.max_pool2d(x, 2) 32 | x = self.dp1(x) 33 | x = torch.flatten(x, 1) 34 | x = self.fc1(x) 35 | x = F.relu(x) 36 | x = self.dp2(x) 37 | x = self.fc2(x) 38 | op = F.log_softmax(x, dim=1) 39 | return op 40 | 41 | 42 | def train(cpu_num, args): 43 | rank = args.machine_id * args.num_processes + cpu_num 44 | dist.init_process_group( 45 | backend='gloo', 46 | init_method='env://', 47 | world_size=args.world_size, 48 | rank=rank 49 | ) 50 | torch.manual_seed(0) 51 | device = torch.device("cpu") 52 | train_dataset = datasets.MNIST('../data', train=True, download=True, 53 | transform=transforms.Compose([ 54 | transforms.ToTensor(), 55 | transforms.Normalize((0.1302,), (0.3069,))])) 56 | train_sampler = torch.utils.data.distributed.DistributedSampler( 57 | train_dataset, 58 | num_replicas=args.world_size, 59 | rank=rank 60 | ) 61 | train_dataloader = torch.utils.data.DataLoader( 62 | dataset=train_dataset, 63 | batch_size=args.batch_size, 64 | shuffle=False, 65 | num_workers=0, 66 | sampler=train_sampler) 67 | model = ConvNet() 68 | optimizer = optim.Adadelta(model.parameters(), lr=0.5) 69 | model = nn.parallel.DistributedDataParallel(model) 70 | model.train() 71 | for epoch in range(args.epochs): 72 | for b_i, (X, y) in enumerate(train_dataloader): 73 | X, y = X.to(device), y.to(device) 74 | pred_prob = model(X) 75 | loss = F.nll_loss(pred_prob, y) # nll is the negative likelihood loss 76 | optimizer.zero_grad() 77 | loss.backward() 78 | optimizer.step() 79 | if b_i % 10 == 0 and cpu_num==0: 80 | print('epoch: {} [{}/{} ({:.0f}%)]\t training loss: {:.6f}'.format( 81 | epoch, b_i, len(train_dataloader), 82 | 100. * b_i / len(train_dataloader), loss.item())) 83 | 84 | 85 | def main(): 86 | parser = argparse.ArgumentParser() 87 | parser.add_argument('--num-machines', default=1, type=int,) 88 | parser.add_argument('--num-processes', default=1, type=int) 89 | parser.add_argument('--machine-id', default=0, type=int) 90 | parser.add_argument('--epochs', default=1, type=int) 91 | parser.add_argument('--batch-size', default=128, type=int) 92 | args = parser.parse_args() 93 | 94 | args.world_size = args.num_processes * args.num_machines 95 | os.environ['MASTER_ADDR'] = '127.0.0.1' 96 | os.environ['MASTER_PORT'] = '8892' 97 | start = time.time() 98 | mp.spawn(train, nprocs=args.num_processes, args=(args,)) 99 | print(f"Finished training in {time.time()-start} secs") 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /Chapter11/convnet_distributed_cuda.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import torch.optim as optim 5 | from torch.utils.data import DataLoader 6 | from torchvision import datasets, transforms 7 | 8 | import torch.multiprocessing as mp 9 | import torch.distributed as dist 10 | 11 | import os 12 | import time 13 | import argparse 14 | 15 | 16 | class ConvNet(nn.Module): 17 | def __init__(self): 18 | super(ConvNet, self).__init__() 19 | self.cn1 = nn.Conv2d(1, 16, 3, 1) 20 | self.cn2 = nn.Conv2d(16, 32, 3, 1) 21 | self.dp1 = nn.Dropout2d(0.10) 22 | self.dp2 = nn.Dropout2d(0.25) 23 | self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32 24 | self.fc2 = nn.Linear(64, 10) 25 | 26 | def forward(self, x): 27 | x = self.cn1(x) 28 | x = F.relu(x) 29 | x = self.cn2(x) 30 | x = F.relu(x) 31 | x = F.max_pool2d(x, 2) 32 | x = self.dp1(x) 33 | x = torch.flatten(x, 1) 34 | x = self.fc1(x) 35 | x = F.relu(x) 36 | x = self.dp2(x) 37 | x = self.fc2(x) 38 | op = F.log_softmax(x, dim=1) 39 | return op 40 | 41 | 42 | def train(gpu_num, args): 43 | rank = args.machine_id * args.num_processes + cpu_num 44 | dist.init_process_group( 45 | backend='nccl', 46 | init_method='env://', 47 | world_size=args.world_size, 48 | rank=rank 49 | ) 50 | torch.manual_seed(0) 51 | model = ConvNet() 52 | torch.cuda.set_device(gpu_num) 53 | model.cuda(gpu_num) 54 | criterion = nn.NLLLoss().cuda(gpu_num) # nll is the negative likelihood loss 55 | train_dataset = datasets.MNIST('../data', train=True, download=True, 56 | transform=transforms.Compose([ 57 | transforms.ToTensor(), 58 | transforms.Normalize((0.1302,), (0.3069,))])) 59 | train_sampler = torch.utils.data.distributed.DistributedSampler( 60 | train_dataset, 61 | num_replicas=args.world_size, 62 | rank=rank 63 | ) 64 | train_dataloader = torch.utils.data.DataLoader( 65 | dataset=train_dataset, 66 | batch_size=args.batch_size, 67 | shuffle=False, 68 | num_workers=0, 69 | pin_memory=True, 70 | sampler=train_sampler) 71 | optimizer = optim.Adadelta(model.parameters(), lr=0.5) 72 | model = nn.parallel.DistributedDataParallel(model, 73 | device_ids=[gpu_num]) 74 | model.train() 75 | for epoch in range(args.epochs): 76 | for b_i, (X, y) in enumerate(train_dataloader): 77 | X, y = X.cuda(non_blocking=True), y.cuda(non_blocking=True) 78 | pred_prob = model(X) 79 | loss = criterion(pred_prob, y) 80 | optimizer.zero_grad() 81 | loss.backward() 82 | optimizer.step() 83 | if b_i % 10 == 0 and cpu_num==0: 84 | print('epoch: {} [{}/{} ({:.0f}%)]\t training loss: {:.6f}'.format( 85 | epoch, b_i, len(train_dataloader), 86 | 100. * b_i / len(train_dataloader), loss.item())) 87 | 88 | 89 | def main(): 90 | parser = argparse.ArgumentParser() 91 | parser.add_argument('--num-machines', default=1, type=int,) 92 | parser.add_argument('--num-gpu-processes', default=1, type=int) 93 | parser.add_argument('--machine-id', default=0, type=int) 94 | parser.add_argument('--epochs', default=1, type=int) 95 | parser.add_argument('--batch-size', default=128, type=int) 96 | args = parser.parse_args() 97 | 98 | args.world_size = args.num_gpu_processes * args.num_machines 99 | os.environ['MASTER_ADDR'] = '127.0.0.1' 100 | os.environ['MASTER_PORT'] = '8892' 101 | start = time.time() 102 | mp.spawn(train, nprocs=args.num_gpu_processes, args=(args,)) 103 | print(f"Finished training in {time.time()-start} secs") 104 | 105 | if __name__ == '__main__': 106 | main() 107 | -------------------------------------------------------------------------------- /Chapter11/convnet_undistributed.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import torch.optim as optim 5 | from torch.utils.data import DataLoader 6 | from torchvision import datasets, transforms 7 | 8 | import time 9 | import argparse 10 | 11 | 12 | class ConvNet(nn.Module): 13 | def __init__(self): 14 | super(ConvNet, self).__init__() 15 | self.cn1 = nn.Conv2d(1, 16, 3, 1) 16 | self.cn2 = nn.Conv2d(16, 32, 3, 1) 17 | self.dp1 = nn.Dropout2d(0.10) 18 | self.dp2 = nn.Dropout2d(0.25) 19 | self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32 20 | self.fc2 = nn.Linear(64, 10) 21 | 22 | def forward(self, x): 23 | x = self.cn1(x) 24 | x = F.relu(x) 25 | x = self.cn2(x) 26 | x = F.relu(x) 27 | x = F.max_pool2d(x, 2) 28 | x = self.dp1(x) 29 | x = torch.flatten(x, 1) 30 | x = self.fc1(x) 31 | x = F.relu(x) 32 | x = self.dp2(x) 33 | x = self.fc2(x) 34 | op = F.log_softmax(x, dim=1) 35 | return op 36 | 37 | 38 | def train(args): 39 | torch.manual_seed(0) 40 | device = torch.device("cpu") 41 | train_dataloader = torch.utils.data.DataLoader( 42 | datasets.MNIST('../data', train=True, download=True, 43 | transform=transforms.Compose([ 44 | transforms.ToTensor(), 45 | transforms.Normalize((0.1302,), (0.3069,))])), 46 | batch_size=128, shuffle=True) 47 | model = ConvNet() 48 | optimizer = optim.Adadelta(model.parameters(), lr=0.5) 49 | model.train() 50 | for epoch in range(args.epochs): 51 | for b_i, (X, y) in enumerate(train_dataloader): 52 | X, y = X.to(device), y.to(device) 53 | pred_prob = model(X) 54 | loss = F.nll_loss(pred_prob, y) # nll is the negative likelihood loss 55 | optimizer.zero_grad() 56 | loss.backward() 57 | optimizer.step() 58 | if b_i % 10 == 0: 59 | print('epoch: {} [{}/{} ({:.0f}%)]\t training loss: {:.6f}'.format( 60 | epoch, b_i, len(train_dataloader), 61 | 100. * b_i / len(train_dataloader), loss.item())) 62 | 63 | 64 | def main(): 65 | parser = argparse.ArgumentParser() 66 | parser.add_argument('--epochs', default=1, type=int) 67 | args = parser.parse_args() 68 | start = time.time() 69 | train(args) 70 | print(f"Finished training in {time.time()-start} secs") 71 | 72 | if __name__ == '__main__': 73 | main() 74 | -------------------------------------------------------------------------------- /Chapter12/optuna_pytorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## import modules" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import torch\n", 17 | "import torch.nn as nn\n", 18 | "import torch.nn.functional as F\n", 19 | "import torch.optim as optim\n", 20 | "from torch.utils.data import DataLoader\n", 21 | "from torchvision import datasets, transforms\n", 22 | "\n", 23 | "import matplotlib.pyplot as plt\n", 24 | "import optuna\n", 25 | "\n", 26 | "torch.manual_seed(0)\n", 27 | "device = torch.device(\"cpu\")" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "## define model architecture" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "class ConvNet(nn.Module):\n", 44 | " def __init__(self, trial):\n", 45 | " super(ConvNet, self).__init__()\n", 46 | " num_conv_layers = trial.suggest_int(\"num_conv_layers\", 1, 4)\n", 47 | " num_fc_layers = trial.suggest_int(\"num_fc_layers\", 1, 2)\n", 48 | "\n", 49 | " self.layers = []\n", 50 | " input_depth = 1 # grayscale image\n", 51 | " for i in range(num_conv_layers):\n", 52 | " output_depth = trial.suggest_int(f\"conv_depth_{i}\", 16, 64)\n", 53 | " self.layers.append(nn.Conv2d(input_depth, output_depth, 3, 1))\n", 54 | " self.layers.append(nn.ReLU())\n", 55 | " input_depth = output_depth\n", 56 | " self.layers.append(nn.MaxPool2d(2))\n", 57 | " p = trial.suggest_float(f\"conv_dropout_{i}\", 0.1, 0.4)\n", 58 | " self.layers.append(nn.Dropout(p))\n", 59 | " self.layers.append(nn.Flatten())\n", 60 | "\n", 61 | " input_feat = self._get_flatten_shape()\n", 62 | " for i in range(num_fc_layers):\n", 63 | " output_feat = trial.suggest_int(f\"fc_output_feat_{i}\", 16, 64)\n", 64 | " self.layers.append(nn.Linear(input_feat, output_feat))\n", 65 | " self.layers.append(nn.ReLU())\n", 66 | " p = trial.suggest_float(f\"fc_dropout_{i}\", 0.1, 0.4)\n", 67 | " self.layers.append(nn.Dropout(p))\n", 68 | " input_feat = output_feat\n", 69 | " self.layers.append(nn.Linear(input_feat, 10))\n", 70 | " self.layers.append(nn.LogSoftmax(dim=1))\n", 71 | " \n", 72 | " self.model = nn.Sequential(*self.layers)\n", 73 | " \n", 74 | " def _get_flatten_shape(self):\n", 75 | " conv_model = nn.Sequential(*self.layers)\n", 76 | " op_feat = conv_model(torch.rand(1, 1, 28, 28))\n", 77 | " n_size = op_feat.data.view(1, -1).size(1)\n", 78 | " return n_size\n", 79 | " \n", 80 | " def forward(self, x):\n", 81 | " return self.model(x)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "## create data loaders" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 3, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "# The mean and standard deviation values are calculated as the mean of all pixel values of all images in the training dataset\n", 98 | "train_ds = datasets.MNIST('../data', train=True, download=True,\n", 99 | " transform=transforms.Compose([\n", 100 | " transforms.ToTensor(),\n", 101 | " transforms.Normalize((0.1302,), (0.3069,))]))\n", 102 | "test_ds = datasets.MNIST('../data', train=False, \n", 103 | " transform=transforms.Compose([\n", 104 | " transforms.ToTensor(),\n", 105 | " transforms.Normalize((0.1302,), (0.3069,))]))\n", 106 | "\n", 107 | "train_dataloader = torch.utils.data.DataLoader(train_ds, batch_size=32, shuffle=True)\n", 108 | "test_dataloader = torch.utils.data.DataLoader(test_ds, batch_size=500, shuffle=True)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "## define training and inference routines" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 4, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "def train(model, device, train_dataloader, optim, epoch):\n", 125 | " model.train()\n", 126 | " for b_i, (X, y) in enumerate(train_dataloader):\n", 127 | " X, y = X.to(device), y.to(device)\n", 128 | " optim.zero_grad()\n", 129 | " pred_prob = model(X)\n", 130 | " loss = F.nll_loss(pred_prob, y) # nll is the negative likelihood loss\n", 131 | " loss.backward()\n", 132 | " optim.step()\n", 133 | " if b_i % 500 == 0:\n", 134 | " print('epoch: {} [{}/{} ({:.0f}%)]\\t training loss: {:.6f}'.format(\n", 135 | " epoch, b_i * len(X), len(train_dataloader.dataset),\n", 136 | " 100. * b_i / len(train_dataloader), loss.item()))" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 5, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "def test(model, device, test_dataloader):\n", 146 | " model.eval()\n", 147 | " loss = 0\n", 148 | " success = 0\n", 149 | " with torch.no_grad():\n", 150 | " for X, y in test_dataloader:\n", 151 | " X, y = X.to(device), y.to(device)\n", 152 | " pred_prob = model(X)\n", 153 | " loss += F.nll_loss(pred_prob, y, reduction='sum').item() # loss summed across the batch\n", 154 | " pred = pred_prob.argmax(dim=1, keepdim=True) # use argmax to get the most likely prediction\n", 155 | " success += pred.eq(y.view_as(pred)).sum().item()\n", 156 | "\n", 157 | " loss /= len(test_dataloader.dataset)\n", 158 | " \n", 159 | " accuracy = 100. * success / len(test_dataloader.dataset)\n", 160 | "\n", 161 | " print('\\nTest dataset: Overall Loss: {:.4f}, Overall Accuracy: {}/{} ({:.0f}%)\\n'.format(\n", 162 | " loss, success, len(test_dataloader.dataset), accuracy))\n", 163 | " \n", 164 | " return accuracy" 165 | ] 166 | }, 167 | { 168 | "cell_type": "markdown", 169 | "metadata": {}, 170 | "source": [ 171 | "## define optimizer and model training routine" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 6, 177 | "metadata": { 178 | "scrolled": true 179 | }, 180 | "outputs": [], 181 | "source": [ 182 | "def objective(trial):\n", 183 | " \n", 184 | " model = ConvNet(trial)\n", 185 | " opt_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"Adadelta\", \"RMSprop\", \"SGD\"])\n", 186 | " lr = trial.suggest_float(\"lr\", 1e-1, 5e-1, log=True)\n", 187 | " optimizer = getattr(optim, opt_name)(model.parameters(), lr=lr)\n", 188 | " \n", 189 | " for epoch in range(1, 3):\n", 190 | " train(model, device, train_dataloader, optimizer, epoch)\n", 191 | " accuracy = test(model, device, test_dataloader)\n", 192 | " trial.report(accuracy, epoch)\n", 193 | " \n", 194 | " if trial.should_prune():\n", 195 | " raise optuna.exceptions.TrialPruned()\n", 196 | "\n", 197 | " return accuracy" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": {}, 203 | "source": [ 204 | "## run the hyperparameter search" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": {}, 211 | "outputs": [ 212 | { 213 | "name": "stderr", 214 | "output_type": "stream", 215 | "text": [ 216 | "\u001b[32m[I 2020-10-24 10:57:24,406]\u001b[0m A new study created in memory with name: mastering_pytorch\u001b[0m\n" 217 | ] 218 | }, 219 | { 220 | "name": "stdout", 221 | "output_type": "stream", 222 | "text": [ 223 | "epoch: 1 [0/60000 (0%)]\t training loss: 2.306849\n", 224 | "epoch: 1 [16000/60000 (27%)]\t training loss: 2.304207\n", 225 | "epoch: 1 [32000/60000 (53%)]\t training loss: 2.312835\n", 226 | "epoch: 1 [48000/60000 (80%)]\t training loss: 2.299432\n", 227 | "\n", 228 | "Test dataset: Overall Loss: 2.3394, Overall Accuracy: 974/10000 (10%)\n", 229 | "\n", 230 | "epoch: 2 [0/60000 (0%)]\t training loss: 2.288115\n", 231 | "epoch: 2 [16000/60000 (27%)]\t training loss: 2.405160\n", 232 | "epoch: 2 [32000/60000 (53%)]\t training loss: 2.450139\n", 233 | "epoch: 2 [48000/60000 (80%)]\t training loss: 2.314060\n" 234 | ] 235 | }, 236 | { 237 | "name": "stderr", 238 | "output_type": "stream", 239 | "text": [ 240 | "\u001b[32m[I 2020-10-24 11:01:01,311]\u001b[0m Trial 0 finished with value: 10.32 and parameters: {'num_conv_layers': 1, 'num_fc_layers': 1, 'conv_depth_0': 29, 'conv_dropout_0': 0.33975460255509893, 'fc_output_feat_0': 63, 'fc_dropout_0': 0.28457195650592076, 'optimizer': 'Adam', 'lr': 0.4635077277229724}. Best is trial 0 with value: 10.32.\u001b[0m\n" 241 | ] 242 | }, 243 | { 244 | "name": "stdout", 245 | "output_type": "stream", 246 | "text": [ 247 | "\n", 248 | "Test dataset: Overall Loss: 2.3260, Overall Accuracy: 1032/10000 (10%)\n", 249 | "\n", 250 | "epoch: 1 [0/60000 (0%)]\t training loss: 2.330881\n", 251 | "epoch: 1 [16000/60000 (27%)]\t training loss: 2.307726\n", 252 | "epoch: 1 [32000/60000 (53%)]\t training loss: 2.360138\n", 253 | "epoch: 1 [48000/60000 (80%)]\t training loss: 2.434959\n" 254 | ] 255 | } 256 | ], 257 | "source": [ 258 | "study = optuna.create_study(study_name=\"mastering_pytorch\", direction=\"maximize\")\n", 259 | "study.optimize(objective, n_trials=100, timeout=2000)\n", 260 | "\n", 261 | "pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]\n", 262 | "complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]\n", 263 | "\n", 264 | "print(\"results: \")\n", 265 | "print(\"num_trials_conducted: \", len(study.trials))\n", 266 | "print(\"num_trials_pruned: \", len(pruned_trials))\n", 267 | "print(\"num_trials_completed: \", len(complete_trials))\n", 268 | "\n", 269 | "print(\"results from best trial:\")\n", 270 | "trial = study.best_trial\n", 271 | "\n", 272 | "print(\"accuracy: \", trial.value)\n", 273 | "print(\"hyperparameters: \")\n", 274 | "for key, value in trial.params.items():\n", 275 | " print(\"{}: {}\".format(key, value))" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": null, 281 | "metadata": {}, 282 | "outputs": [], 283 | "source": [] 284 | } 285 | ], 286 | "metadata": { 287 | "kernelspec": { 288 | "display_name": "Python 3", 289 | "language": "python", 290 | "name": "python3" 291 | }, 292 | "language_info": { 293 | "codemirror_mode": { 294 | "name": "ipython", 295 | "version": 3 296 | }, 297 | "file_extension": ".py", 298 | "mimetype": "text/x-python", 299 | "name": "python", 300 | "nbconvert_exporter": "python", 301 | "pygments_lexer": "ipython3", 302 | "version": "3.7.6" 303 | } 304 | }, 305 | "nbformat": 4, 306 | "nbformat_minor": 4 307 | } 308 | -------------------------------------------------------------------------------- /Chapter14/fastai.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "from fastai.vision.all import *" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "path = untar_data(URLs.MNIST)\n", 20 | "print(path)" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "scrolled": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "files = get_image_files(path/\"training\")\n", 32 | "print(len(files))\n", 33 | "print(files[0])" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "metadata": { 40 | "scrolled": false 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "def label_func(f): return f.parent.name\n", 45 | "dls = ImageDataLoaders.from_path_func(path, fnames=files, label_func=label_func, num_workers=0)\n", 46 | "dls.show_batch()" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "learn = cnn_learner(dls, arch=resnet18, metrics=accuracy)" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "learn.lr_find()" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "learn.fine_tune(epochs=2, base_lr=0.0209, freeze_epochs=1)" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "learn.show_results()" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "interp = Interpretation.from_learner(learn)\n", 92 | "interp.plot_top_losses(9, figsize=(15,10))" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [] 101 | } 102 | ], 103 | "metadata": { 104 | "kernelspec": { 105 | "display_name": "Python 3", 106 | "language": "python", 107 | "name": "python3" 108 | }, 109 | "language_info": { 110 | "codemirror_mode": { 111 | "name": "ipython", 112 | "version": 3 113 | }, 114 | "file_extension": ".py", 115 | "mimetype": "text/x-python", 116 | "name": "python", 117 | "nbconvert_exporter": "python", 118 | "pygments_lexer": "ipython3", 119 | "version": "3.7.6" 120 | } 121 | }, 122 | "nbformat": 4, 123 | "nbformat_minor": 4 124 | } 125 | -------------------------------------------------------------------------------- /Chapter14/pytorch_lightning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "\n", 11 | "import torch\n", 12 | "import torch.nn as nn\n", 13 | "from torch.nn import functional as F\n", 14 | "from torch.utils.data import DataLoader\n", 15 | "from torchvision.datasets import MNIST\n", 16 | "from torchvision import transforms\n", 17 | "import pytorch_lightning as pl" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "class ConvNet(pl.LightningModule):\n", 27 | "\n", 28 | " def __init__(self):\n", 29 | " super(ConvNet, self).__init__()\n", 30 | " self.cn1 = nn.Conv2d(1, 16, 3, 1)\n", 31 | " self.cn2 = nn.Conv2d(16, 32, 3, 1)\n", 32 | " self.dp1 = nn.Dropout2d(0.10)\n", 33 | " self.dp2 = nn.Dropout2d(0.25)\n", 34 | " self.fc1 = nn.Linear(4608, 64) # 4608 is basically 12 X 12 X 32\n", 35 | " self.fc2 = nn.Linear(64, 10)\n", 36 | " \n", 37 | " def forward(self, x):\n", 38 | " x = self.cn1(x)\n", 39 | " x = F.relu(x)\n", 40 | " x = self.cn2(x)\n", 41 | " x = F.relu(x)\n", 42 | " x = F.max_pool2d(x, 2)\n", 43 | " x = self.dp1(x)\n", 44 | " x = torch.flatten(x, 1)\n", 45 | " x = self.fc1(x)\n", 46 | " x = F.relu(x)\n", 47 | " x = self.dp2(x)\n", 48 | " x = self.fc2(x)\n", 49 | " op = F.log_softmax(x, dim=1)\n", 50 | " return op\n", 51 | "\n", 52 | " def training_step(self, batch, batch_num):\n", 53 | " train_x, train_y = batch\n", 54 | " y_pred = self(train_x)\n", 55 | " training_loss = F.cross_entropy(y_pred, train_y)\n", 56 | " # optional\n", 57 | " self.log('train_loss', training_loss, on_epoch=True, prog_bar=True)\n", 58 | " return training_loss\n", 59 | "\n", 60 | " def validation_step(self, batch, batch_num):\n", 61 | " # optional\n", 62 | " val_x, val_y = batch\n", 63 | " y_pred = self(val_x)\n", 64 | " val_loss = F.cross_entropy(y_pred, val_y)\n", 65 | " # optional\n", 66 | " self.log('val_loss', val_loss, on_step=True, on_epoch=True, prog_bar=True)\n", 67 | " return val_loss\n", 68 | "\n", 69 | " def validation_epoch_end(self, outputs):\n", 70 | " # optional\n", 71 | " avg_loss = torch.stack(outputs).mean()\n", 72 | " return avg_loss\n", 73 | "\n", 74 | " def test_step(self, batch, batch_num):\n", 75 | " # optional\n", 76 | " test_x, test_y = batch\n", 77 | " y_pred = self(test_x)\n", 78 | " test_loss = F.cross_entropy(y_pred, test_y)\n", 79 | " # optional\n", 80 | " self.log('test_loss', test_loss, on_step=True, on_epoch=True, prog_bar=True)\n", 81 | " return test_loss\n", 82 | "\n", 83 | " def test_epoch_end(self, outputs):\n", 84 | " # optional\n", 85 | " avg_loss = torch.stack(outputs).mean()\n", 86 | " return avg_loss\n", 87 | "\n", 88 | " def configure_optimizers(self):\n", 89 | " return torch.optim.Adadelta(self.parameters(), lr=0.5)\n", 90 | "\n", 91 | " def train_dataloader(self):\n", 92 | " return DataLoader(MNIST(os.getcwd(), train=True, download=True, \n", 93 | " transform=transforms.Compose([transforms.ToTensor(),\n", 94 | " transforms.Normalize((0.1302,), (0.3069,))])), \n", 95 | " batch_size=32, num_workers=4)\n", 96 | "\n", 97 | " def val_dataloader(self):\n", 98 | " # optional\n", 99 | " return DataLoader(MNIST(os.getcwd(), train=True, download=True, \n", 100 | " transform=transforms.Compose([transforms.ToTensor(),\n", 101 | " transforms.Normalize((0.1302,), (0.3069,))])), \n", 102 | " batch_size=32, num_workers=4)\n", 103 | "\n", 104 | " def test_dataloader(self):\n", 105 | " # optional\n", 106 | " return DataLoader(MNIST(os.getcwd(), train=False, download=True, \n", 107 | " transform=transforms.Compose([transforms.ToTensor(),\n", 108 | " transforms.Normalize((0.1302,), (0.3069,))])), \n", 109 | " batch_size=32, num_workers=4)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 3, 115 | "metadata": { 116 | "scrolled": false 117 | }, 118 | "outputs": [ 119 | { 120 | "name": "stderr", 121 | "output_type": "stream", 122 | "text": [ 123 | "GPU available: False, used: False\n", 124 | "TPU available: False, using: 0 TPU cores\n", 125 | "\n", 126 | " | Name | Type | Params\n", 127 | "-----------------------------------\n", 128 | "0 | cn1 | Conv2d | 160 \n", 129 | "1 | cn2 | Conv2d | 4 K \n", 130 | "2 | dp1 | Dropout2d | 0 \n", 131 | "3 | dp2 | Dropout2d | 0 \n", 132 | "4 | fc1 | Linear | 294 K \n", 133 | "5 | fc2 | Linear | 650 \n" 134 | ] 135 | }, 136 | { 137 | "data": { 138 | "application/vnd.jupyter.widget-view+json": { 139 | "model_id": "a14398c66e6543b2b423c7c6569bedc9", 140 | "version_major": 2, 141 | "version_minor": 0 142 | }, 143 | "text/plain": [ 144 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…" 145 | ] 146 | }, 147 | "metadata": {}, 148 | "output_type": "display_data" 149 | }, 150 | { 151 | "name": "stdout", 152 | "output_type": "stream", 153 | "text": [ 154 | "\r" 155 | ] 156 | }, 157 | { 158 | "name": "stderr", 159 | "output_type": "stream", 160 | "text": [ 161 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:45: UserWarning: The validation_epoch_end should not return anything as of 9.1.to log, use self.log(...) or self.write(...) directly in the LightningModule\n", 162 | " warnings.warn(*args, **kwargs)\n" 163 | ] 164 | }, 165 | { 166 | "data": { 167 | "application/vnd.jupyter.widget-view+json": { 168 | "model_id": "95a92418ea5547089220007ac8c50cba", 169 | "version_major": 2, 170 | "version_minor": 0 171 | }, 172 | "text/plain": [ 173 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" 174 | ] 175 | }, 176 | "metadata": {}, 177 | "output_type": "display_data" 178 | }, 179 | { 180 | "data": { 181 | "application/vnd.jupyter.widget-view+json": { 182 | "model_id": "6ab571ce34e9489781f59fe355b7a741", 183 | "version_major": 2, 184 | "version_minor": 0 185 | }, 186 | "text/plain": [ 187 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 188 | ] 189 | }, 190 | "metadata": {}, 191 | "output_type": "display_data" 192 | }, 193 | { 194 | "data": { 195 | "application/vnd.jupyter.widget-view+json": { 196 | "model_id": "a70b04f612d54aeebb3dee271cea8f6b", 197 | "version_major": 2, 198 | "version_minor": 0 199 | }, 200 | "text/plain": [ 201 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 202 | ] 203 | }, 204 | "metadata": {}, 205 | "output_type": "display_data" 206 | }, 207 | { 208 | "data": { 209 | "application/vnd.jupyter.widget-view+json": { 210 | "model_id": "c27e36c9fff6496ab46665cc5be8b74b", 211 | "version_major": 2, 212 | "version_minor": 0 213 | }, 214 | "text/plain": [ 215 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 216 | ] 217 | }, 218 | "metadata": {}, 219 | "output_type": "display_data" 220 | }, 221 | { 222 | "data": { 223 | "application/vnd.jupyter.widget-view+json": { 224 | "model_id": "ba582e7219704182aeae9eb4e4fd86f2", 225 | "version_major": 2, 226 | "version_minor": 0 227 | }, 228 | "text/plain": [ 229 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 230 | ] 231 | }, 232 | "metadata": {}, 233 | "output_type": "display_data" 234 | }, 235 | { 236 | "data": { 237 | "application/vnd.jupyter.widget-view+json": { 238 | "model_id": "c8012c41719e4d25b882e58e267d72e8", 239 | "version_major": 2, 240 | "version_minor": 0 241 | }, 242 | "text/plain": [ 243 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 244 | ] 245 | }, 246 | "metadata": {}, 247 | "output_type": "display_data" 248 | }, 249 | { 250 | "data": { 251 | "application/vnd.jupyter.widget-view+json": { 252 | "model_id": "61f88b41279e459aa7864699bdc614cb", 253 | "version_major": 2, 254 | "version_minor": 0 255 | }, 256 | "text/plain": [ 257 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 258 | ] 259 | }, 260 | "metadata": {}, 261 | "output_type": "display_data" 262 | }, 263 | { 264 | "data": { 265 | "application/vnd.jupyter.widget-view+json": { 266 | "model_id": "4a92204452f348ab87ee61d48c25dd34", 267 | "version_major": 2, 268 | "version_minor": 0 269 | }, 270 | "text/plain": [ 271 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 272 | ] 273 | }, 274 | "metadata": {}, 275 | "output_type": "display_data" 276 | }, 277 | { 278 | "data": { 279 | "application/vnd.jupyter.widget-view+json": { 280 | "model_id": "c8928a387e054fd0bd8c29bdaf03c641", 281 | "version_major": 2, 282 | "version_minor": 0 283 | }, 284 | "text/plain": [ 285 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 286 | ] 287 | }, 288 | "metadata": {}, 289 | "output_type": "display_data" 290 | }, 291 | { 292 | "data": { 293 | "application/vnd.jupyter.widget-view+json": { 294 | "model_id": "381ba8821c9e43489969f33005669aa5", 295 | "version_major": 2, 296 | "version_minor": 0 297 | }, 298 | "text/plain": [ 299 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 300 | ] 301 | }, 302 | "metadata": {}, 303 | "output_type": "display_data" 304 | }, 305 | { 306 | "data": { 307 | "application/vnd.jupyter.widget-view+json": { 308 | "model_id": "ec82db627e1c4cd78192ad05d00c27e2", 309 | "version_major": 2, 310 | "version_minor": 0 311 | }, 312 | "text/plain": [ 313 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…" 314 | ] 315 | }, 316 | "metadata": {}, 317 | "output_type": "display_data" 318 | }, 319 | { 320 | "name": "stdout", 321 | "output_type": "stream", 322 | "text": [ 323 | "\n" 324 | ] 325 | }, 326 | { 327 | "data": { 328 | "text/plain": [ 329 | "1" 330 | ] 331 | }, 332 | "execution_count": 3, 333 | "metadata": {}, 334 | "output_type": "execute_result" 335 | } 336 | ], 337 | "source": [ 338 | "model = ConvNet()\n", 339 | "\n", 340 | "trainer = pl.Trainer(progress_bar_refresh_rate=20, max_epochs=10) \n", 341 | "trainer.fit(model) " 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 4, 347 | "metadata": {}, 348 | "outputs": [ 349 | { 350 | "data": { 351 | "application/vnd.jupyter.widget-view+json": { 352 | "model_id": "bde37fc022994a29899f6d498f43bce0", 353 | "version_major": 2, 354 | "version_minor": 0 355 | }, 356 | "text/plain": [ 357 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Testing', layout=Layout(flex='2'), max=…" 358 | ] 359 | }, 360 | "metadata": {}, 361 | "output_type": "display_data" 362 | }, 363 | { 364 | "name": "stdout", 365 | "output_type": "stream", 366 | "text": [ 367 | "--------------------------------------------------------------------------------\n", 368 | "DATALOADER:0 TEST RESULTS\n", 369 | "{'test_loss': tensor(4.7981e-06),\n", 370 | " 'test_loss_epoch': tensor(0.0380),\n", 371 | " 'train_loss': tensor(0.0037),\n", 372 | " 'train_loss_epoch': tensor(0.0260),\n", 373 | " 'train_loss_step': tensor(0.0037),\n", 374 | " 'val_loss': tensor(1.0013e-05),\n", 375 | " 'val_loss_epoch': tensor(0.0097)}\n", 376 | "--------------------------------------------------------------------------------\n", 377 | "\n" 378 | ] 379 | }, 380 | { 381 | "name": "stderr", 382 | "output_type": "stream", 383 | "text": [ 384 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:45: UserWarning: The testing_epoch_end should not return anything as of 9.1.to log, use self.log(...) or self.write(...) directly in the LightningModule\n", 385 | " warnings.warn(*args, **kwargs)\n" 386 | ] 387 | }, 388 | { 389 | "data": { 390 | "text/plain": [ 391 | "[{'train_loss_step': 0.0036528168711811304,\n", 392 | " 'train_loss': 0.0036528168711811304,\n", 393 | " 'val_loss_epoch': 0.009698653593659401,\n", 394 | " 'val_loss': 1.0012968232331332e-05,\n", 395 | " 'train_loss_epoch': 0.02603502944111824,\n", 396 | " 'test_loss_epoch': 0.037956178188323975,\n", 397 | " 'test_loss': 4.798079771717312e-06}]" 398 | ] 399 | }, 400 | "execution_count": 4, 401 | "metadata": {}, 402 | "output_type": "execute_result" 403 | } 404 | ], 405 | "source": [ 406 | "trainer.test()" 407 | ] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": 5, 412 | "metadata": {}, 413 | "outputs": [ 414 | { 415 | "name": "stderr", 416 | "output_type": "stream", 417 | "text": [ 418 | "/Users/ashish.jha/opt/anaconda3/lib/python3.7/site-packages/google/colab/data_table.py:30: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n", 419 | " from IPython.utils import traitlets as _traitlets\n" 420 | ] 421 | }, 422 | { 423 | "data": { 424 | "text/plain": [ 425 | "Reusing TensorBoard on port 6007 (pid 21690), started 22:03:23 ago. (Use '!kill 21690' to kill it.)" 426 | ] 427 | }, 428 | "metadata": {}, 429 | "output_type": "display_data" 430 | }, 431 | { 432 | "data": { 433 | "application/javascript": [ 434 | "\n", 435 | " (async () => {\n", 436 | " const url = await google.colab.kernel.proxyPort(6007, {\"cache\": true});\n", 437 | " const iframe = document.createElement('iframe');\n", 438 | " iframe.src = url;\n", 439 | " iframe.setAttribute('width', '100%');\n", 440 | " iframe.setAttribute('height', '800');\n", 441 | " iframe.setAttribute('frameborder', 0);\n", 442 | " document.body.appendChild(iframe);\n", 443 | " })();\n", 444 | " " 445 | ], 446 | "text/plain": [ 447 | "" 448 | ] 449 | }, 450 | "metadata": {}, 451 | "output_type": "display_data" 452 | } 453 | ], 454 | "source": [ 455 | "# Start tensorboard.\n", 456 | "%reload_ext tensorboard\n", 457 | "%tensorboard --logdir lightning_logs/" 458 | ] 459 | } 460 | ], 461 | "metadata": { 462 | "kernelspec": { 463 | "display_name": "Python 3", 464 | "language": "python", 465 | "name": "python3" 466 | }, 467 | "language_info": { 468 | "codemirror_mode": { 469 | "name": "ipython", 470 | "version": 3 471 | }, 472 | "file_extension": ".py", 473 | "mimetype": "text/x-python", 474 | "name": "python", 475 | "nbconvert_exporter": "python", 476 | "pygments_lexer": "ipython3", 477 | "version": "3.7.6" 478 | } 479 | }, 480 | "nbformat": 4, 481 | "nbformat_minor": 4 482 | } 483 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Packt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: mpytc 2 | dependencies: 3 | - python=3.7 4 | - jupyter 5 | - notebook 6 | - tqdm 7 | - pytorch::pytorch==1.7.1 8 | - pytorch::torchtext 9 | - pytorch::torchvision 10 | - fastai::fastai==2.1.8 # https://forums.fast.ai/t/part1-2020-02-production-found-at-least-two-devices-cuda-0-and-cpu/87253/18 11 | - cython # should be installed before installing cocoapi 12 | - numpy==1.18.5 # to avoid conflict between versions selected by conda and pip 13 | - Pillow<8.3.0 # https://forums.fast.ai/t/runtimeerror-could-not-infer-dtype-of-pilimage/89697 14 | - pip 15 | - pip: 16 | - matplotlib 17 | - nltk 18 | - git+https://github.com/philferriere/cocoapi.git#subdirectory=PythonAPI # pycocotools alternative https://changhsinlee.com/pycocotools/ 19 | - torchviz 20 | - networkx 21 | - sentencepiece 22 | - transformers 23 | - opencv-python 24 | - atari-py 25 | - gym<0.20 # https://www.reddit.com/r/reinforcementlearning/comments/przpu6/gym_cannot_make_atari_envs_work_on_windows_machine/ 26 | - ale-py<0.7.1 # https://stackoverflow.com/questions/69371528/attributeerror-module-ale-py-gym-has-no-attribute-algymenv 27 | - autoPyTorch 28 | - configspace 29 | - git+https://github.com/shukon/HpBandSter.git 30 | - optuna 31 | - captum 32 | - torch-model-archiver==0.2.0 33 | - torchserve 34 | - Flask 35 | - onnx 36 | - onnx-tf 37 | - tensorflow<=1.15.5 38 | -------------------------------------------------------------------------------- /environment_gpu.yml: -------------------------------------------------------------------------------- 1 | name: mpytc_gpu 2 | dependencies: 3 | - python=3.7.16 4 | - jupyter 5 | - notebook 6 | - tqdm 7 | - cudatoolkit 8 | - pytorch::pytorch==1.7.1 9 | - pytorch::torchvision==0.8.2 10 | - pytorch::torchaudio==0.7.2 11 | - pytorch::torchtext==0.8.1 12 | - pytorch::torchserve==0.5.2 13 | - pytorch::torch-model-archiver==0.2.0 14 | - pytorch::captum==0.6.0 15 | - tensorflow==1.15.0 16 | - tensorflow-estimator==1.15.1 17 | - conda-forge::nltk==3.8.1 18 | - conda-forge::pycocotools==2.0.4 19 | - conda-forge::gym==0.19.0 # https://www.reddit.com/r/reinforcementlearning/comments/przpu6/gym_cannot_make_atari_envs_work_on_windows_machine/ 20 | - conda-forge::networkx==2.5.1 21 | - conda-forge::zipp==3.15.0 22 | - fastai::fastai==2.1.8 # https://forums.fast.ai/t/part1-2020-02-production-found-at-least-two-devices-cuda-0-and-cpu/87253/18 23 | - cython # should be installed before installing cocoapi 24 | - numpy==1.21.5 25 | - Pillow==9.3.0 26 | - pip 27 | - pip: 28 | - matplotlib 29 | - torchviz 30 | - sentencepiece 31 | - transformers 32 | - opencv-python 33 | - atari-py 34 | - ale-py==0.7.0 # https://stackoverflow.com/questions/69371528/attributeerror-module-ale-py-gym-has-no-attribute-algymenv 35 | - autoPyTorch 36 | - configspace 37 | - hpbandster 38 | - optuna 39 | - Flask 40 | - onnx==1.5.0 41 | - onnx-tf==1.5.0 42 | - torch==1.7.1 43 | --------------------------------------------------------------------------------