├── README.md ├── Fine_tune_GPT_2_on_Given_Corpus.ipynb ├── corpus.txt └── TF_RNN.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # DickensSpeaks 2 | Text Generation trained on the Short Stories of Charles Dickens using RNN, LSTM, BiLSTM and GPT2 3 | -------------------------------------------------------------------------------- /Fine_tune_GPT_2_on_Given_Corpus.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Fine-tune GPT-2 on Given Corpus", 7 | "provenance": [], 8 | "toc_visible": true, 9 | "authorship_tag": "ABX9TyPwnYGrzxdGGowZFP4l110t", 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | }, 16 | "accelerator": "TPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "id": "view-in-github", 23 | "colab_type": "text" 24 | }, 25 | "source": [ 26 | "\"Open" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "metadata": { 32 | "id": "qZJz3W6II4i5", 33 | "colab_type": "code", 34 | "outputId": "e627ba24-b98b-433d-d0ec-fe8d9cf13ce6", 35 | "colab": { 36 | "base_uri": "https://localhost:8080/", 37 | "height": 264 38 | } 39 | }, 40 | "source": [ 41 | "!pip uninstall tensorflow" 42 | ], 43 | "execution_count": 1, 44 | "outputs": [ 45 | { 46 | "output_type": "stream", 47 | "text": [ 48 | "Uninstalling tensorflow-2.2.0:\n", 49 | " Would remove:\n", 50 | " /usr/local/bin/estimator_ckpt_converter\n", 51 | " /usr/local/bin/saved_model_cli\n", 52 | " /usr/local/bin/tensorboard\n", 53 | " /usr/local/bin/tf_upgrade_v2\n", 54 | " /usr/local/bin/tflite_convert\n", 55 | " /usr/local/bin/toco\n", 56 | " /usr/local/bin/toco_from_protos\n", 57 | " /usr/local/lib/python3.6/dist-packages/tensorflow-2.2.0.dist-info/*\n", 58 | " /usr/local/lib/python3.6/dist-packages/tensorflow/*\n", 59 | "Proceed (y/n)? y\n", 60 | " Successfully uninstalled tensorflow-2.2.0\n" 61 | ], 62 | "name": "stdout" 63 | } 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "metadata": { 69 | "id": "8NEGfFC5JP_9", 70 | "colab_type": "code", 71 | "outputId": "b3aa8e22-72a5-410b-aa4a-ec9ded728a78", 72 | "colab": { 73 | "base_uri": "https://localhost:8080/", 74 | "height": 683 75 | } 76 | }, 77 | "source": [ 78 | "!pip install tensorflow==1.14" 79 | ], 80 | "execution_count": 2, 81 | "outputs": [ 82 | { 83 | "output_type": "stream", 84 | "text": [ 85 | "Collecting tensorflow==1.14\n", 86 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/de/f0/96fb2e0412ae9692dbf400e5b04432885f677ad6241c088ccc5fe7724d69/tensorflow-1.14.0-cp36-cp36m-manylinux1_x86_64.whl (109.2MB)\n", 87 | "\u001b[K |████████████████████████████████| 109.2MB 94kB/s \n", 88 | "\u001b[?25hRequirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.12.1)\n", 89 | "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (3.10.0)\n", 90 | "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.8.1)\n", 91 | "Collecting tensorboard<1.15.0,>=1.14.0\n", 92 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/91/2d/2ed263449a078cd9c8a9ba50ebd50123adf1f8cfbea1492f9084169b89d9/tensorboard-1.14.0-py3-none-any.whl (3.1MB)\n", 93 | "\u001b[K |████████████████████████████████| 3.2MB 43.8MB/s \n", 94 | "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.1.0)\n", 95 | "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.34.2)\n", 96 | "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.3.3)\n", 97 | "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.2.0)\n", 98 | "Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.18.4)\n", 99 | "Collecting tensorflow-estimator<1.15.0rc0,>=1.14.0rc0\n", 100 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/3c/d5/21860a5b11caf0678fbc8319341b0ae21a07156911132e0e71bffed0510d/tensorflow_estimator-1.14.0-py2.py3-none-any.whl (488kB)\n", 101 | "\u001b[K |████████████████████████████████| 491kB 41.3MB/s \n", 102 | "\u001b[?25hRequirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.12.0)\n", 103 | "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.0.8)\n", 104 | "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.9.0)\n", 105 | "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.28.1)\n", 106 | "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.1.0)\n", 107 | "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==1.14) (46.1.3)\n", 108 | "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (1.0.1)\n", 109 | "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (3.2.1)\n", 110 | "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.14) (2.10.0)\n", 111 | "Installing collected packages: tensorboard, tensorflow-estimator, tensorflow\n", 112 | " Found existing installation: tensorboard 2.2.1\n", 113 | " Uninstalling tensorboard-2.2.1:\n", 114 | " Successfully uninstalled tensorboard-2.2.1\n", 115 | " Found existing installation: tensorflow-estimator 2.2.0\n", 116 | " Uninstalling tensorflow-estimator-2.2.0:\n", 117 | " Successfully uninstalled tensorflow-estimator-2.2.0\n", 118 | "Successfully installed tensorboard-1.14.0 tensorflow-1.14.0 tensorflow-estimator-1.14.0\n" 119 | ], 120 | "name": "stdout" 121 | } 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "metadata": { 127 | "id": "vqSvya51JfqS", 128 | "colab_type": "code", 129 | "colab": { 130 | "base_uri": "https://localhost:8080/", 131 | "height": 36 132 | }, 133 | "outputId": "319cc1ea-be8b-4091-a125-873c6e6bedc7" 134 | }, 135 | "source": [ 136 | "!pip install -q gpt-2-simple" 137 | ], 138 | "execution_count": 3, 139 | "outputs": [ 140 | { 141 | "output_type": "stream", 142 | "text": [ 143 | " Building wheel for gpt-2-simple (setup.py) ... \u001b[?25l\u001b[?25hdone\n" 144 | ], 145 | "name": "stdout" 146 | } 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "metadata": { 152 | "id": "5FWWweE7JpZJ", 153 | "colab_type": "code", 154 | "outputId": "5bfcb63c-4d7e-4499-8445-718c844048a7", 155 | "colab": { 156 | "base_uri": "https://localhost:8080/", 157 | "height": 493 158 | } 159 | }, 160 | "source": [ 161 | "import gpt_2_simple as gpt2" 162 | ], 163 | "execution_count": 2, 164 | "outputs": [ 165 | { 166 | "output_type": "stream", 167 | "text": [ 168 | "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 169 | " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", 170 | "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 171 | " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", 172 | "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 173 | " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", 174 | "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 175 | " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", 176 | "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 177 | " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", 178 | "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 179 | " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n", 180 | "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 181 | " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", 182 | "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 183 | " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", 184 | "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 185 | " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", 186 | "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 187 | " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", 188 | "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 189 | " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", 190 | "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", 191 | " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n" 192 | ], 193 | "name": "stderr" 194 | } 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "metadata": { 200 | "id": "bICnd3PXJt_N", 201 | "colab_type": "code", 202 | "colab": {} 203 | }, 204 | "source": [ 205 | "from datetime import datetime\n", 206 | "from google.colab import files" 207 | ], 208 | "execution_count": 0, 209 | "outputs": [] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "metadata": { 214 | "id": "fr-R027uJydy", 215 | "colab_type": "code", 216 | "outputId": "c1b7c89c-a086-4b90-9b58-183a0683fa8f", 217 | "colab": { 218 | "base_uri": "https://localhost:8080/", 219 | "height": 150 220 | } 221 | }, 222 | "source": [ 223 | "gpt2.download_gpt2(model_name=\"124M\")" 224 | ], 225 | "execution_count": 4, 226 | "outputs": [ 227 | { 228 | "output_type": "stream", 229 | "text": [ 230 | "Fetching checkpoint: 1.05Mit [00:00, 435Mit/s] \n", 231 | "Fetching encoder.json: 1.05Mit [00:00, 95.1Mit/s] \n", 232 | "Fetching hparams.json: 1.05Mit [00:00, 312Mit/s] \n", 233 | "Fetching model.ckpt.data-00000-of-00001: 498Mit [00:02, 228Mit/s] \n", 234 | "Fetching model.ckpt.index: 1.05Mit [00:00, 241Mit/s] \n", 235 | "Fetching model.ckpt.meta: 1.05Mit [00:00, 144Mit/s] \n", 236 | "Fetching vocab.bpe: 1.05Mit [00:00, 148Mit/s] \n" 237 | ], 238 | "name": "stderr" 239 | } 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "metadata": { 245 | "id": "mgWvwvaDJ4W1", 246 | "colab_type": "code", 247 | "outputId": "f38ef6fe-1fcb-4ea0-e497-07c9a28f1225", 248 | "colab": { 249 | "base_uri": "https://localhost:8080/", 250 | "height": 132 251 | } 252 | }, 253 | "source": [ 254 | "gpt2.mount_gdrive()" 255 | ], 256 | "execution_count": 7, 257 | "outputs": [ 258 | { 259 | "output_type": "stream", 260 | "text": [ 261 | "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n", 262 | "\n", 263 | "Enter your authorization code:\n", 264 | "··········\n", 265 | "Mounted at /content/drive\n" 266 | ], 267 | "name": "stdout" 268 | } 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "metadata": { 274 | "id": "rOAcd5cFJ-XS", 275 | "colab_type": "code", 276 | "colab": {} 277 | }, 278 | "source": [ 279 | "file_name = \"text.csv\"" 280 | ], 281 | "execution_count": 0, 282 | "outputs": [] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "metadata": { 287 | "id": "NZdjVQWcKklw", 288 | "colab_type": "code", 289 | "colab": {} 290 | }, 291 | "source": [ 292 | "gpt2.copy_file_from_gdrive(file_name)" 293 | ], 294 | "execution_count": 0, 295 | "outputs": [] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "metadata": { 300 | "id": "gxyVqBs_KmMY", 301 | "colab_type": "code", 302 | "outputId": "8188d181-f90e-4845-b73a-d7a21e422617", 303 | "colab": { 304 | "base_uri": "https://localhost:8080/", 305 | "height": 474 306 | } 307 | }, 308 | "source": [ 309 | "sess = gpt2.start_tf_sess()\n", 310 | "\n", 311 | "gpt2.finetune(sess,\n", 312 | " dataset=file_name,\n", 313 | " model_name='124M',\n", 314 | " steps=100,\n", 315 | " restore_from='fresh',\n", 316 | " run_name='run1',\n", 317 | " print_every=10,\n", 318 | " sample_every=100\n", 319 | " )" 320 | ], 321 | "execution_count": 11, 322 | "outputs": [ 323 | { 324 | "output_type": "stream", 325 | "text": [ 326 | "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/gpt_2_simple/src/sample.py:17: add_dispatch_support..wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n", 327 | "Instructions for updating:\n", 328 | "Use tf.where in 2.0, which has the same broadcast rule as np.where\n", 329 | "Loading checkpoint models/124M/model.ckpt\n", 330 | "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", 331 | "Instructions for updating:\n", 332 | "Use standard file APIs to check for files with this prefix.\n", 333 | "INFO:tensorflow:Restoring parameters from models/124M/model.ckpt\n" 334 | ], 335 | "name": "stdout" 336 | }, 337 | { 338 | "output_type": "stream", 339 | "text": [ 340 | "100%|██████████| 1/1 [00:00<00:00, 293.31it/s]" 341 | ], 342 | "name": "stderr" 343 | }, 344 | { 345 | "output_type": "stream", 346 | "text": [ 347 | "Loading dataset...\n", 348 | "dataset has 11190 tokens\n", 349 | "Training...\n" 350 | ], 351 | "name": "stdout" 352 | }, 353 | { 354 | "output_type": "stream", 355 | "text": [ 356 | "\n" 357 | ], 358 | "name": "stderr" 359 | }, 360 | { 361 | "output_type": "stream", 362 | "text": [ 363 | "[10 | 1205.23] loss=1.68 avg=1.68\n", 364 | "[20 | 2381.14] loss=1.28 avg=1.48\n", 365 | "[30 | 3561.63] loss=0.89 avg=1.28\n", 366 | "[40 | 4747.43] loss=0.52 avg=1.09\n", 367 | "[50 | 5934.34] loss=0.23 avg=0.91\n", 368 | "[60 | 7122.34] loss=0.11 avg=0.77\n", 369 | "[70 | 8312.93] loss=0.07 avg=0.67\n", 370 | "[80 | 9499.73] loss=0.05 avg=0.59\n", 371 | "[90 | 10681.79] loss=0.04 avg=0.53\n", 372 | "[100 | 11859.97] loss=0.05 avg=0.48\n", 373 | "Saving checkpoint/run1/model-100\n" 374 | ], 375 | "name": "stdout" 376 | } 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "metadata": { 382 | "id": "VHdTL8NDbAh3", 383 | "colab_type": "code", 384 | "colab": {} 385 | }, 386 | "source": [ 387 | "gpt2.copy_checkpoint_to_gdrive(run_name='run1')" 388 | ], 389 | "execution_count": 0, 390 | "outputs": [] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "metadata": { 395 | "colab_type": "code", 396 | "id": "7gZKspXtI56l", 397 | "outputId": "150073a8-5604-43aa-b006-043929e01003", 398 | "colab": { 399 | "base_uri": "https://localhost:8080/", 400 | "height": 132 401 | } 402 | }, 403 | "source": [ 404 | "sess = gpt2.start_tf_sess()\n", 405 | "gpt2.load_gpt2(sess, run_name='run1')" 406 | ], 407 | "execution_count": 5, 408 | "outputs": [ 409 | { 410 | "output_type": "stream", 411 | "text": [ 412 | "Loading checkpoint checkpoint/run1/model-100\n", 413 | "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", 414 | "Instructions for updating:\n", 415 | "Use standard file APIs to check for files with this prefix.\n", 416 | "INFO:tensorflow:Restoring parameters from checkpoint/run1/model-100\n" 417 | ], 418 | "name": "stdout" 419 | } 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "metadata": { 425 | "id": "9jrgm7NJJLXs", 426 | "colab_type": "code", 427 | "outputId": "d70771bf-a16c-488c-fe93-fb815acde3d6", 428 | "colab": { 429 | "base_uri": "https://localhost:8080/", 430 | "height": 36 431 | } 432 | }, 433 | "source": [ 434 | "gpt2.generate(sess, run_name='run1',\n", 435 | " length=100,\n", 436 | " prefix=\"<|startoftext|>\",\n", 437 | " truncate=\"<|endoftext|>\",\n", 438 | " include_prefix=False)" 439 | ], 440 | "execution_count": 6, 441 | "outputs": [ 442 | { 443 | "output_type": "stream", 444 | "text": [ 445 | "with such powers, and neither could he be depraved\n" 446 | ], 447 | "name": "stdout" 448 | } 449 | ] 450 | }, 451 | { 452 | "cell_type": "code", 453 | "metadata": { 454 | "id": "t_lUYnO4JNJ7", 455 | "colab_type": "code", 456 | "colab": { 457 | "base_uri": "https://localhost:8080/", 458 | "height": 416 459 | }, 460 | "outputId": "92b8c59a-3a68-405e-d04d-b692160edf01" 461 | }, 462 | "source": [ 463 | "gpt2.generate(sess,\n", 464 | " length=100,\n", 465 | " temperature=1.0,\n", 466 | " nsamples=10,\n", 467 | " batch_size=10,\n", 468 | " prefix=\"<|startoftext|>\",\n", 469 | " truncate=\"<|endoftext|>\",\n", 470 | " include_prefix=False\n", 471 | " )" 472 | ], 473 | "execution_count": 7, 474 | "outputs": [ 475 | { 476 | "output_type": "stream", 477 | "text": [ 478 | "your breast, and down to your lance and the lancet. Then, he gently strokes your nipple, and caresses it gently back and forth. As the thought of\n", 479 | "====================\n", 480 | "beyond what we say or do heretofore\n", 481 | "====================\n", 482 | "bathed in a fenced garden, were covered with tallgrass, and\n", 483 | "====================\n", 484 | "higher days than mine. My natural term of\n", 485 | "====================\n", 486 | "place of resort; especially in such a case, as that of a\n", 487 | "====================\n", 488 | "kind. Even such-\n", 489 | "<|startoftext|>as the young lady sat upon, sobbing aloud, clasping her hands passionately.\n", 490 | "====================\n", 491 | "is always false, or imaginary. Men often make up their own mind whether they really are mad,\n", 492 | "====================\n", 493 | "upstairs. I did so, not long after, and he having sat down by my side, by the fire, caused a dissipation in the\n", 494 | "====================\n", 495 | "There\n", 496 | "====================\n", 497 | "before him,' said the woman, pointing to the\n", 498 | "====================\n" 499 | ], 500 | "name": "stdout" 501 | } 502 | ] 503 | }, 504 | { 505 | "cell_type": "code", 506 | "metadata": { 507 | "id": "HEbzNdbNJYwV", 508 | "colab_type": "code", 509 | "colab": {} 510 | }, 511 | "source": [ 512 | "gen_file = 'gpt2_gentext_{:%Y%m%d_%H%M%S}.txt'.format(datetime.utcnow())\n", 513 | "\n", 514 | "gpt2.generate_to_file(sess,\n", 515 | " destination_path=gen_file,\n", 516 | " length=100,\n", 517 | " temperature=1.0,\n", 518 | " nsamples=100,\n", 519 | " batch_size=20,\n", 520 | " prefix=\"<|startoftext|>\",\n", 521 | " truncate=\"<|endoftext|>\",\n", 522 | " include_prefix=False,\n", 523 | " sample_delim=''\n", 524 | " )" 525 | ], 526 | "execution_count": 0, 527 | "outputs": [] 528 | }, 529 | { 530 | "cell_type": "code", 531 | "metadata": { 532 | "id": "Cy53OJzAJaPQ", 533 | "colab_type": "code", 534 | "colab": {} 535 | }, 536 | "source": [ 537 | "# may have to run twice to get file to download\n", 538 | "files.download(gen_file)" 539 | ], 540 | "execution_count": 0, 541 | "outputs": [] 542 | }, 543 | { 544 | "cell_type": "code", 545 | "metadata": { 546 | "id": "159XCDPD9Np4", 547 | "colab_type": "code", 548 | "colab": {} 549 | }, 550 | "source": [ 551 | "" 552 | ], 553 | "execution_count": 0, 554 | "outputs": [] 555 | } 556 | ] 557 | } -------------------------------------------------------------------------------- /corpus.txt: -------------------------------------------------------------------------------- 1 | One winter's evening, towards the close of the year 1800, or within a year 2 | or two of that time, a young medical practitioner, recently established in 3 | business, was seated by a cheerful fire in his little parlour, listening to the 4 | wind which was beating the rain in pattering drops against the window, or 5 | rumbling dismally in the chimney. The night was wet and cold; he had 6 | been walking through mud and water the whole day, and was now 7 | comfortably reposing in his dressing-gown and slippers, more than half 8 | asleep and less than half awake, revolving a thousand matters in his 9 | wandering imagination. First, he thought how hard the wind was blowing, 10 | and how the cold, sharp rain would be at that moment beating in his face, 11 | if he were not comfortably housed at home. Then, his mind reverted to 12 | his annual Christmas visit to his native place and dearest friends; he 13 | thought how glad they would all be to see him, and how happy it would 14 | make Rose if he could only tell her that he had found a patient at last, 15 | and hoped to have more, and to come down again, in a few months' time, 16 | and marry her, and take her home to gladden his lonely fireside, and 17 | stimulate him to fresh exertions. Then, he began to wonder when his first 18 | patient would appear, or whether he was destined, by a special 19 | dispensation of Providence, never to have any patients at all; and then, 20 | he thought about Rose again, and dropped to sleep and dreamed about 21 | her, till the tones of her sweet merry voice sounded in his ears, and her 22 | soft tiny hand rested on his shoulder. 23 | There WAS a hand upon his shoulder, but it was neither soft nor tiny; its 24 | owner being a corpulent round-headed boy, who, in consideration of the 25 | sum of one shilling per week and his food, was let out by the parish to 26 | carry medicine and messages. As there was no demand for the medicine, 27 | however, and no necessity for the messages, he usually occupied his 28 | unemployed hours--averaging fourteen a day--in abstracting peppermint 29 | drops, taking animal nourishment, and going to sleep. 30 | 'A lady, sir--a lady!' whispered the boy, rousing his master with a shake. 31 | 'What lady?' cried our friend, starting up, not quite certain that his dream 32 | was an illusion, and half expecting that it might be Rose herself.--'What 33 | lady? Where?' 34 | 'THERE, sir!' replied the boy, pointing to the glass door leading into the 35 | surgery, with an expression of alarm which the very unusual apparition of 36 | a customer might have tended to excite. 37 | The surgeon looked towards the door, and started himself, for an instant, 38 | on beholding the appearance of his unlooked-for visitor. 39 | It was a singularly tall woman, dressed in deep mourning, and standing 40 | so close to the door that her face almost touched the glass. The upper 41 | part of her figure was carefully muffled in a black shawl, as if for the 42 | purpose of concealment; and her face was shrouded by a thick black veil. 43 | She stood perfectly erect, her figure was drawn up to its full height, and 44 | though the surgeon felt that the eyes beneath the veil were fixed on him, 45 | she stood perfectly motionless, and evinced, by no gesture whatever, the 46 | slightest consciousness of his having turned towards her. 47 | 'Do you wish to consult me?' he inquired, with some hesitation, holding 48 | open the door. It opened inwards, and therefore the action did not alter 49 | the position of the figure, which still remained motionless on the same 50 | spot. 51 | She slightly inclined her head, in token of acquiescence. 52 | 'Pray walk in,' said the surgeon. 53 | The figure moved a step forward; and then, turning its head in the 54 | direction of the boy--to his infinite horror--appeared to hesitate. 55 | 'Leave the room, Tom,' said the young man, addressing the boy, whose 56 | large round eyes had been extended to their utmost width during this 57 | brief interview. 'Draw the curtain, and shut the door.' 58 | The boy drew a green curtain across the glass part of the door, retired 59 | into the surgery, closed the door after him, and immediately applied one 60 | of his large eyes to the keyhole on the other side. 61 | The surgeon drew a chair to the fire, and motioned the visitor to a seat. 62 | The mysterious figure slowly moved towards it. As the blaze shone upon 63 | the black dress, the surgeon observed that the bottom of it was saturated 64 | with mud and rain. 65 | 'You are very wet,' be said. 66 | 'I am,' said the stranger, in a low deep voice. 67 | 'And you are ill?' added the surgeon, compassionately, for the tone was 68 | that of a person in pain. 69 | 'I am,' was the reply--'very ill; not bodily, but mentally. It is not for 70 | myself, or on my own behalf,' continued the stranger, 'that I come to you. 71 | If I laboured under bodily disease, I should not be out, alone, at such an 72 | hour, or on such a night as this; and if I were afflicted with it, twenty-four 73 | hours hence, God knows how gladly I would lie down and pray to die. It is 74 | for another that I beseech your aid, sir. I may be mad to ask it for him--I 75 | think I am; but, night after night, through the long dreary hours of 76 | watching and weeping, the thought has been ever present to my mind; 77 | and though even _I_ see the hopelessness of human assistance availing 78 | him, the bare thought of laying him in his grave without it makes my 79 | blood run cold!' And a shudder, such as the surgeon well knew art could 80 | not produce, trembled through the speaker's frame. 81 | There was a desperate earnestness in this woman's manner, that went to 82 | the young man's heart. He was young in his profession, and had not yet 83 | witnessed enough of the miseries which are daily presented before the 84 | eyes of its members, to have grown comparatively callous to human 85 | suffering. 86 | 'If,' he said, rising hastily, 'the person of whom you speak, be in so 87 | hopeless a condition as you describe, not a moment is to be lost. I will go 88 | with you instantly. Why did you not obtain medical advice before?' 89 | 'Because it would have been useless before--because it is useless even 90 | now,' replied the woman, clasping her hands passionately. 91 | The surgeon gazed, for a moment, on the black veil, as if to ascertain the 92 | expression of the features beneath it: its thickness, however, rendered 93 | such a result impossible. 94 | 'You ARE ill,' he said, gently, 'although you do not know it. The fever 95 | which has enabled you to bear, without feeling it, the fatigue you have 96 | evidently undergone, is burning within you now. Put that to your lips,' he 97 | continued, pouring out a glass of water- -'compose yourself for a few 98 | moments, and then tell me, as calmly as you can, what the disease of the 99 | patient is, and how long he has been ill. When I know what it is necessary 100 | I should know, to render my visit serviceable to him, I am ready to 101 | accompany you.' 102 | The stranger lifted the glass of water to her mouth, without raising the 103 | veil; put it down again untasted; and burst into tears. 104 | 'I know,' she said, sobbing aloud, 'that what I say to you now, seems like 105 | the ravings of fever. I have been told so before, less kindly than by you. I 106 | am not a young woman; and they do say, that as life steals on towards its 107 | final close, the last short remnant, worthless as it may seem to all beside, 108 | is dearer to its possessor than all the years that have gone before, 109 | connected though they be with the recollection of old friends long since 110 | dead, and young ones--children perhaps--who have fallen off from, and 111 | forgotten one as completely as if they had died too. My natural term of 112 | life cannot be many years longer, and should be dear on that account; 113 | but I would lay it down without a sigh--with cheerfulness--with joy--if 114 | what I tell you now, were only false, or imaginary. To- morrow morning 115 | he of whom I speak will be, I KNOW, though I would fain think otherwise, 116 | beyond the reach of human aid; and yet, to- night, though he is in deadly 117 | peril, you must not see, and could not serve, him.' 118 | 'I am unwilling to increase your distress,' said the surgeon, after a short 119 | pause, 'by making any comment on what you have just said, or appearing 120 | desirous to investigate a subject you are so anxious to conceal; but there 121 | is an inconsistency in your statement which I cannot reconcile with 122 | probability. This person is dying to-night, and I cannot see him when my 123 | assistance might possibly avail; you apprehend it will be useless tomorrow, and yet you would have me see him then! If he be, indeed, as 124 | dear to you, as your words and manner would imply, why not try to save 125 | his life before delay and the progress of his disease render it 126 | impracticable?' 127 | 'God help me!' exclaimed the woman, weeping bitterly, 'how can I hope 128 | strangers will believe what appears incredible, even to myself? You will 129 | NOT see him then, sir?' she added, rising suddenly. 130 | 'I did not say that I declined to see him,' replied the surgeon; 'but I warn 131 | you, that if you persist in this extraordinary procrastination, and the 132 | individual dies, a fearful responsibility rests with you.' 133 | 'The responsibility will rest heavily somewhere,' replied the stranger 134 | bitterly. 'Whatever responsibility rests with me, I am content to bear, and 135 | ready to answer.' 136 | 'As I incur none,' continued the surgeon, 'by acceding to your request, I 137 | will see him in the morning, if you leave me the address. At what hour 138 | can he be seen?' 139 | 'NINE,' replied the stranger. 140 | 'You must excuse my pressing these inquiries,' said the surgeon. 'But is 141 | he in your charge now?' 142 | 'He is not,' was the rejoinder. 143 | 'Then, if I gave you instructions for his treatment through the night, you 144 | could not assist him?' 145 | The woman wept bitterly, as she replied, 'I could not.' 146 | Finding that there was but little prospect of obtaining more information by 147 | prolonging the interview; and anxious to spare the woman's feelings, 148 | which, subdued at first by a violent effort, were now irrepressible and 149 | most painful to witness; the surgeon repeated his promise of calling in the 150 | morning at the appointed hour. His visitor, after giving him a direction to 151 | an obscure part of Walworth, left the house in the same mysterious 152 | manner in which she had entered it. 153 | It will be readily believed that so extraordinary a visit produced a 154 | considerable impression on the mind of the young surgeon; and that he 155 | speculated a great deal and to very little purpose on the possible 156 | circumstances of the case. In common with the generality of people, he 157 | had often heard and read of singular instances, in which a presentiment 158 | of death, at a particular day, or even minute, had been entertained and 159 | realised. At one moment he was inclined to think that the present might 160 | be such a case; but, then, it occurred to him that all the anecdotes of the 161 | kind he had ever heard, were of persons who had been troubled with a 162 | foreboding of their own death. This woman, however, spoke of another 163 | person--a man; and it was impossible to suppose that a mere dream or 164 | delusion of fancy would induce her to speak of his approaching dissolution 165 | with such terrible certainty as she had spoken. It could not be that the 166 | man was to be murdered in the morning, and that the woman, originally a 167 | consenting party, and bound to secrecy by an oath, had relented, and, 168 | though unable to prevent the commission of some outrage on the victim, 169 | had determined to prevent his death if possible, by the timely 170 | interposition of medical aid? The idea of such things happening within two 171 | miles of the metropolis appeared too wild and preposterous to be 172 | entertained beyond the instant. Then, his original impression that the 173 | woman's intellects were disordered, recurred; and, as it was the only 174 | mode of solving the difficulty with any degree of satisfaction, he 175 | obstinately made up his mind to believe that she was mad. Certain 176 | misgivings upon this point, however, stole upon his thoughts at the time, 177 | and presented themselves again and again through the long dull course of 178 | a sleepless night; during which, in spite of all his efforts to the contrary, 179 | he was unable to banish the black veil from his disturbed imagination. 180 | The back part of Walworth, at its greatest distance from town, is a 181 | straggling miserable place enough, even in these days; but, five- andthirty years ago, the greater portion of it was little better than a dreary 182 | waste, inhabited by a few scattered people of questionable character, 183 | whose poverty prevented their living in any better neighbourhood, or 184 | whose pursuits and mode of life rendered its solitude desirable. Very 185 | many of the houses which have since sprung up on all sides, were not 186 | built until some years afterwards; and the great majority even of those 187 | which were sprinkled about, at irregular intervals, were of the rudest and 188 | most miserable description. 189 | The appearance of the place through which he walked in the morning, 190 | was not calculated to raise the spirits of the young surgeon, or to dispel 191 | any feeling of anxiety or depression which the singular kind of visit he 192 | was about to make, had awakened. Striking off from the high road, his 193 | way lay across a marshy common, through irregular lanes, with here and 194 | there a ruinous and dismantled cottage fast falling to pieces with decay 195 | and neglect. A stunted tree, or pool of stagnant water, roused into a 196 | sluggish action by the heavy rain of the preceding night, skirted the path 197 | occasionally; and, now and then, a miserable patch of garden-ground, 198 | with a few old boards knocked together for a summer-house, and old 199 | palings imperfectly mended with stakes pilfered from the neighbouring 200 | hedges, bore testimony, at once to the poverty of the inhabitants, and the 201 | little scruple they entertained in appropriating the property of other 202 | people to their own use. Occasionally, a filthy-looking woman would make 203 | her appearance from the door of a dirty house, to empty the contents of 204 | some cooking utensil into the gutter in front, or to scream after a little 205 | slip-shod girl, who had contrived to stagger a few yards from the door 206 | under the weight of a sallow infant almost as big as herself; but, scarcely 207 | anything was stirring around: and so much of the prospect as could be 208 | faintly traced through the cold damp mist which hung heavily over it, 209 | presented a lonely and dreary appearance perfectly in keeping with the 210 | objects we have described. 211 | After plodding wearily through the mud and mire; making many inquiries 212 | for the place to which he had been directed; and receiving as many 213 | contradictory and unsatisfactory replies in return; the young man at 214 | length arrived before the house which had been pointed out to him as the 215 | object of his destination. It was a small low building, one story above the 216 | ground, with even a more desolate and unpromising exterior than any he 217 | had yet passed. An old yellow curtain was closely drawn across the 218 | window up-stairs, and the parlour shutters were closed, but not fastened. 219 | The house was detached from any other, and, as it stood at an angle of a 220 | narrow lane, there was no other habitation in sight. 221 | When we say that the surgeon hesitated, and walked a few paces beyond 222 | the house, before he could prevail upon himself to lift the knocker, we say 223 | nothing that need raise a smile upon the face of the boldest reader. The 224 | police of London were a very different body in that day; the isolated 225 | position of the suburbs, when the rage for building and the progress of 226 | improvement had not yet begun to connect them with the main body of 227 | the city and its environs, rendered many of them (and this in particular) a 228 | place of resort for the worst and most depraved characters. Even the 229 | streets in the gayest parts of London were imperfectly lighted, at that 230 | time; and such places as these, were left entirely to the mercy of the 231 | moon and stars. The chances of detecting desperate characters, or of 232 | tracing them to their haunts, were thus rendered very few, and their 233 | offences naturally increased in boldness, as the consciousness of 234 | comparative security became the more impressed upon them by daily 235 | experience. Added to these considerations, it must be remembered that 236 | the young man had spent some time in the public hospitals of the 237 | metropolis; and, although neither Burke nor Bishop had then gained a 238 | horrible notoriety, his own observation might have suggested to him how 239 | easily the atrocities to which the former has since given his name, might 240 | be committed. Be this as it may, whatever reflection made him hesitate, 241 | he DID hesitate: but, being a young man of strong mind and great 242 | personal courage, it was only for an instant;--he stepped briskly back and 243 | knocked gently at the door. 244 | A low whispering was audible, immediately afterwards, as if some person 245 | at the end of the passage were conversing stealthily with another on the 246 | landing above. It was succeeded by the noise of a pair of heavy boots 247 | upon the bare floor. The door-chain was softly unfastened; the door 248 | opened; and a tall, ill-favoured man, with black hair, and a face, as the 249 | surgeon often declared afterwards, as pale and haggard, as the 250 | countenance of any dead man he ever saw, presented himself. 251 | 'Walk in, sir,' he said in a low tone. 252 | The surgeon did so, and the man having secured the door again, by the 253 | chain, led the way to a small back parlour at the extremity of the 254 | passage. 255 | 'Am I in time?' 256 | 'Too soon!' replied the man. The surgeon turned hastily round, with a 257 | gesture of astonishment not unmixed with alarm, which he found it 258 | impossible to repress. 259 | 'If you'll step in here, sir,' said the man, who had evidently noticed the 260 | action--'if you'll step in here, sir, you won't be detained five minutes, I 261 | assure you.' 262 | The surgeon at once walked into the room. The man closed the door, and 263 | left him alone. 264 | It was a little cold room, with no other furniture than two deal chairs, and 265 | a table of the same material. A handful of fire, unguarded by any fender, 266 | was burning in the grate, which brought out the damp if it served no 267 | more comfortable purpose, for the unwholesome moisture was stealing 268 | down the walls, in long slug-like tracks. The window, which was broken 269 | and patched in many places, looked into a small enclosed piece of ground, 270 | almost covered with water. Not a sound was to be heard, either within the 271 | house, or without. The young surgeon sat down by the fireplace, to await 272 | the result of his first professional visit. 273 | He had not remained in this position many minutes, when the noise of 274 | some approaching vehicle struck his ear. It stopped; the street-door was 275 | opened; a low talking succeeded, accompanied with a shuffling noise of 276 | footsteps, along the passage and on the stairs, as if two or three men 277 | were engaged in carrying some heavy body to the room above. The 278 | creaking of the stairs, a few seconds afterwards, announced that the newcomers having completed their task, whatever it was, were leaving the 279 | house. The door was again closed, and the former silence was restored. 280 | Another five minutes had elapsed, and the surgeon had resolved to 281 | explore the house, in search of some one to whom he might make his 282 | errand known, when the room-door opened, and his last night's visitor, 283 | dressed in exactly the same manner, with the veil lowered as before, 284 | motioned him to advance. The singular height of her form, coupled with 285 | the circumstance of her not speaking, caused the idea to pass across his 286 | brain for an instant, that it might be a man disguised in woman's attire. 287 | The hysteric sobs which issued from beneath the veil, and the convulsive 288 | attitude of grief of the whole figure, however, at once exposed the 289 | absurdity of the suspicion; and he hastily followed. 290 | The woman led the way up-stairs to the front room, and paused at the 291 | door, to let him enter first. It was scantily furnished with an old deal box, 292 | a few chairs, and a tent bedstead, without hangings or cross-rails, which 293 | was covered with a patchwork counterpane. The dim light admitted 294 | through the curtain which he had noticed from the outside, rendered the 295 | objects in the room so indistinct, and communicated to all of them so 296 | uniform a hue, that he did not, at first, perceive the object on which his 297 | eye at once rested when the woman rushed frantically past him, and flung 298 | herself on her knees by the bedside. 299 | Stretched upon the bed, closely enveloped in a linen wrapper, and 300 | covered with blankets, lay a human form, stiff and motionless. The head 301 | and face, which were those of a man, were uncovered, save by a bandage 302 | which passed over the head and under the chin. The eyes were closed. 303 | The left arm lay heavily across the bed, and the woman held the passive 304 | hand. 305 | The surgeon gently pushed the woman aside, and took the hand in his. 306 | 'My God!' he exclaimed, letting it fall involuntarily--'the man is dead!' 307 | The woman started to her feet and beat her hands together. 308 | 'Oh! don't say so, sir,' she exclaimed, with a burst of passion, amounting 309 | almost to frenzy. 'Oh! don't say so, sir! I can't bear it! Men have been 310 | brought to life, before, when unskilful people have given them up for lost; 311 | and men have died, who might have been restored, if proper means had 312 | been resorted to. Don't let him lie here, sir, without one effort to save 313 | him! This very moment life may be passing away. Do try, sir,--do, for 314 | Heaven's sake!'--And while speaking, she hurriedly chafed, first the 315 | forehead, and then the breast, of the senseless form before her; and 316 | then, wildly beat the cold hands, which, when she ceased to hold them, 317 | fell listlessly and heavily back on the coverlet. 318 | 'It is of no use, my good woman,' said the surgeon, soothingly, as he 319 | withdrew his hand from the man's breast. 'Stay--undraw that curtain!' 320 | 'Why?' said the woman, starting up. 321 | 'Undraw that curtain!' repeated the surgeon in an agitated tone. 322 | 'I darkened the room on purpose,' said the woman, throwing herself 323 | before him as he rose to undraw it.--'Oh! sir, have pity on me! If it can be 324 | of no use, and he is really dead, do not expose that form to other eyes 325 | than mine!' 326 | 'This man died no natural or easy death,' said the surgeon. 'I MUST see 327 | the body!' With a motion so sudden, that the woman hardly knew that he 328 | had slipped from beside her, he tore open the curtain, admitted the full 329 | light of day, and returned to the bedside. 330 | 'There has been violence here,' he said, pointing towards the body, and 331 | gazing intently on the face, from which the black veil was now, for the 332 | first time, removed. In the excitement of a minute before, the female had 333 | thrown off the bonnet and veil, and now stood with her eyes fixed upon 334 | him. Her features were those of a woman about fifty, who had once been 335 | handsome. Sorrow and weeping had left traces upon them which not time 336 | itself would ever have produced without their aid; her face was deadly 337 | pale; and there was a nervous contortion of the lip, and an unnatural fire 338 | in her eye, which showed too plainly that her bodily and mental powers 339 | had nearly sunk, beneath an accumulation of misery. 340 | 'There has been violence here,' said the surgeon, preserving his searching 341 | glance. 342 | 'There has!' replied the woman. 343 | 'This man has been murdered.' 344 | 'That I call God to witness he has,' said the woman, passionately; 345 | 'pitilessly, inhumanly murdered!' 346 | 'By whom?' said the surgeon, seizing the woman by the arm. 347 | 'Look at the butchers' marks, and then ask me!' she replied. 348 | The surgeon turned his face towards the bed, and bent over the body 349 | which now lay full in the light of the window. The throat was swollen, and 350 | a livid mark encircled it. The truth flashed suddenly upon him. 351 | 'This is one of the men who were hanged this morning!' he exclaimed, 352 | turning away with a shudder. 353 | 'It is,' replied the woman, with a cold, unmeaning stare. 354 | 'Who was he?' inquired the surgeon. 355 | 'MY SON,' rejoined the woman; and fell senseless at his feet. 356 | It was true. A companion, equally guilty with himself, had been acquitted 357 | for want of evidence; and this man had been left for death, and executed. 358 | To recount the circumstances of the case, at this distant period, must be 359 | unnecessary, and might give pain to some persons still alive. The history 360 | was an every-day one. The mother was a widow without friends or 361 | money, and had denied herself necessaries to bestow them on her orphan 362 | boy. That boy, unmindful of her prayers, and forgetful of the sufferings 363 | she had endured for him--incessant anxiety of mind, and voluntary 364 | starvation of body-- had plunged into a career of dissipation and crime. 365 | And this was the result; his own death by the hangman's hands, and his 366 | mother's shame, and incurable insanity. 367 | For many years after this occurrence, and when profitable and arduous 368 | avocations would have led many men to forget that such a miserable 369 | being existed, the young surgeon was a daily visitor at the side of the 370 | harmless mad woman; not only soothing her by his presence and 371 | kindness, but alleviating the rigour of her condition by pecuniary 372 | donations for her comfort and support, bestowed with no sparing hand. In 373 | the transient gleam of recollection and consciousness which preceded her 374 | death, a prayer for his welfare and protection, as fervent as mortal ever 375 | breathed, rose from the lips of this poor friendless creature. That prayer 376 | flew to Heaven, and was heard. The blessings he was instrumental in 377 | conferring, have been repaid to him a thousand-fold; but, amid all the 378 | honours of rank and station which have since been heaped upon him, and 379 | which he has so well earned, he can have no reminiscence more 380 | gratifying to his heart than that connected with The Black Veil. -------------------------------------------------------------------------------- /TF_RNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 7, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "#!pip install tensorflow" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 134, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import tensorflow as tf\n", 19 | "\n", 20 | "import numpy as np\n", 21 | "import os\n", 22 | "import time" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 135, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "# Read, then decode for py2 compat.\n", 32 | "text = open('corpus.txt','rb').read().decode(encoding='utf-8')" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 136, 38 | "metadata": {}, 39 | "outputs": [ 40 | { 41 | "name": "stdout", 42 | "output_type": "stream", 43 | "text": [ 44 | "Length of text: 25439 characters\n" 45 | ] 46 | } 47 | ], 48 | "source": [ 49 | "# length of text is the number of characters in it\n", 50 | "print ('Length of text: {} characters'.format(len(text)))" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 137, 56 | "metadata": {}, 57 | "outputs": [ 58 | { 59 | "name": "stdout", 60 | "output_type": "stream", 61 | "text": [ 62 | "One winter's evening, towards the close of the year 1800, or within a year\n", 63 | "or two of that time, a young medical practitioner, recently established in\n", 64 | "business, was seated by a cheerful fire in his little parlour, listening to the\n", 65 | "wind which was be\n" 66 | ] 67 | } 68 | ], 69 | "source": [ 70 | "# Take a look at the first 250 characters in text\n", 71 | "print(text[:250])" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 138, 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "name": "stdout", 81 | "output_type": "stream", 82 | "text": [ 83 | "65 unique characters\n" 84 | ] 85 | } 86 | ], 87 | "source": [ 88 | "# The unique characters in the file\n", 89 | "vocab = sorted(set(text))\n", 90 | "print ('{} unique characters'.format(len(vocab)))" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 139, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "# Creating a mapping from unique characters to indices\n", 100 | "char2idx = {u:i for i, u in enumerate(vocab)}\n", 101 | "idx2char = np.array(vocab)\n", 102 | "\n", 103 | "text_as_int = np.array([char2idx[c] for c in text])" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 140, 109 | "metadata": {}, 110 | "outputs": [ 111 | { 112 | "name": "stdout", 113 | "output_type": "stream", 114 | "text": [ 115 | "{\n", 116 | " '\\n': 0,\n", 117 | " '\\r': 1,\n", 118 | " ' ' : 2,\n", 119 | " '!' : 3,\n", 120 | " \"'\" : 4,\n", 121 | " '(' : 5,\n", 122 | " ')' : 6,\n", 123 | " ',' : 7,\n", 124 | " '-' : 8,\n", 125 | " '.' : 9,\n", 126 | " '0' : 10,\n", 127 | " '1' : 11,\n", 128 | " '8' : 12,\n", 129 | " ':' : 13,\n", 130 | " ';' : 14,\n", 131 | " '?' : 15,\n", 132 | " 'A' : 16,\n", 133 | " 'B' : 17,\n", 134 | " 'C' : 18,\n", 135 | " 'D' : 19,\n", 136 | " ...\n", 137 | "}\n" 138 | ] 139 | } 140 | ], 141 | "source": [ 142 | "print('{')\n", 143 | "for char,_ in zip(char2idx, range(20)):\n", 144 | " print(' {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n", 145 | "print(' ...\\n}')" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 141, 151 | "metadata": {}, 152 | "outputs": [ 153 | { 154 | "name": "stdout", 155 | "output_type": "stream", 156 | "text": [ 157 | "\"One winter's \" ---- characters mapped to int ---- > [29 52 43 2 61 47 52 58 43 56 4 57 2]\n" 158 | ] 159 | } 160 | ], 161 | "source": [ 162 | "# Show how the first 13 characters from the text are mapped to integers\n", 163 | "print ('{} ---- characters mapped to int ---- > {}'.format(repr(text[:13]), text_as_int[:13]))" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 142, 169 | "metadata": {}, 170 | "outputs": [ 171 | { 172 | "name": "stdout", 173 | "output_type": "stream", 174 | "text": [ 175 | "O\n", 176 | "n\n", 177 | "e\n", 178 | " \n", 179 | "w\n" 180 | ] 181 | } 182 | ], 183 | "source": [ 184 | "# The maximum length sentence we want for a single input in characters\n", 185 | "seq_length = 100\n", 186 | "examples_per_epoch = len(text)//(seq_length+1)\n", 187 | "\n", 188 | "# Create training examples / targets\n", 189 | "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n", 190 | "\n", 191 | "for i in char_dataset.take(5):\n", 192 | " print(idx2char[i.numpy()])" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 143, 198 | "metadata": {}, 199 | "outputs": [ 200 | { 201 | "name": "stdout", 202 | "output_type": "stream", 203 | "text": [ 204 | "\"One winter's evening, towards the close of the year 1800, or within a year\\r\\nor two of that time, a yo\"\n", 205 | "'ung medical practitioner, recently established in\\r\\nbusiness, was seated by a cheerful fire in his lit'\n", 206 | "'tle parlour, listening to the\\r\\nwind which was beating the rain in pattering drops against the window,'\n", 207 | "' or\\r\\nrumbling dismally in the chimney. The night was wet and cold; he had\\r\\nbeen walking through mud a'\n", 208 | "'nd water the whole day, and was now\\r\\ncomfortably reposing in his dressing-gown and slippers, more tha'\n" 209 | ] 210 | } 211 | ], 212 | "source": [ 213 | "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n", 214 | "\n", 215 | "for item in sequences.take(5):\n", 216 | " print(repr(''.join(idx2char[item.numpy()])))" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 144, 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [ 225 | "def split_input_target(chunk):\n", 226 | " input_text = chunk[:-1]\n", 227 | " target_text = chunk[1:]\n", 228 | " return input_text, target_text\n", 229 | "\n", 230 | "dataset = sequences.map(split_input_target)" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": 145, 236 | "metadata": {}, 237 | "outputs": [ 238 | { 239 | "name": "stdout", 240 | "output_type": "stream", 241 | "text": [ 242 | "Input data: \"One winter's evening, towards the close of the year 1800, or within a year\\r\\nor two of that time, a y\"\n", 243 | "Target data: \"ne winter's evening, towards the close of the year 1800, or within a year\\r\\nor two of that time, a yo\"\n" 244 | ] 245 | } 246 | ], 247 | "source": [ 248 | "for input_example, target_example in dataset.take(1):\n", 249 | " print ('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n", 250 | " print ('Target data:', repr(''.join(idx2char[target_example.numpy()])))" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 146, 256 | "metadata": {}, 257 | "outputs": [ 258 | { 259 | "name": "stdout", 260 | "output_type": "stream", 261 | "text": [ 262 | "Step 0\n", 263 | " input: 29 ('O')\n", 264 | " expected output: 52 ('n')\n", 265 | "Step 1\n", 266 | " input: 52 ('n')\n", 267 | " expected output: 43 ('e')\n", 268 | "Step 2\n", 269 | " input: 43 ('e')\n", 270 | " expected output: 2 (' ')\n", 271 | "Step 3\n", 272 | " input: 2 (' ')\n", 273 | " expected output: 61 ('w')\n", 274 | "Step 4\n", 275 | " input: 61 ('w')\n", 276 | " expected output: 47 ('i')\n" 277 | ] 278 | } 279 | ], 280 | "source": [ 281 | "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n", 282 | " print(\"Step {:4d}\".format(i))\n", 283 | " print(\" input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n", 284 | " print(\" expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))" 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": 147, 290 | "metadata": {}, 291 | "outputs": [ 292 | { 293 | "data": { 294 | "text/plain": [ 295 | "" 296 | ] 297 | }, 298 | "execution_count": 147, 299 | "metadata": {}, 300 | "output_type": "execute_result" 301 | } 302 | ], 303 | "source": [ 304 | "# Batch size\n", 305 | "BATCH_SIZE = 64\n", 306 | "\n", 307 | "# Buffer size to shuffle the dataset\n", 308 | "# (TF data is designed to work with possibly infinite sequences,\n", 309 | "# so it doesn't attempt to shuffle the entire sequence in memory. Instead,\n", 310 | "# it maintains a buffer in which it shuffles elements).\n", 311 | "BUFFER_SIZE = 10000\n", 312 | "\n", 313 | "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n", 314 | "\n", 315 | "dataset" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": 148, 321 | "metadata": {}, 322 | "outputs": [], 323 | "source": [ 324 | "# Length of the vocabulary in chars\n", 325 | "vocab_size = len(vocab)\n", 326 | "\n", 327 | "# The embedding dimension\n", 328 | "embedding_dim = 256\n", 329 | "\n", 330 | "# Number of RNN units\n", 331 | "rnn_units = 1024" 332 | ] 333 | }, 334 | { 335 | "cell_type": "code", 336 | "execution_count": 149, 337 | "metadata": {}, 338 | "outputs": [], 339 | "source": [ 340 | "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n", 341 | " model = tf.keras.Sequential([\n", 342 | " tf.keras.layers.Embedding(vocab_size, embedding_dim,\n", 343 | " batch_input_shape=[batch_size, None]),\n", 344 | " tf.keras.layers.GRU(rnn_units,\n", 345 | " return_sequences=True,\n", 346 | " stateful=True,\n", 347 | " recurrent_initializer='glorot_uniform'),\n", 348 | " tf.keras.layers.Dense(vocab_size)\n", 349 | " ])\n", 350 | " return model" 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 150, 356 | "metadata": {}, 357 | "outputs": [], 358 | "source": [ 359 | "model = build_model(\n", 360 | " vocab_size = len(vocab),\n", 361 | " embedding_dim=embedding_dim,\n", 362 | " rnn_units=rnn_units,\n", 363 | " batch_size=BATCH_SIZE)" 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": 151, 369 | "metadata": {}, 370 | "outputs": [ 371 | { 372 | "name": "stdout", 373 | "output_type": "stream", 374 | "text": [ 375 | "(64, 100, 65) # (batch_size, sequence_length, vocab_size)\n" 376 | ] 377 | } 378 | ], 379 | "source": [ 380 | "for input_example_batch, target_example_batch in dataset.take(1):\n", 381 | " example_batch_predictions = model(input_example_batch)\n", 382 | " print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")" 383 | ] 384 | }, 385 | { 386 | "cell_type": "code", 387 | "execution_count": 152, 388 | "metadata": {}, 389 | "outputs": [ 390 | { 391 | "name": "stdout", 392 | "output_type": "stream", 393 | "text": [ 394 | "Model: \"sequential_6\"\n", 395 | "_________________________________________________________________\n", 396 | "Layer (type) Output Shape Param # \n", 397 | "=================================================================\n", 398 | "embedding_6 (Embedding) (64, None, 256) 16640 \n", 399 | "_________________________________________________________________\n", 400 | "gru_6 (GRU) (64, None, 1024) 3938304 \n", 401 | "_________________________________________________________________\n", 402 | "dense_6 (Dense) (64, None, 65) 66625 \n", 403 | "=================================================================\n", 404 | "Total params: 4,021,569\n", 405 | "Trainable params: 4,021,569\n", 406 | "Non-trainable params: 0\n", 407 | "_________________________________________________________________\n" 408 | ] 409 | } 410 | ], 411 | "source": [ 412 | "model.summary()" 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "execution_count": 153, 418 | "metadata": {}, 419 | "outputs": [], 420 | "source": [ 421 | "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n", 422 | "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()" 423 | ] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": 154, 428 | "metadata": {}, 429 | "outputs": [ 430 | { 431 | "data": { 432 | "text/plain": [ 433 | "array([ 6, 62, 60, 27, 6, 27, 64, 17, 53, 8, 5, 63, 33, 38, 16, 18, 11,\n", 434 | " 37, 1, 38, 62, 36, 44, 61, 25, 21, 46, 20, 11, 46, 35, 47, 57, 58,\n", 435 | " 59, 14, 46, 54, 19, 63, 0, 3, 19, 36, 19, 60, 52, 57, 51, 28, 61,\n", 436 | " 18, 8, 0, 49, 17, 51, 34, 63, 1, 8, 63, 17, 17, 12, 51, 30, 1,\n", 437 | " 41, 41, 6, 59, 6, 51, 2, 64, 54, 11, 56, 33, 24, 60, 46, 41, 8,\n", 438 | " 1, 43, 6, 10, 16, 7, 15, 43, 37, 14, 36, 3, 2, 36, 19])" 439 | ] 440 | }, 441 | "execution_count": 154, 442 | "metadata": {}, 443 | "output_type": "execute_result" 444 | } 445 | ], 446 | "source": [ 447 | "sampled_indices" 448 | ] 449 | }, 450 | { 451 | "cell_type": "code", 452 | "execution_count": 155, 453 | "metadata": {}, 454 | "outputs": [ 455 | { 456 | "name": "stdout", 457 | "output_type": "stream", 458 | "text": [ 459 | "Input: \n", 460 | " '; and, now and then, a miserable patch of garden-ground,\\r\\nwith a few old boards knocked together for'\n", 461 | "\n", 462 | "Next Char Predictions: \n", 463 | " ')xvM)MzBo-(yT_AC1Y\\r_xWfwKFhE1hVistu;hpDy\\n!DWDvnsmNwC-\\nkBmUy\\r-yBB8mP\\rcc)u)m zp1rTIvhc-\\re)0A,?eY;W! WD'\n" 464 | ] 465 | } 466 | ], 467 | "source": [ 468 | "print(\"Input: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n", 469 | "print()\n", 470 | "print(\"Next Char Predictions: \\n\", repr(\"\".join(idx2char[sampled_indices ])))" 471 | ] 472 | }, 473 | { 474 | "cell_type": "code", 475 | "execution_count": 156, 476 | "metadata": {}, 477 | "outputs": [ 478 | { 479 | "name": "stdout", 480 | "output_type": "stream", 481 | "text": [ 482 | "Prediction shape: (64, 100, 65) # (batch_size, sequence_length, vocab_size)\n", 483 | "scalar_loss: 4.1737595\n" 484 | ] 485 | } 486 | ], 487 | "source": [ 488 | "def loss(labels, logits):\n", 489 | " return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n", 490 | "\n", 491 | "example_batch_loss = loss(target_example_batch, example_batch_predictions)\n", 492 | "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n", 493 | "print(\"scalar_loss: \", example_batch_loss.numpy().mean())" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": 157, 499 | "metadata": {}, 500 | "outputs": [], 501 | "source": [ 502 | "model.compile(optimizer='adam', loss=loss)" 503 | ] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "execution_count": 158, 508 | "metadata": {}, 509 | "outputs": [], 510 | "source": [ 511 | "# Directory where the checkpoints will be saved\n", 512 | "checkpoint_dir = './training_checkpoints'\n", 513 | "# Name of the checkpoint files\n", 514 | "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n", 515 | "\n", 516 | "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n", 517 | " filepath=checkpoint_prefix,\n", 518 | " save_weights_only=True)" 519 | ] 520 | }, 521 | { 522 | "cell_type": "code", 523 | "execution_count": 159, 524 | "metadata": {}, 525 | "outputs": [], 526 | "source": [ 527 | "EPOCHS=50" 528 | ] 529 | }, 530 | { 531 | "cell_type": "code", 532 | "execution_count": 160, 533 | "metadata": {}, 534 | "outputs": [ 535 | { 536 | "name": "stdout", 537 | "output_type": "stream", 538 | "text": [ 539 | "Train for 3 steps\n", 540 | "Epoch 1/50\n", 541 | "3/3 [==============================] - 6s 2s/step - loss: 4.1229\n", 542 | "Epoch 2/50\n", 543 | "3/3 [==============================] - 6s 2s/step - loss: 4.7004\n", 544 | "Epoch 3/50\n", 545 | "3/3 [==============================] - 6s 2s/step - loss: 3.6706\n", 546 | "Epoch 4/50\n", 547 | "3/3 [==============================] - 5s 2s/step - loss: 3.6113\n", 548 | "Epoch 5/50\n", 549 | "3/3 [==============================] - 5s 2s/step - loss: 3.4851\n", 550 | "Epoch 6/50\n", 551 | "3/3 [==============================] - 5s 2s/step - loss: 3.3605\n", 552 | "Epoch 7/50\n", 553 | "3/3 [==============================] - 5s 2s/step - loss: 3.1765\n", 554 | "Epoch 8/50\n", 555 | "3/3 [==============================] - 5s 2s/step - loss: 3.0148\n", 556 | "Epoch 9/50\n", 557 | "3/3 [==============================] - 5s 2s/step - loss: 3.0000\n", 558 | "Epoch 10/50\n", 559 | "3/3 [==============================] - 5s 2s/step - loss: 2.9265\n", 560 | "Epoch 11/50\n", 561 | "3/3 [==============================] - 6s 2s/step - loss: 2.8678\n", 562 | "Epoch 12/50\n", 563 | "3/3 [==============================] - 6s 2s/step - loss: 2.8282\n", 564 | "Epoch 13/50\n", 565 | "3/3 [==============================] - 6s 2s/step - loss: 2.7618\n", 566 | "Epoch 14/50\n", 567 | "3/3 [==============================] - 5s 2s/step - loss: 2.7032\n", 568 | "Epoch 15/50\n", 569 | "3/3 [==============================] - 6s 2s/step - loss: 2.6463\n", 570 | "Epoch 16/50\n", 571 | "3/3 [==============================] - 6s 2s/step - loss: 2.5814\n", 572 | "Epoch 17/50\n", 573 | "3/3 [==============================] - 6s 2s/step - loss: 2.5307\n", 574 | "Epoch 18/50\n", 575 | "3/3 [==============================] - 6s 2s/step - loss: 2.4905\n", 576 | "Epoch 19/50\n", 577 | "3/3 [==============================] - 6s 2s/step - loss: 2.4659\n", 578 | "Epoch 20/50\n", 579 | "3/3 [==============================] - 5s 2s/step - loss: 2.4188\n", 580 | "Epoch 21/50\n", 581 | "3/3 [==============================] - 6s 2s/step - loss: 2.4072\n", 582 | "Epoch 22/50\n", 583 | "3/3 [==============================] - 5s 2s/step - loss: 2.3888\n", 584 | "Epoch 23/50\n", 585 | "3/3 [==============================] - 5s 2s/step - loss: 2.3594\n", 586 | "Epoch 24/50\n", 587 | "3/3 [==============================] - 6s 2s/step - loss: 2.3371\n", 588 | "Epoch 25/50\n", 589 | "3/3 [==============================] - 6s 2s/step - loss: 2.3150\n", 590 | "Epoch 26/50\n", 591 | "3/3 [==============================] - 5s 2s/step - loss: 2.3006\n", 592 | "Epoch 27/50\n", 593 | "3/3 [==============================] - 6s 2s/step - loss: 2.2873\n", 594 | "Epoch 28/50\n", 595 | "3/3 [==============================] - 6s 2s/step - loss: 2.2723\n", 596 | "Epoch 29/50\n", 597 | "3/3 [==============================] - 6s 2s/step - loss: 2.2504\n", 598 | "Epoch 30/50\n", 599 | "3/3 [==============================] - 6s 2s/step - loss: 2.2471\n", 600 | "Epoch 31/50\n", 601 | "3/3 [==============================] - 6s 2s/step - loss: 2.2362\n", 602 | "Epoch 32/50\n", 603 | "3/3 [==============================] - 6s 2s/step - loss: 2.2193\n", 604 | "Epoch 33/50\n", 605 | "3/3 [==============================] - 5s 2s/step - loss: 2.2160\n", 606 | "Epoch 34/50\n", 607 | "3/3 [==============================] - 6s 2s/step - loss: 2.1977\n", 608 | "Epoch 35/50\n", 609 | "3/3 [==============================] - 6s 2s/step - loss: 2.1830\n", 610 | "Epoch 36/50\n", 611 | "3/3 [==============================] - 6s 2s/step - loss: 2.1765\n", 612 | "Epoch 37/50\n", 613 | "3/3 [==============================] - 6s 2s/step - loss: 2.1648\n", 614 | "Epoch 38/50\n", 615 | "3/3 [==============================] - 5s 2s/step - loss: 2.1538\n", 616 | "Epoch 39/50\n", 617 | "3/3 [==============================] - 5s 2s/step - loss: 2.1447\n", 618 | "Epoch 40/50\n", 619 | "3/3 [==============================] - 5s 2s/step - loss: 2.1291\n", 620 | "Epoch 41/50\n", 621 | "3/3 [==============================] - 5s 2s/step - loss: 2.1230\n", 622 | "Epoch 42/50\n", 623 | "3/3 [==============================] - 5s 2s/step - loss: 2.1063\n", 624 | "Epoch 43/50\n", 625 | "3/3 [==============================] - 5s 2s/step - loss: 2.0907\n", 626 | "Epoch 44/50\n", 627 | "3/3 [==============================] - 5s 2s/step - loss: 2.0796\n", 628 | "Epoch 45/50\n", 629 | "3/3 [==============================] - 5s 2s/step - loss: 2.0872\n", 630 | "Epoch 46/50\n", 631 | "3/3 [==============================] - 5s 2s/step - loss: 2.0582\n", 632 | "Epoch 47/50\n", 633 | "3/3 [==============================] - 6s 2s/step - loss: 2.0529\n", 634 | "Epoch 48/50\n", 635 | "3/3 [==============================] - 5s 2s/step - loss: 2.0433\n", 636 | "Epoch 49/50\n", 637 | "3/3 [==============================] - 5s 2s/step - loss: 2.0308\n", 638 | "Epoch 50/50\n", 639 | "3/3 [==============================] - 5s 2s/step - loss: 2.0107\n" 640 | ] 641 | } 642 | ], 643 | "source": [ 644 | "history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])" 645 | ] 646 | }, 647 | { 648 | "cell_type": "code", 649 | "execution_count": 161, 650 | "metadata": {}, 651 | "outputs": [ 652 | { 653 | "data": { 654 | "text/plain": [ 655 | "'./training_checkpoints/ckpt_50'" 656 | ] 657 | }, 658 | "execution_count": 161, 659 | "metadata": {}, 660 | "output_type": "execute_result" 661 | } 662 | ], 663 | "source": [ 664 | "tf.train.latest_checkpoint(checkpoint_dir)" 665 | ] 666 | }, 667 | { 668 | "cell_type": "code", 669 | "execution_count": 162, 670 | "metadata": {}, 671 | "outputs": [], 672 | "source": [ 673 | "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n", 674 | "\n", 675 | "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", 676 | "\n", 677 | "model.build(tf.TensorShape([1, None]))" 678 | ] 679 | }, 680 | { 681 | "cell_type": "code", 682 | "execution_count": 163, 683 | "metadata": {}, 684 | "outputs": [ 685 | { 686 | "name": "stdout", 687 | "output_type": "stream", 688 | "text": [ 689 | "Model: \"sequential_7\"\n", 690 | "_________________________________________________________________\n", 691 | "Layer (type) Output Shape Param # \n", 692 | "=================================================================\n", 693 | "embedding_7 (Embedding) (1, None, 256) 16640 \n", 694 | "_________________________________________________________________\n", 695 | "gru_7 (GRU) (1, None, 1024) 3938304 \n", 696 | "_________________________________________________________________\n", 697 | "dense_7 (Dense) (1, None, 65) 66625 \n", 698 | "=================================================================\n", 699 | "Total params: 4,021,569\n", 700 | "Trainable params: 4,021,569\n", 701 | "Non-trainable params: 0\n", 702 | "_________________________________________________________________\n" 703 | ] 704 | } 705 | ], 706 | "source": [ 707 | "model.summary()" 708 | ] 709 | }, 710 | { 711 | "cell_type": "code", 712 | "execution_count": 215, 713 | "metadata": {}, 714 | "outputs": [], 715 | "source": [ 716 | "def generate_text(model, start_string):\n", 717 | " # Evaluation step (generating text using the learned model)\n", 718 | "\n", 719 | " # Number of characters to generate\n", 720 | " num_generate = 5\n", 721 | "\n", 722 | " # Converting our start string to numbers (vectorizing)\n", 723 | " input_eval = [char2idx[s] for s in start_string]\n", 724 | " input_eval = tf.expand_dims(input_eval, 0)\n", 725 | "\n", 726 | " # Empty string to store our results\n", 727 | " text_generated = []\n", 728 | "\n", 729 | " # Low temperatures results in more predictable text.\n", 730 | " # Higher temperatures results in more surprising text.\n", 731 | " # Experiment to find the best setting.\n", 732 | " temperature = 1.0\n", 733 | "\n", 734 | " # Here batch size == 1\n", 735 | " model.reset_states()\n", 736 | " for i in range(num_generate):\n", 737 | " predictions = model(input_eval)\n", 738 | " # remove the batch dimension\n", 739 | " predictions = tf.squeeze(predictions, 0)\n", 740 | "\n", 741 | " # using a categorical distribution to predict the character returned by the model\n", 742 | " predictions = predictions / temperature\n", 743 | " predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n", 744 | "\n", 745 | " # We pass the predicted character as the next input to the model\n", 746 | " # along with the previous hidden state\n", 747 | " input_eval = tf.expand_dims([predicted_id], 0)\n", 748 | "\n", 749 | " text_generated.append(idx2char[predicted_id])\n", 750 | "\n", 751 | " return (start_string + ''.join(text_generated))" 752 | ] 753 | }, 754 | { 755 | "cell_type": "code", 756 | "execution_count": 216, 757 | "metadata": {}, 758 | "outputs": [ 759 | { 760 | "name": "stdout", 761 | "output_type": "stream", 762 | "text": [ 763 | "The sky is caig\n" 764 | ] 765 | } 766 | ], 767 | "source": [ 768 | "print(generate_text(model, start_string=u\"The sky is\"))" 769 | ] 770 | }, 771 | { 772 | "cell_type": "code", 773 | "execution_count": 217, 774 | "metadata": {}, 775 | "outputs": [ 776 | { 777 | "name": "stdout", 778 | "output_type": "stream", 779 | "text": [ 780 | "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.iter\n", 781 | "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_1\n", 782 | "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_2\n", 783 | "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.decay\n", 784 | "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.learning_rate\n", 785 | "WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/guide/checkpoint#loading_mechanics for details.\n" 786 | ] 787 | } 788 | ], 789 | "source": [ 790 | "model = build_model(\n", 791 | " vocab_size = len(vocab),\n", 792 | " embedding_dim=embedding_dim,\n", 793 | " rnn_units=rnn_units,\n", 794 | " batch_size=BATCH_SIZE)" 795 | ] 796 | }, 797 | { 798 | "cell_type": "code", 799 | "execution_count": 218, 800 | "metadata": {}, 801 | "outputs": [], 802 | "source": [ 803 | "optimizer = tf.keras.optimizers.Adam()" 804 | ] 805 | }, 806 | { 807 | "cell_type": "code", 808 | "execution_count": 219, 809 | "metadata": {}, 810 | "outputs": [], 811 | "source": [ 812 | "@tf.function\n", 813 | "def train_step(inp, target):\n", 814 | " with tf.GradientTape() as tape:\n", 815 | " predictions = model(inp)\n", 816 | " loss = tf.reduce_mean(\n", 817 | " tf.keras.losses.sparse_categorical_crossentropy(\n", 818 | " target, predictions, from_logits=True))\n", 819 | " grads = tape.gradient(loss, model.trainable_variables)\n", 820 | " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", 821 | "\n", 822 | " return loss" 823 | ] 824 | }, 825 | { 826 | "cell_type": "code", 827 | "execution_count": 220, 828 | "metadata": {}, 829 | "outputs": [ 830 | { 831 | "name": "stdout", 832 | "output_type": "stream", 833 | "text": [ 834 | "Epoch 1 Batch 0 Loss 4.175134658813477\n", 835 | "Epoch 1 Loss 4.0693\n", 836 | "Time taken for 1 epoch 6.118037939071655 sec\n", 837 | "\n", 838 | "Epoch 2 Batch 0 Loss 3.8320510387420654\n", 839 | "Epoch 2 Loss 3.5987\n", 840 | "Time taken for 1 epoch 5.000967025756836 sec\n", 841 | "\n", 842 | "Epoch 3 Batch 0 Loss 3.8342347145080566\n", 843 | "Epoch 3 Loss 3.9507\n", 844 | "Time taken for 1 epoch 5.501431703567505 sec\n", 845 | "\n", 846 | "Epoch 4 Batch 0 Loss 3.955986261367798\n", 847 | "Epoch 4 Loss 3.9279\n", 848 | "Time taken for 1 epoch 5.202840566635132 sec\n", 849 | "\n", 850 | "Epoch 5 Batch 0 Loss 3.906343936920166\n", 851 | "Epoch 5 Loss 3.8335\n", 852 | "Time taken for 1 epoch 5.206571102142334 sec\n", 853 | "\n", 854 | "Epoch 6 Batch 0 Loss 3.7908694744110107\n", 855 | "Epoch 6 Loss 3.6584\n", 856 | "Time taken for 1 epoch 5.032634735107422 sec\n", 857 | "\n", 858 | "Epoch 7 Batch 0 Loss 3.572174072265625\n", 859 | "Epoch 7 Loss 3.3720\n", 860 | "Time taken for 1 epoch 5.031881809234619 sec\n", 861 | "\n", 862 | "Epoch 8 Batch 0 Loss 3.3343067169189453\n", 863 | "Epoch 8 Loss 3.1891\n", 864 | "Time taken for 1 epoch 5.054827928543091 sec\n", 865 | "\n", 866 | "Epoch 9 Batch 0 Loss 3.074802875518799\n", 867 | "Epoch 9 Loss 2.9634\n", 868 | "Time taken for 1 epoch 5.0290751457214355 sec\n", 869 | "\n", 870 | "Epoch 10 Batch 0 Loss 3.0132524967193604\n", 871 | "Epoch 10 Loss 2.9682\n", 872 | "Time taken for 1 epoch 5.139891147613525 sec\n", 873 | "\n" 874 | ] 875 | } 876 | ], 877 | "source": [ 878 | "# Training step\n", 879 | "EPOCHS = 10\n", 880 | "\n", 881 | "for epoch in range(EPOCHS):\n", 882 | " start = time.time()\n", 883 | "\n", 884 | " # initializing the hidden state at the start of every epoch\n", 885 | " # initally hidden is None\n", 886 | " hidden = model.reset_states()\n", 887 | "\n", 888 | " for (batch_n, (inp, target)) in enumerate(dataset):\n", 889 | " loss = train_step(inp, target)\n", 890 | "\n", 891 | " if batch_n % 100 == 0:\n", 892 | " template = 'Epoch {} Batch {} Loss {}'\n", 893 | " print(template.format(epoch+1, batch_n, loss))\n", 894 | "\n", 895 | " # saving (checkpoint) the model every 5 epochs\n", 896 | " if (epoch + 1) % 5 == 0:\n", 897 | " model.save_weights(checkpoint_prefix.format(epoch=epoch))\n", 898 | "\n", 899 | " print ('Epoch {} Loss {:.4f}'.format(epoch+1, loss))\n", 900 | " print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n", 901 | "\n", 902 | "model.save_weights(checkpoint_prefix.format(epoch=epoch))" 903 | ] 904 | }, 905 | { 906 | "cell_type": "code", 907 | "execution_count": null, 908 | "metadata": {}, 909 | "outputs": [], 910 | "source": [] 911 | } 912 | ], 913 | "metadata": { 914 | "kernelspec": { 915 | "display_name": "Python 3", 916 | "language": "python", 917 | "name": "python3" 918 | }, 919 | "language_info": { 920 | "codemirror_mode": { 921 | "name": "ipython", 922 | "version": 3 923 | }, 924 | "file_extension": ".py", 925 | "mimetype": "text/x-python", 926 | "name": "python", 927 | "nbconvert_exporter": "python", 928 | "pygments_lexer": "ipython3", 929 | "version": "3.7.6" 930 | } 931 | }, 932 | "nbformat": 4, 933 | "nbformat_minor": 4 934 | } 935 | --------------------------------------------------------------------------------