├── .ipynb_checkpoints ├── 03. PyTorch Fundamentals - Matrices-checkpoint.ipynb ├── 04. PyTorch Fundamentals - Variables and Gradients-checkpoint.ipynb ├── 05. Linear Regression with PyTorch-checkpoint.ipynb ├── 06. Logistic Regression with PyTorch-checkpoint.ipynb ├── 07. Feedforward Neural Network with PyTorch-checkpoint.ipynb ├── 08. Convolutional Neural Network (CNN) with PyTorch-checkpoint.ipynb ├── 09. Recurrent Neural Networks (RNN)-checkpoint.ipynb └── 10. Long Short-Term Memory Networks (LSTM)-checkpoint.ipynb ├── 03. PyTorch Fundamentals - Matrices.ipynb ├── 04. PyTorch Fundamentals - Variables and Gradients.ipynb ├── 05. Linear Regression with PyTorch.ipynb ├── 06. Logistic Regression with PyTorch.ipynb ├── 07. Feedforward Neural Network with PyTorch.ipynb ├── 08. Convolutional Neural Network (CNN) with PyTorch.ipynb ├── 09. Recurrent Neural Networks (RNN).ipynb ├── 10. Long Short-Term Memory Networks (LSTM).ipynb └── data ├── processed ├── test.pt └── training.pt └── raw ├── t10k-images-idx3-ubyte ├── t10k-labels-idx1-ubyte ├── train-images-idx3-ubyte └── train-labels-idx1-ubyte /.ipynb_checkpoints/03. PyTorch Fundamentals - Matrices-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Matrix Basics" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 2, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 3, 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | "[[1, 2], [3, 4]]\n" 29 | ] 30 | } 31 | ], 32 | "source": [ 33 | "# Creating a 2x2 array\n", 34 | "arr = [[1, 2], [3, 4]]\n", 35 | "print(arr)" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 4, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "data": { 45 | "text/plain": [ 46 | "array([[1, 2],\n", 47 | " [3, 4]])" 48 | ] 49 | }, 50 | "execution_count": 4, 51 | "metadata": {}, 52 | "output_type": "execute_result" 53 | } 54 | ], 55 | "source": [ 56 | "# Convert to NumPy\n", 57 | "np.array(arr)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 5, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "import torch" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 6, 72 | "metadata": {}, 73 | "outputs": [ 74 | { 75 | "data": { 76 | "text/plain": [ 77 | "tensor([[ 1., 2.],\n", 78 | " [ 3., 4.]])" 79 | ] 80 | }, 81 | "execution_count": 6, 82 | "metadata": {}, 83 | "output_type": "execute_result" 84 | } 85 | ], 86 | "source": [ 87 | "# Convert to PyTorch Tensor\n", 88 | "torch.Tensor(arr)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 7, 94 | "metadata": {}, 95 | "outputs": [ 96 | { 97 | "data": { 98 | "text/plain": [ 99 | "array([[1., 1.],\n", 100 | " [1., 1.]])" 101 | ] 102 | }, 103 | "execution_count": 7, 104 | "metadata": {}, 105 | "output_type": "execute_result" 106 | } 107 | ], 108 | "source": [ 109 | "np.ones((2, 2))" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 8, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "data": { 119 | "text/plain": [ 120 | "tensor([[ 1., 1.],\n", 121 | " [ 1., 1.]])" 122 | ] 123 | }, 124 | "execution_count": 8, 125 | "metadata": {}, 126 | "output_type": "execute_result" 127 | } 128 | ], 129 | "source": [ 130 | "torch.ones((2, 2))" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 9, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "data": { 140 | "text/plain": [ 141 | "array([[0.54314946, 0.32265673],\n", 142 | " [0.76294125, 0.58130214]])" 143 | ] 144 | }, 145 | "execution_count": 9, 146 | "metadata": {}, 147 | "output_type": "execute_result" 148 | } 149 | ], 150 | "source": [ 151 | "np.random.rand(2, 2)" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 10, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "data": { 161 | "text/plain": [ 162 | "tensor([[ 0.1709, 0.6083],\n", 163 | " [ 0.6287, 0.5718]])" 164 | ] 165 | }, 166 | "execution_count": 10, 167 | "metadata": {}, 168 | "output_type": "execute_result" 169 | } 170 | ], 171 | "source": [ 172 | "torch.rand(2, 2)" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": {}, 178 | "source": [ 179 | "# Seed for Reproducibility" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 11, 185 | "metadata": {}, 186 | "outputs": [ 187 | { 188 | "data": { 189 | "text/plain": [ 190 | "array([[0.5488135 , 0.71518937],\n", 191 | " [0.60276338, 0.54488318]])" 192 | ] 193 | }, 194 | "execution_count": 11, 195 | "metadata": {}, 196 | "output_type": "execute_result" 197 | } 198 | ], 199 | "source": [ 200 | "# Seed\n", 201 | "np.random.seed(0)\n", 202 | "np.random.rand(2, 2)" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 12, 208 | "metadata": {}, 209 | "outputs": [ 210 | { 211 | "data": { 212 | "text/plain": [ 213 | "array([[0.5488135 , 0.71518937],\n", 214 | " [0.60276338, 0.54488318]])" 215 | ] 216 | }, 217 | "execution_count": 12, 218 | "metadata": {}, 219 | "output_type": "execute_result" 220 | } 221 | ], 222 | "source": [ 223 | "# Seed\n", 224 | "np.random.seed(0)\n", 225 | "np.random.rand(2, 2)" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 13, 231 | "metadata": {}, 232 | "outputs": [ 233 | { 234 | "data": { 235 | "text/plain": [ 236 | "array([[0.4236548 , 0.64589411],\n", 237 | " [0.43758721, 0.891773 ]])" 238 | ] 239 | }, 240 | "execution_count": 13, 241 | "metadata": {}, 242 | "output_type": "execute_result" 243 | } 244 | ], 245 | "source": [ 246 | "# No seed\n", 247 | "np.random.rand(2, 2)" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 14, 253 | "metadata": {}, 254 | "outputs": [ 255 | { 256 | "data": { 257 | "text/plain": [ 258 | "array([[0.96366276, 0.38344152],\n", 259 | " [0.79172504, 0.52889492]])" 260 | ] 261 | }, 262 | "execution_count": 14, 263 | "metadata": {}, 264 | "output_type": "execute_result" 265 | } 266 | ], 267 | "source": [ 268 | "# No seed\n", 269 | "np.random.rand(2, 2)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 15, 275 | "metadata": {}, 276 | "outputs": [ 277 | { 278 | "data": { 279 | "text/plain": [ 280 | "tensor([[ 0.4963, 0.7682],\n", 281 | " [ 0.0885, 0.1320]])" 282 | ] 283 | }, 284 | "execution_count": 15, 285 | "metadata": {}, 286 | "output_type": "execute_result" 287 | } 288 | ], 289 | "source": [ 290 | "# Torch Seed\n", 291 | "torch.manual_seed(0)\n", 292 | "torch.rand(2, 2)" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": 16, 298 | "metadata": {}, 299 | "outputs": [ 300 | { 301 | "data": { 302 | "text/plain": [ 303 | "tensor([[ 0.4963, 0.7682],\n", 304 | " [ 0.0885, 0.1320]])" 305 | ] 306 | }, 307 | "execution_count": 16, 308 | "metadata": {}, 309 | "output_type": "execute_result" 310 | } 311 | ], 312 | "source": [ 313 | "# Torch Seed\n", 314 | "torch.manual_seed(0)\n", 315 | "torch.rand(2, 2)" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": 17, 321 | "metadata": {}, 322 | "outputs": [ 323 | { 324 | "data": { 325 | "text/plain": [ 326 | "tensor([[ 0.3074, 0.6341],\n", 327 | " [ 0.4901, 0.8964]])" 328 | ] 329 | }, 330 | "execution_count": 17, 331 | "metadata": {}, 332 | "output_type": "execute_result" 333 | } 334 | ], 335 | "source": [ 336 | "# Torch No Seed\n", 337 | "torch.rand(2, 2)" 338 | ] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": 18, 343 | "metadata": {}, 344 | "outputs": [ 345 | { 346 | "data": { 347 | "text/plain": [ 348 | "tensor([[ 0.4556, 0.6323],\n", 349 | " [ 0.3489, 0.4017]])" 350 | ] 351 | }, 352 | "execution_count": 18, 353 | "metadata": {}, 354 | "output_type": "execute_result" 355 | } 356 | ], 357 | "source": [ 358 | "# Torch No Seed\n", 359 | "torch.rand(2, 2)" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": 19, 365 | "metadata": {}, 366 | "outputs": [ 367 | { 368 | "data": { 369 | "text/plain": [ 370 | "False" 371 | ] 372 | }, 373 | "execution_count": 19, 374 | "metadata": {}, 375 | "output_type": "execute_result" 376 | } 377 | ], 378 | "source": [ 379 | "torch.cuda.is_available()" 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": 20, 385 | "metadata": {}, 386 | "outputs": [], 387 | "source": [ 388 | "if torch.cuda.is_available():\n", 389 | " torch.cuda.manual_seed_all(0)" 390 | ] 391 | }, 392 | { 393 | "cell_type": "markdown", 394 | "metadata": {}, 395 | "source": [ 396 | "# Torch to NumPy Bridge" 397 | ] 398 | }, 399 | { 400 | "cell_type": "code", 401 | "execution_count": 21, 402 | "metadata": {}, 403 | "outputs": [], 404 | "source": [ 405 | "# Numpy array\n", 406 | "np_array = np.ones((2, 2))" 407 | ] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": 22, 412 | "metadata": {}, 413 | "outputs": [ 414 | { 415 | "name": "stdout", 416 | "output_type": "stream", 417 | "text": [ 418 | "[[1. 1.]\n", 419 | " [1. 1.]]\n" 420 | ] 421 | } 422 | ], 423 | "source": [ 424 | "print(np_array)" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": 23, 430 | "metadata": {}, 431 | "outputs": [ 432 | { 433 | "name": "stdout", 434 | "output_type": "stream", 435 | "text": [ 436 | "\n" 437 | ] 438 | } 439 | ], 440 | "source": [ 441 | "print(type(np_array))" 442 | ] 443 | }, 444 | { 445 | "cell_type": "code", 446 | "execution_count": 24, 447 | "metadata": {}, 448 | "outputs": [], 449 | "source": [ 450 | "# Convert to Torch Tensor\n", 451 | "torch_tensor = torch.from_numpy(np_array)" 452 | ] 453 | }, 454 | { 455 | "cell_type": "code", 456 | "execution_count": 25, 457 | "metadata": {}, 458 | "outputs": [ 459 | { 460 | "name": "stdout", 461 | "output_type": "stream", 462 | "text": [ 463 | "tensor([[ 1., 1.],\n", 464 | " [ 1., 1.]], dtype=torch.float64)\n" 465 | ] 466 | } 467 | ], 468 | "source": [ 469 | "print(torch_tensor)" 470 | ] 471 | }, 472 | { 473 | "cell_type": "code", 474 | "execution_count": 26, 475 | "metadata": {}, 476 | "outputs": [ 477 | { 478 | "name": "stdout", 479 | "output_type": "stream", 480 | "text": [ 481 | "\n" 482 | ] 483 | } 484 | ], 485 | "source": [ 486 | "print(type(torch_tensor))" 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 27, 492 | "metadata": {}, 493 | "outputs": [ 494 | { 495 | "ename": "TypeError", 496 | "evalue": "can't convert np.ndarray of type numpy.int8. The only supported types are: double, float, float16, int64, int32, and uint8.", 497 | "output_type": "error", 498 | "traceback": [ 499 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 500 | "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", 501 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Data types matter: intentional error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mnp_array_new\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp_array_new\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 502 | "\u001b[0;31mTypeError\u001b[0m: can't convert np.ndarray of type numpy.int8. The only supported types are: double, float, float16, int64, int32, and uint8." 503 | ] 504 | } 505 | ], 506 | "source": [ 507 | "# Data types matter: intentional error\n", 508 | "np_array_new = np.ones((2, 2), dtype=np.int8)\n", 509 | "torch.from_numpy(np_array_new)" 510 | ] 511 | }, 512 | { 513 | "cell_type": "code", 514 | "execution_count": 30, 515 | "metadata": {}, 516 | "outputs": [ 517 | { 518 | "data": { 519 | "text/plain": [ 520 | "tensor([[ 1, 1],\n", 521 | " [ 1, 1]])" 522 | ] 523 | }, 524 | "execution_count": 30, 525 | "metadata": {}, 526 | "output_type": "execute_result" 527 | } 528 | ], 529 | "source": [ 530 | "# Data types matter\n", 531 | "np_array_new = np.ones((2, 2), dtype=np.int64)\n", 532 | "torch.from_numpy(np_array_new)" 533 | ] 534 | }, 535 | { 536 | "cell_type": "code", 537 | "execution_count": 33, 538 | "metadata": {}, 539 | "outputs": [ 540 | { 541 | "data": { 542 | "text/plain": [ 543 | "tensor([[ 1, 1],\n", 544 | " [ 1, 1]], dtype=torch.int32)" 545 | ] 546 | }, 547 | "execution_count": 33, 548 | "metadata": {}, 549 | "output_type": "execute_result" 550 | } 551 | ], 552 | "source": [ 553 | "# Data types matter\n", 554 | "np_array_new = np.ones((2, 2), dtype=np.int32)\n", 555 | "torch.from_numpy(np_array_new)" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": 36, 561 | "metadata": {}, 562 | "outputs": [ 563 | { 564 | "data": { 565 | "text/plain": [ 566 | "tensor([[ 1, 1],\n", 567 | " [ 1, 1]], dtype=torch.uint8)" 568 | ] 569 | }, 570 | "execution_count": 36, 571 | "metadata": {}, 572 | "output_type": "execute_result" 573 | } 574 | ], 575 | "source": [ 576 | "# Data types matter\n", 577 | "np_array_new = np.ones((2, 2), dtype=np.uint8)\n", 578 | "torch.from_numpy(np_array_new)" 579 | ] 580 | }, 581 | { 582 | "cell_type": "code", 583 | "execution_count": 39, 584 | "metadata": {}, 585 | "outputs": [ 586 | { 587 | "data": { 588 | "text/plain": [ 589 | "tensor([[ 1., 1.],\n", 590 | " [ 1., 1.]], dtype=torch.float64)" 591 | ] 592 | }, 593 | "execution_count": 39, 594 | "metadata": {}, 595 | "output_type": "execute_result" 596 | } 597 | ], 598 | "source": [ 599 | "# Data types matter\n", 600 | "np_array_new = np.ones((2, 2), dtype=np.float64)\n", 601 | "torch.from_numpy(np_array_new)" 602 | ] 603 | }, 604 | { 605 | "cell_type": "code", 606 | "execution_count": 42, 607 | "metadata": {}, 608 | "outputs": [ 609 | { 610 | "data": { 611 | "text/plain": [ 612 | "tensor([[ 1., 1.],\n", 613 | " [ 1., 1.]])" 614 | ] 615 | }, 616 | "execution_count": 42, 617 | "metadata": {}, 618 | "output_type": "execute_result" 619 | } 620 | ], 621 | "source": [ 622 | "# Data types matter\n", 623 | "np_array_new = np.ones((2, 2), dtype=np.float32)\n", 624 | "torch.from_numpy(np_array_new)" 625 | ] 626 | }, 627 | { 628 | "cell_type": "code", 629 | "execution_count": 45, 630 | "metadata": {}, 631 | "outputs": [ 632 | { 633 | "data": { 634 | "text/plain": [ 635 | "tensor([[ 1., 1.],\n", 636 | " [ 1., 1.]], dtype=torch.float64)" 637 | ] 638 | }, 639 | "execution_count": 45, 640 | "metadata": {}, 641 | "output_type": "execute_result" 642 | } 643 | ], 644 | "source": [ 645 | "# Data types matter\n", 646 | "np_array_new = np.ones((2, 2), dtype=np.double)\n", 647 | "torch.from_numpy(np_array_new)" 648 | ] 649 | }, 650 | { 651 | "cell_type": "markdown", 652 | "metadata": {}, 653 | "source": [ 654 | "# NumPy to Torch Bridge" 655 | ] 656 | }, 657 | { 658 | "cell_type": "code", 659 | "execution_count": 46, 660 | "metadata": {}, 661 | "outputs": [], 662 | "source": [ 663 | "torch_tensor = torch.ones(2, 2)" 664 | ] 665 | }, 666 | { 667 | "cell_type": "code", 668 | "execution_count": 47, 669 | "metadata": {}, 670 | "outputs": [ 671 | { 672 | "data": { 673 | "text/plain": [ 674 | "torch.Tensor" 675 | ] 676 | }, 677 | "execution_count": 47, 678 | "metadata": {}, 679 | "output_type": "execute_result" 680 | } 681 | ], 682 | "source": [ 683 | "type(torch_tensor)" 684 | ] 685 | }, 686 | { 687 | "cell_type": "code", 688 | "execution_count": 48, 689 | "metadata": {}, 690 | "outputs": [], 691 | "source": [ 692 | "torch_to_numpy = torch_tensor.numpy()" 693 | ] 694 | }, 695 | { 696 | "cell_type": "code", 697 | "execution_count": 49, 698 | "metadata": { 699 | "scrolled": true 700 | }, 701 | "outputs": [ 702 | { 703 | "data": { 704 | "text/plain": [ 705 | "numpy.ndarray" 706 | ] 707 | }, 708 | "execution_count": 49, 709 | "metadata": {}, 710 | "output_type": "execute_result" 711 | } 712 | ], 713 | "source": [ 714 | "type(torch_to_numpy)" 715 | ] 716 | }, 717 | { 718 | "cell_type": "markdown", 719 | "metadata": {}, 720 | "source": [ 721 | "# GPU and CPU Toggling" 722 | ] 723 | }, 724 | { 725 | "cell_type": "code", 726 | "execution_count": 51, 727 | "metadata": {}, 728 | "outputs": [], 729 | "source": [ 730 | "# CPU\n", 731 | "tensor_cpu = torch.ones(2, 2)" 732 | ] 733 | }, 734 | { 735 | "cell_type": "code", 736 | "execution_count": 54, 737 | "metadata": {}, 738 | "outputs": [], 739 | "source": [ 740 | "# CPU to GPU\n", 741 | "if torch.cuda.is_available():\n", 742 | " tensor_cpu.cuda()" 743 | ] 744 | }, 745 | { 746 | "cell_type": "code", 747 | "execution_count": 55, 748 | "metadata": { 749 | "scrolled": true 750 | }, 751 | "outputs": [ 752 | { 753 | "data": { 754 | "text/plain": [ 755 | "tensor([[ 1., 1.],\n", 756 | " [ 1., 1.]])" 757 | ] 758 | }, 759 | "execution_count": 55, 760 | "metadata": {}, 761 | "output_type": "execute_result" 762 | } 763 | ], 764 | "source": [ 765 | "# GPU to CPU\n", 766 | "tensor_cpu.cpu()" 767 | ] 768 | }, 769 | { 770 | "cell_type": "markdown", 771 | "metadata": {}, 772 | "source": [ 773 | "# Basic Mathematical Tensor Operations" 774 | ] 775 | }, 776 | { 777 | "cell_type": "code", 778 | "execution_count": 56, 779 | "metadata": {}, 780 | "outputs": [ 781 | { 782 | "name": "stdout", 783 | "output_type": "stream", 784 | "text": [ 785 | "tensor([[ 1., 1.],\n", 786 | " [ 1., 1.]])\n" 787 | ] 788 | } 789 | ], 790 | "source": [ 791 | "a = torch.ones(2, 2)\n", 792 | "print(a)" 793 | ] 794 | }, 795 | { 796 | "cell_type": "code", 797 | "execution_count": 57, 798 | "metadata": {}, 799 | "outputs": [ 800 | { 801 | "name": "stdout", 802 | "output_type": "stream", 803 | "text": [ 804 | "torch.Size([2, 2])\n" 805 | ] 806 | } 807 | ], 808 | "source": [ 809 | "print(a.size())" 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": 58, 815 | "metadata": {}, 816 | "outputs": [ 817 | { 818 | "data": { 819 | "text/plain": [ 820 | "tensor([ 1., 1., 1., 1.])" 821 | ] 822 | }, 823 | "execution_count": 58, 824 | "metadata": {}, 825 | "output_type": "execute_result" 826 | } 827 | ], 828 | "source": [ 829 | "a.view(4)" 830 | ] 831 | }, 832 | { 833 | "cell_type": "code", 834 | "execution_count": 59, 835 | "metadata": {}, 836 | "outputs": [ 837 | { 838 | "data": { 839 | "text/plain": [ 840 | "torch.Size([4])" 841 | ] 842 | }, 843 | "execution_count": 59, 844 | "metadata": {}, 845 | "output_type": "execute_result" 846 | } 847 | ], 848 | "source": [ 849 | "a.view(4).size()" 850 | ] 851 | }, 852 | { 853 | "cell_type": "code", 854 | "execution_count": 60, 855 | "metadata": {}, 856 | "outputs": [ 857 | { 858 | "name": "stdout", 859 | "output_type": "stream", 860 | "text": [ 861 | "tensor([[ 1., 1.],\n", 862 | " [ 1., 1.]])\n" 863 | ] 864 | } 865 | ], 866 | "source": [ 867 | "a = torch.ones(2, 2)\n", 868 | "print(a)" 869 | ] 870 | }, 871 | { 872 | "cell_type": "code", 873 | "execution_count": 61, 874 | "metadata": {}, 875 | "outputs": [ 876 | { 877 | "name": "stdout", 878 | "output_type": "stream", 879 | "text": [ 880 | "tensor([[ 1., 1.],\n", 881 | " [ 1., 1.]])\n" 882 | ] 883 | } 884 | ], 885 | "source": [ 886 | "b = torch.ones(2, 2)\n", 887 | "print(b)" 888 | ] 889 | }, 890 | { 891 | "cell_type": "code", 892 | "execution_count": 62, 893 | "metadata": {}, 894 | "outputs": [ 895 | { 896 | "name": "stdout", 897 | "output_type": "stream", 898 | "text": [ 899 | "tensor([[ 2., 2.],\n", 900 | " [ 2., 2.]])\n" 901 | ] 902 | } 903 | ], 904 | "source": [ 905 | "# Element-wise addition\n", 906 | "c = a + b\n", 907 | "print(c)" 908 | ] 909 | }, 910 | { 911 | "cell_type": "code", 912 | "execution_count": 63, 913 | "metadata": {}, 914 | "outputs": [ 915 | { 916 | "name": "stdout", 917 | "output_type": "stream", 918 | "text": [ 919 | "tensor([[ 2., 2.],\n", 920 | " [ 2., 2.]])\n" 921 | ] 922 | } 923 | ], 924 | "source": [ 925 | "# Element-wise addition\n", 926 | "c = torch.add(a, b)\n", 927 | "print(c)" 928 | ] 929 | }, 930 | { 931 | "cell_type": "code", 932 | "execution_count": 64, 933 | "metadata": {}, 934 | "outputs": [ 935 | { 936 | "name": "stdout", 937 | "output_type": "stream", 938 | "text": [ 939 | "Old c tensor\n", 940 | "tensor([[ 2., 2.],\n", 941 | " [ 2., 2.]])\n", 942 | "------------------------------------------------------------\n", 943 | "New c tensor\n", 944 | "tensor([[ 3., 3.],\n", 945 | " [ 3., 3.]])\n" 946 | ] 947 | } 948 | ], 949 | "source": [ 950 | "# In-place addition\n", 951 | "print('Old c tensor')\n", 952 | "print(c)\n", 953 | "\n", 954 | "c.add_(a)\n", 955 | "\n", 956 | "print('-'*60)\n", 957 | "print('New c tensor')\n", 958 | "print(c)" 959 | ] 960 | }, 961 | { 962 | "cell_type": "code", 963 | "execution_count": 65, 964 | "metadata": {}, 965 | "outputs": [ 966 | { 967 | "name": "stdout", 968 | "output_type": "stream", 969 | "text": [ 970 | "tensor([[ 1., 1.],\n", 971 | " [ 1., 1.]])\n", 972 | "tensor([[ 1., 1.],\n", 973 | " [ 1., 1.]])\n" 974 | ] 975 | } 976 | ], 977 | "source": [ 978 | "print(a)\n", 979 | "print(b)" 980 | ] 981 | }, 982 | { 983 | "cell_type": "code", 984 | "execution_count": 66, 985 | "metadata": {}, 986 | "outputs": [ 987 | { 988 | "data": { 989 | "text/plain": [ 990 | "tensor([[ 0., 0.],\n", 991 | " [ 0., 0.]])" 992 | ] 993 | }, 994 | "execution_count": 66, 995 | "metadata": {}, 996 | "output_type": "execute_result" 997 | } 998 | ], 999 | "source": [ 1000 | "a - b" 1001 | ] 1002 | }, 1003 | { 1004 | "cell_type": "code", 1005 | "execution_count": 67, 1006 | "metadata": {}, 1007 | "outputs": [ 1008 | { 1009 | "name": "stdout", 1010 | "output_type": "stream", 1011 | "text": [ 1012 | "tensor([[ 0., 0.],\n", 1013 | " [ 0., 0.]])\n", 1014 | "tensor([[ 1., 1.],\n", 1015 | " [ 1., 1.]])\n" 1016 | ] 1017 | } 1018 | ], 1019 | "source": [ 1020 | "# Not in-place\n", 1021 | "print(a.sub(b))\n", 1022 | "print(a)" 1023 | ] 1024 | }, 1025 | { 1026 | "cell_type": "code", 1027 | "execution_count": 68, 1028 | "metadata": {}, 1029 | "outputs": [ 1030 | { 1031 | "name": "stdout", 1032 | "output_type": "stream", 1033 | "text": [ 1034 | "tensor([[ 0., 0.],\n", 1035 | " [ 0., 0.]])\n", 1036 | "tensor([[ 0., 0.],\n", 1037 | " [ 0., 0.]])\n" 1038 | ] 1039 | } 1040 | ], 1041 | "source": [ 1042 | "# Inplace\n", 1043 | "print(a.sub_(b))\n", 1044 | "print(a)" 1045 | ] 1046 | }, 1047 | { 1048 | "cell_type": "code", 1049 | "execution_count": 69, 1050 | "metadata": {}, 1051 | "outputs": [ 1052 | { 1053 | "name": "stdout", 1054 | "output_type": "stream", 1055 | "text": [ 1056 | "tensor([[ 1., 1.],\n", 1057 | " [ 1., 1.]])\n", 1058 | "tensor([[ 0., 0.],\n", 1059 | " [ 0., 0.]])\n" 1060 | ] 1061 | } 1062 | ], 1063 | "source": [ 1064 | "a = torch.ones(2, 2)\n", 1065 | "print(a)\n", 1066 | "b = torch.zeros(2, 2)\n", 1067 | "print(b)" 1068 | ] 1069 | }, 1070 | { 1071 | "cell_type": "code", 1072 | "execution_count": 70, 1073 | "metadata": {}, 1074 | "outputs": [ 1075 | { 1076 | "data": { 1077 | "text/plain": [ 1078 | "tensor([[ 0., 0.],\n", 1079 | " [ 0., 0.]])" 1080 | ] 1081 | }, 1082 | "execution_count": 70, 1083 | "metadata": {}, 1084 | "output_type": "execute_result" 1085 | } 1086 | ], 1087 | "source": [ 1088 | "a * b" 1089 | ] 1090 | }, 1091 | { 1092 | "cell_type": "code", 1093 | "execution_count": 71, 1094 | "metadata": {}, 1095 | "outputs": [ 1096 | { 1097 | "name": "stdout", 1098 | "output_type": "stream", 1099 | "text": [ 1100 | "tensor([[ 0., 0.],\n", 1101 | " [ 0., 0.]])\n", 1102 | "tensor([[ 1., 1.],\n", 1103 | " [ 1., 1.]])\n" 1104 | ] 1105 | } 1106 | ], 1107 | "source": [ 1108 | "# Not in-place\n", 1109 | "print(torch.mul(a, b))\n", 1110 | "print(a)" 1111 | ] 1112 | }, 1113 | { 1114 | "cell_type": "code", 1115 | "execution_count": 72, 1116 | "metadata": {}, 1117 | "outputs": [ 1118 | { 1119 | "name": "stdout", 1120 | "output_type": "stream", 1121 | "text": [ 1122 | "tensor([[ 0., 0.],\n", 1123 | " [ 0., 0.]])\n", 1124 | "tensor([[ 0., 0.],\n", 1125 | " [ 0., 0.]])\n" 1126 | ] 1127 | } 1128 | ], 1129 | "source": [ 1130 | "# In-place\n", 1131 | "print(a.mul_(b))\n", 1132 | "print(a)" 1133 | ] 1134 | }, 1135 | { 1136 | "cell_type": "code", 1137 | "execution_count": 73, 1138 | "metadata": {}, 1139 | "outputs": [], 1140 | "source": [ 1141 | "a = torch.ones(2, 2)\n", 1142 | "b = torch.zeros(2, 2)" 1143 | ] 1144 | }, 1145 | { 1146 | "cell_type": "code", 1147 | "execution_count": 74, 1148 | "metadata": {}, 1149 | "outputs": [ 1150 | { 1151 | "data": { 1152 | "text/plain": [ 1153 | "tensor([[ 0., 0.],\n", 1154 | " [ 0., 0.]])" 1155 | ] 1156 | }, 1157 | "execution_count": 74, 1158 | "metadata": {}, 1159 | "output_type": "execute_result" 1160 | } 1161 | ], 1162 | "source": [ 1163 | "b / a" 1164 | ] 1165 | }, 1166 | { 1167 | "cell_type": "code", 1168 | "execution_count": 75, 1169 | "metadata": {}, 1170 | "outputs": [ 1171 | { 1172 | "data": { 1173 | "text/plain": [ 1174 | "tensor([[ 0., 0.],\n", 1175 | " [ 0., 0.]])" 1176 | ] 1177 | }, 1178 | "execution_count": 75, 1179 | "metadata": {}, 1180 | "output_type": "execute_result" 1181 | } 1182 | ], 1183 | "source": [ 1184 | "torch.div(b, a)" 1185 | ] 1186 | }, 1187 | { 1188 | "cell_type": "code", 1189 | "execution_count": 76, 1190 | "metadata": {}, 1191 | "outputs": [ 1192 | { 1193 | "data": { 1194 | "text/plain": [ 1195 | "tensor([[ 0., 0.],\n", 1196 | " [ 0., 0.]])" 1197 | ] 1198 | }, 1199 | "execution_count": 76, 1200 | "metadata": {}, 1201 | "output_type": "execute_result" 1202 | } 1203 | ], 1204 | "source": [ 1205 | "# Inplace\n", 1206 | "b.div_(a)" 1207 | ] 1208 | }, 1209 | { 1210 | "cell_type": "code", 1211 | "execution_count": 77, 1212 | "metadata": {}, 1213 | "outputs": [ 1214 | { 1215 | "data": { 1216 | "text/plain": [ 1217 | "torch.Size([10])" 1218 | ] 1219 | }, 1220 | "execution_count": 77, 1221 | "metadata": {}, 1222 | "output_type": "execute_result" 1223 | } 1224 | ], 1225 | "source": [ 1226 | "a = torch.Tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n", 1227 | "a.size()" 1228 | ] 1229 | }, 1230 | { 1231 | "cell_type": "code", 1232 | "execution_count": 81, 1233 | "metadata": {}, 1234 | "outputs": [ 1235 | { 1236 | "data": { 1237 | "text/plain": [ 1238 | "tensor(5.5000)" 1239 | ] 1240 | }, 1241 | "execution_count": 81, 1242 | "metadata": {}, 1243 | "output_type": "execute_result" 1244 | } 1245 | ], 1246 | "source": [ 1247 | "a.mean(dim=0)" 1248 | ] 1249 | }, 1250 | { 1251 | "cell_type": "code", 1252 | "execution_count": 82, 1253 | "metadata": {}, 1254 | "outputs": [ 1255 | { 1256 | "ename": "RuntimeError", 1257 | "evalue": "dimension out of range (expected to be in range of [-1, 0], but got 1)", 1258 | "output_type": "error", 1259 | "traceback": [ 1260 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 1261 | "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", 1262 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 1263 | "\u001b[0;31mRuntimeError\u001b[0m: dimension out of range (expected to be in range of [-1, 0], but got 1)" 1264 | ] 1265 | } 1266 | ], 1267 | "source": [ 1268 | "a.mean(dim=1)" 1269 | ] 1270 | }, 1271 | { 1272 | "cell_type": "code", 1273 | "execution_count": 83, 1274 | "metadata": {}, 1275 | "outputs": [], 1276 | "source": [ 1277 | "a = torch.Tensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]])" 1278 | ] 1279 | }, 1280 | { 1281 | "cell_type": "code", 1282 | "execution_count": 84, 1283 | "metadata": {}, 1284 | "outputs": [ 1285 | { 1286 | "data": { 1287 | "text/plain": [ 1288 | "torch.Size([2, 10])" 1289 | ] 1290 | }, 1291 | "execution_count": 84, 1292 | "metadata": {}, 1293 | "output_type": "execute_result" 1294 | } 1295 | ], 1296 | "source": [ 1297 | "a.size()" 1298 | ] 1299 | }, 1300 | { 1301 | "cell_type": "code", 1302 | "execution_count": 87, 1303 | "metadata": {}, 1304 | "outputs": [ 1305 | { 1306 | "data": { 1307 | "text/plain": [ 1308 | "tensor([ 5.5000, 5.5000])" 1309 | ] 1310 | }, 1311 | "execution_count": 87, 1312 | "metadata": {}, 1313 | "output_type": "execute_result" 1314 | } 1315 | ], 1316 | "source": [ 1317 | "a.mean(dim=1)" 1318 | ] 1319 | }, 1320 | { 1321 | "cell_type": "code", 1322 | "execution_count": 88, 1323 | "metadata": {}, 1324 | "outputs": [ 1325 | { 1326 | "data": { 1327 | "text/plain": [ 1328 | "tensor(3.0277)" 1329 | ] 1330 | }, 1331 | "execution_count": 88, 1332 | "metadata": {}, 1333 | "output_type": "execute_result" 1334 | } 1335 | ], 1336 | "source": [ 1337 | "a = torch.Tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n", 1338 | "a.std(dim=0)" 1339 | ] 1340 | }, 1341 | { 1342 | "cell_type": "code", 1343 | "execution_count": null, 1344 | "metadata": {}, 1345 | "outputs": [], 1346 | "source": [] 1347 | } 1348 | ], 1349 | "metadata": { 1350 | "kernelspec": { 1351 | "display_name": "Python 3", 1352 | "language": "python", 1353 | "name": "python3" 1354 | }, 1355 | "language_info": { 1356 | "codemirror_mode": { 1357 | "name": "ipython", 1358 | "version": 3 1359 | }, 1360 | "file_extension": ".py", 1361 | "mimetype": "text/x-python", 1362 | "name": "python", 1363 | "nbconvert_exporter": "python", 1364 | "pygments_lexer": "ipython3", 1365 | "version": "3.6.5" 1366 | } 1367 | }, 1368 | "nbformat": 4, 1369 | "nbformat_minor": 2 1370 | } 1371 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/04. PyTorch Fundamentals - Variables and Gradients-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Variables" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import torch\n", 17 | "from torch.autograd import Variable" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [ 25 | { 26 | "data": { 27 | "text/plain": [ 28 | "tensor([[ 1., 1.],\n", 29 | " [ 1., 1.]])" 30 | ] 31 | }, 32 | "execution_count": 2, 33 | "metadata": {}, 34 | "output_type": "execute_result" 35 | } 36 | ], 37 | "source": [ 38 | "a = Variable(torch.ones(2, 2), requires_grad=True)\n", 39 | "a" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 3, 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/plain": [ 50 | "tensor([[ 1., 1., 1.],\n", 51 | " [ 1., 1., 1.]])" 52 | ] 53 | }, 54 | "execution_count": 3, 55 | "metadata": {}, 56 | "output_type": "execute_result" 57 | } 58 | ], 59 | "source": [ 60 | "# Not a variable\n", 61 | "torch.ones(2, 3)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 4, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "name": "stdout", 71 | "output_type": "stream", 72 | "text": [ 73 | "tensor([[ 2., 2.],\n", 74 | " [ 2., 2.]])\n", 75 | "tensor([[ 2., 2.],\n", 76 | " [ 2., 2.]])\n" 77 | ] 78 | } 79 | ], 80 | "source": [ 81 | "# Behaves similarly to tensors\n", 82 | "b = Variable(torch.ones(2, 2), requires_grad=True)\n", 83 | "print(a + b)\n", 84 | "print(torch.add(a, b))" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 5, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "name": "stdout", 94 | "output_type": "stream", 95 | "text": [ 96 | "tensor([[ 1., 1.],\n", 97 | " [ 1., 1.]])\n", 98 | "tensor([[ 1., 1.],\n", 99 | " [ 1., 1.]])\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "print(a * b)\n", 105 | "print(torch.mul(a, b))" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "# Gradients" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 17, 118 | "metadata": {}, 119 | "outputs": [ 120 | { 121 | "data": { 122 | "text/plain": [ 123 | "tensor([ 1., 1.])" 124 | ] 125 | }, 126 | "execution_count": 17, 127 | "metadata": {}, 128 | "output_type": "execute_result" 129 | } 130 | ], 131 | "source": [ 132 | "x = Variable(torch.ones(2), requires_grad=True)\n", 133 | "x" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 18, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "data": { 143 | "text/plain": [ 144 | "tensor([ 20., 20.])" 145 | ] 146 | }, 147 | "execution_count": 18, 148 | "metadata": {}, 149 | "output_type": "execute_result" 150 | } 151 | ], 152 | "source": [ 153 | "y = 5 * (x + 1) ** 2\n", 154 | "y" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": 19, 160 | "metadata": {}, 161 | "outputs": [ 162 | { 163 | "data": { 164 | "text/plain": [ 165 | "tensor(20.)" 166 | ] 167 | }, 168 | "execution_count": 19, 169 | "metadata": {}, 170 | "output_type": "execute_result" 171 | } 172 | ], 173 | "source": [ 174 | "o = (1/2) * torch.sum(y)\n", 175 | "o" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 13, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "o.backward()" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 14, 190 | "metadata": {}, 191 | "outputs": [ 192 | { 193 | "data": { 194 | "text/plain": [ 195 | "tensor([ 10., 10.])" 196 | ] 197 | }, 198 | "execution_count": 14, 199 | "metadata": {}, 200 | "output_type": "execute_result" 201 | } 202 | ], 203 | "source": [ 204 | "x.grad" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 20, 210 | "metadata": {}, 211 | "outputs": [ 212 | { 213 | "data": { 214 | "text/plain": [ 215 | "tensor([ 10., 10.])" 216 | ] 217 | }, 218 | "execution_count": 20, 219 | "metadata": {}, 220 | "output_type": "execute_result" 221 | } 222 | ], 223 | "source": [ 224 | "# backward in detail\n", 225 | "o.backward(torch.FloatTensor([1.0, 1.0]))\n", 226 | "x.grad" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": null, 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [] 235 | } 236 | ], 237 | "metadata": { 238 | "kernelspec": { 239 | "display_name": "Python 3", 240 | "language": "python", 241 | "name": "python3" 242 | }, 243 | "language_info": { 244 | "codemirror_mode": { 245 | "name": "ipython", 246 | "version": 3 247 | }, 248 | "file_extension": ".py", 249 | "mimetype": "text/x-python", 250 | "name": "python", 251 | "nbconvert_exporter": "python", 252 | "pygments_lexer": "ipython3", 253 | "version": "3.6.5" 254 | } 255 | }, 256 | "nbformat": 4, 257 | "nbformat_minor": 2 258 | } 259 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/07. Feedforward Neural Network with PyTorch-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 2 6 | } 7 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/08. Convolutional Neural Network (CNN) with PyTorch-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# CNN in PyTorch" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import torch\n", 17 | "import torch.nn as nn\n", 18 | "import torchvision.transforms as transforms\n", 19 | "import torchvision.datasets as dsets\n", 20 | "from torch.autograd import Variable" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "train_dataset = dsets.MNIST(root='./data',\n", 30 | " train=True,\n", 31 | " transform=transforms.ToTensor(),\n", 32 | " download=True)\n", 33 | "\n", 34 | "test_dataset = dsets.MNIST(root='./data',\n", 35 | " train=False,\n", 36 | " transform=transforms.ToTensor())" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 3, 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "torch.Size([60000, 28, 28])\n" 49 | ] 50 | } 51 | ], 52 | "source": [ 53 | "print(train_dataset.train_data.size())" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 4, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "name": "stdout", 63 | "output_type": "stream", 64 | "text": [ 65 | "torch.Size([60000])\n" 66 | ] 67 | } 68 | ], 69 | "source": [ 70 | "print(train_dataset.train_labels.size())" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 5, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "torch.Size([10000, 28, 28])\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "print(test_dataset.test_data.size())" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 6, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "torch.Size([10000])\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "print(test_dataset.test_labels.size())" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 7, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "batch_size = 100\n", 114 | "n_iters = 3000\n", 115 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 116 | "num_epochs = int(num_epochs)\n", 117 | "\n", 118 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 119 | " batch_size=batch_size,\n", 120 | " shuffle=True)\n", 121 | "\n", 122 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 123 | " batch_size=batch_size,\n", 124 | " shuffle=False)" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 10, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "class CNNModel(nn.Module):\n", 134 | " def __init__(self):\n", 135 | " super(CNNModel, self).__init__()\n", 136 | " \n", 137 | " # Convolution 1\n", 138 | " self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=2)\n", 139 | " self.relu1 = nn.ReLU()\n", 140 | " \n", 141 | " # Max pool 1\n", 142 | " self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n", 143 | " \n", 144 | " # Convolution 2\n", 145 | " self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)\n", 146 | " self.relu2 = nn.ReLU()\n", 147 | " \n", 148 | " # Max pool 2\n", 149 | " self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n", 150 | " \n", 151 | " # Fully connected 1 (readout)\n", 152 | " self.fc1 = nn.Linear(32 * 7 * 7, 10)\n", 153 | " \n", 154 | " def forward(self, x):\n", 155 | " # Convolution 1\n", 156 | " out = self.cnn1(x)\n", 157 | " out = self.relu1(out)\n", 158 | " \n", 159 | " # Max pool 1\n", 160 | " out = self.maxpool1(out)\n", 161 | " \n", 162 | " # Convolution 2\n", 163 | " out = self.cnn2(out)\n", 164 | " out = self.relu2(out)\n", 165 | " \n", 166 | " # Max pool 2\n", 167 | " out = self.maxpool2(out)\n", 168 | " \n", 169 | " # Resize\n", 170 | " # Original size: (100, 32, 7, 7)\n", 171 | " # out.size(0): 100\n", 172 | " # New out size: (100, 32*7*7)\n", 173 | " out = out.view(out.size(0), -1)\n", 174 | " \n", 175 | " # Linear function (readout)\n", 176 | " out = self.fc1(out)\n", 177 | " \n", 178 | " return out" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 11, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "model = CNNModel()" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 12, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "criterion = nn.CrossEntropyLoss()" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 13, 202 | "metadata": {}, 203 | "outputs": [], 204 | "source": [ 205 | "learning_rate = 0.01\n", 206 | "\n", 207 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 14, 213 | "metadata": {}, 214 | "outputs": [ 215 | { 216 | "name": "stdout", 217 | "output_type": "stream", 218 | "text": [ 219 | "\n", 220 | "6\n", 221 | "torch.Size([16, 1, 5, 5])\n", 222 | "torch.Size([16])\n", 223 | "torch.Size([32, 16, 5, 5])\n", 224 | "torch.Size([32])\n", 225 | "torch.Size([10, 1568])\n", 226 | "torch.Size([10])\n" 227 | ] 228 | } 229 | ], 230 | "source": [ 231 | "print(model.parameters())\n", 232 | "\n", 233 | "print(len(list(model.parameters())))\n", 234 | "\n", 235 | "# Convolution 1: 16 Kernels\n", 236 | "print(list(model.parameters())[0].size())\n", 237 | "\n", 238 | "# Convolution 1 Bias: 16 Kernels\n", 239 | "print(list(model.parameters())[1].size())\n", 240 | "\n", 241 | "# Convolution 2: 32 Kernels with depth = 16\n", 242 | "print(list(model.parameters())[2].size())\n", 243 | "\n", 244 | "# Convolution 2 Bias: 32 Kernels with depth = 16\n", 245 | "print(list(model.parameters())[3].size())\n", 246 | "\n", 247 | "# Fully Connected Layer 1\n", 248 | "print(list(model.parameters())[4].size())\n", 249 | "\n", 250 | "# Fully Connected Layer Bias\n", 251 | "print(list(model.parameters())[5].size())" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": 15, 257 | "metadata": {}, 258 | "outputs": [ 259 | { 260 | "name": "stderr", 261 | "output_type": "stream", 262 | "text": [ 263 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:49: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 264 | ] 265 | }, 266 | { 267 | "name": "stdout", 268 | "output_type": "stream", 269 | "text": [ 270 | "Iteration: 500. Loss: 0.22800475358963013. Accuracy: 91\n", 271 | "Iteration: 1000. Loss: 0.16966554522514343. Accuracy: 93\n", 272 | "Iteration: 1500. Loss: 0.12278220057487488. Accuracy: 94\n", 273 | "Iteration: 2000. Loss: 0.21044640243053436. Accuracy: 95\n", 274 | "Iteration: 2500. Loss: 0.09591661393642426. Accuracy: 96\n", 275 | "Iteration: 3000. Loss: 0.08583936095237732. Accuracy: 96\n" 276 | ] 277 | } 278 | ], 279 | "source": [ 280 | "iter = 0\n", 281 | "for epoch in range(num_epochs):\n", 282 | " for i, (images, labels) in enumerate(train_loader):\n", 283 | " # Load images as Variable\n", 284 | " images = Variable(images)\n", 285 | " labels = Variable(labels)\n", 286 | " \n", 287 | " # Clear gradients w.r.t. parameters\n", 288 | " optimizer.zero_grad()\n", 289 | " \n", 290 | " # Forward pass to get output/logits\n", 291 | " outputs = model(images)\n", 292 | " \n", 293 | " # Calculate Loss: softmax --> cross entropy loss\n", 294 | " loss = criterion(outputs, labels)\n", 295 | " \n", 296 | " # Getting gradients w.r.t. parameters\n", 297 | " loss.backward()\n", 298 | " \n", 299 | " # Updating parameters\n", 300 | " optimizer.step()\n", 301 | " \n", 302 | " iter += 1\n", 303 | " \n", 304 | " if iter % 500 == 0:\n", 305 | " # Calculate Accuracy\n", 306 | " correct = 0\n", 307 | " total = 0\n", 308 | " # Iterate through test dataset\n", 309 | " for images, labels in test_loader:\n", 310 | " # Load images to a Torch Variable\n", 311 | " images = Variable(images)\n", 312 | " \n", 313 | " # Forward pass only to get logits/output\n", 314 | " outputs = model(images)\n", 315 | " \n", 316 | " # Get predictions from the maximum value\n", 317 | " _, predicted = torch.max(outputs.data, 1)\n", 318 | " \n", 319 | " # Total number of labels\n", 320 | " total += labels.size(0)\n", 321 | " \n", 322 | " # Total correct predictions\n", 323 | " correct += (predicted == labels).sum()\n", 324 | " \n", 325 | " accuracy = 100 * correct / total\n", 326 | " \n", 327 | " # Print Loss\n", 328 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 329 | ] 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "metadata": {}, 334 | "source": [ 335 | "# More CNN Models in PyTorch" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": 19, 341 | "metadata": {}, 342 | "outputs": [ 343 | { 344 | "name": "stderr", 345 | "output_type": "stream", 346 | "text": [ 347 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:157: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 348 | ] 349 | }, 350 | { 351 | "name": "stdout", 352 | "output_type": "stream", 353 | "text": [ 354 | "Iteration: 500. Loss: 0.5118485689163208. Accuracy: 86\n", 355 | "Iteration: 1000. Loss: 0.29064464569091797. Accuracy: 89\n", 356 | "Iteration: 1500. Loss: 0.373428076505661. Accuracy: 90\n", 357 | "Iteration: 2000. Loss: 0.448984831571579. Accuracy: 91\n", 358 | "Iteration: 2500. Loss: 0.20878082513809204. Accuracy: 91\n", 359 | "Iteration: 3000. Loss: 0.16859890520572662. Accuracy: 92\n" 360 | ] 361 | } 362 | ], 363 | "source": [ 364 | "import torch\n", 365 | "import torch.nn as nn\n", 366 | "import torchvision.transforms as transforms\n", 367 | "import torchvision.datasets as dsets\n", 368 | "from torch.autograd import Variable\n", 369 | "\n", 370 | "'''\n", 371 | "STEP 1: LOADING DATASET\n", 372 | "'''\n", 373 | "\n", 374 | "train_dataset = dsets.MNIST(root='./data',\n", 375 | " train=True,\n", 376 | " transform=transforms.ToTensor(),\n", 377 | " download=True)\n", 378 | "\n", 379 | "test_dataset = dsets.MNIST(root='./data',\n", 380 | " train=False,\n", 381 | " transform=transforms.ToTensor())\n", 382 | "\n", 383 | "'''\n", 384 | "STEP 2: MAKING DATASET ITERABLE\n", 385 | "'''\n", 386 | "\n", 387 | "batch_size = 100\n", 388 | "n_iters = 3000\n", 389 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 390 | "num_epochs = int(num_epochs)\n", 391 | "\n", 392 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 393 | " batch_size=batch_size,\n", 394 | " shuffle=True)\n", 395 | "\n", 396 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 397 | " batch_size=batch_size,\n", 398 | " shuffle=False)\n", 399 | "\n", 400 | "'''\n", 401 | "STEP 3: CREATE MODEL CLASS\n", 402 | "'''\n", 403 | "class CNNModel(nn.Module):\n", 404 | " def __init__(self):\n", 405 | " super(CNNModel, self).__init__()\n", 406 | " \n", 407 | " # Convolution 1\n", 408 | " self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=2)\n", 409 | " self.relu1 = nn.ReLU()\n", 410 | " \n", 411 | " # Average pool 1\n", 412 | " self.avgpool1 = nn.AvgPool2d(kernel_size=2)\n", 413 | " \n", 414 | " # Convolution 2\n", 415 | " self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)\n", 416 | " self.relu2 = nn.ReLU()\n", 417 | " \n", 418 | " # Average pool 2\n", 419 | " self.avgpool2 = nn.AvgPool2d(kernel_size=2)\n", 420 | " \n", 421 | " # Fully connected 1 (readout)\n", 422 | " self.fc1 = nn.Linear(32 * 7 * 7, 10)\n", 423 | " \n", 424 | " def forward(self, x):\n", 425 | " # Convolution 1\n", 426 | " out = self.cnn1(x)\n", 427 | " out = self.relu1(out)\n", 428 | " \n", 429 | " # Average pool 1\n", 430 | " out = self.avgpool1(out)\n", 431 | " \n", 432 | " # Convolution 2\n", 433 | " out = self.cnn2(out)\n", 434 | " out = self.relu2(out)\n", 435 | " \n", 436 | " # Max pool 2\n", 437 | " out = self.avgpool2(out)\n", 438 | " \n", 439 | " # Resize\n", 440 | " # Original size: (100, 32, 7, 7)\n", 441 | " # out.size(0): 100\n", 442 | " # New out size: (100, 32*7*7)\n", 443 | " out = out.view(out.size(0), -1)\n", 444 | " \n", 445 | " # Linear function (readout)\n", 446 | " out = self.fc1(out)\n", 447 | " \n", 448 | " return out\n", 449 | " \n", 450 | "'''\n", 451 | "STEP 4: INSTANTIATE MODEL CLASS\n", 452 | "'''\n", 453 | "\n", 454 | "model = CNNModel()\n", 455 | "\n", 456 | "'''\n", 457 | "STEP 5: INSTANTIATE LOSS CLASS\n", 458 | "'''\n", 459 | "criterion = nn.CrossEntropyLoss()\n", 460 | "\n", 461 | "\n", 462 | "'''\n", 463 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 464 | "'''\n", 465 | "learning_rate = 0.01\n", 466 | "\n", 467 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 468 | "\n", 469 | "'''\n", 470 | "STEP 7: TRAIN THE MODEL\n", 471 | "'''\n", 472 | "iter = 0\n", 473 | "for epoch in range(num_epochs):\n", 474 | " for i, (images, labels) in enumerate(train_loader):\n", 475 | " # Load images as Variable\n", 476 | " images = Variable(images)\n", 477 | " labels = Variable(labels)\n", 478 | " \n", 479 | " # Clear gradients w.r.t. parameters\n", 480 | " optimizer.zero_grad()\n", 481 | " \n", 482 | " # Forward pass to get output/logits\n", 483 | " outputs = model(images)\n", 484 | " \n", 485 | " # Calculate Loss: softmax --> cross entropy loss\n", 486 | " loss = criterion(outputs, labels)\n", 487 | " \n", 488 | " # Getting gradients w.r.t. parameters\n", 489 | " loss.backward()\n", 490 | " \n", 491 | " # Updating parameters\n", 492 | " optimizer.step()\n", 493 | " \n", 494 | " iter += 1\n", 495 | " \n", 496 | " if iter % 500 == 0:\n", 497 | " # Calculate Accuracy\n", 498 | " correct = 0\n", 499 | " total = 0\n", 500 | " # Iterate through test dataset\n", 501 | " for images, labels in test_loader:\n", 502 | " # Load images to a Torch Variable\n", 503 | " images = Variable(images)\n", 504 | " \n", 505 | " # Forward pass only to get logits/output\n", 506 | " outputs = model(images)\n", 507 | " \n", 508 | " # Get predictions from the maximum value\n", 509 | " _, predicted = torch.max(outputs.data, 1)\n", 510 | " \n", 511 | " # Total number of labels\n", 512 | " total += labels.size(0)\n", 513 | " \n", 514 | " # Total correct predictions\n", 515 | " correct += (predicted == labels).sum()\n", 516 | " \n", 517 | " accuracy = 100 * correct / total\n", 518 | " \n", 519 | " # Print Loss\n", 520 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 521 | ] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "execution_count": 20, 526 | "metadata": {}, 527 | "outputs": [ 528 | { 529 | "name": "stderr", 530 | "output_type": "stream", 531 | "text": [ 532 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:157: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 533 | ] 534 | }, 535 | { 536 | "name": "stdout", 537 | "output_type": "stream", 538 | "text": [ 539 | "Iteration: 500. Loss: 0.23876222968101501. Accuracy: 91\n", 540 | "Iteration: 1000. Loss: 0.22581394016742706. Accuracy: 93\n", 541 | "Iteration: 1500. Loss: 0.18389980494976044. Accuracy: 95\n", 542 | "Iteration: 2000. Loss: 0.09902214258909225. Accuracy: 96\n", 543 | "Iteration: 2500. Loss: 0.14246611297130585. Accuracy: 96\n", 544 | "Iteration: 3000. Loss: 0.17715296149253845. Accuracy: 96\n" 545 | ] 546 | } 547 | ], 548 | "source": [ 549 | "import torch\n", 550 | "import torch.nn as nn\n", 551 | "import torchvision.transforms as transforms\n", 552 | "import torchvision.datasets as dsets\n", 553 | "from torch.autograd import Variable\n", 554 | "\n", 555 | "'''\n", 556 | "STEP 1: LOADING DATASET\n", 557 | "'''\n", 558 | "\n", 559 | "train_dataset = dsets.MNIST(root='./data',\n", 560 | " train=True,\n", 561 | " transform=transforms.ToTensor(),\n", 562 | " download=True)\n", 563 | "\n", 564 | "test_dataset = dsets.MNIST(root='./data',\n", 565 | " train=False,\n", 566 | " transform=transforms.ToTensor())\n", 567 | "\n", 568 | "'''\n", 569 | "STEP 2: MAKING DATASET ITERABLE\n", 570 | "'''\n", 571 | "\n", 572 | "batch_size = 100\n", 573 | "n_iters = 3000\n", 574 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 575 | "num_epochs = int(num_epochs)\n", 576 | "\n", 577 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 578 | " batch_size=batch_size,\n", 579 | " shuffle=True)\n", 580 | "\n", 581 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 582 | " batch_size=batch_size,\n", 583 | " shuffle=False)\n", 584 | "\n", 585 | "'''\n", 586 | "STEP 3: CREATE MODEL CLASS\n", 587 | "'''\n", 588 | "class CNNModel(nn.Module):\n", 589 | " def __init__(self):\n", 590 | " super(CNNModel, self).__init__()\n", 591 | " \n", 592 | " # Convolution 1\n", 593 | " self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)\n", 594 | " self.relu1 = nn.ReLU()\n", 595 | " \n", 596 | " # Max pool 1\n", 597 | " self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n", 598 | " \n", 599 | " # Convolution 2\n", 600 | " self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)\n", 601 | " self.relu2 = nn.ReLU()\n", 602 | " \n", 603 | " # Max pool 2\n", 604 | " self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n", 605 | " \n", 606 | " # Fully connected 1 (readout)\n", 607 | " self.fc1 = nn.Linear(32 * 4 * 4, 10)\n", 608 | " \n", 609 | " def forward(self, x):\n", 610 | " # Convolution 1\n", 611 | " out = self.cnn1(x)\n", 612 | " out = self.relu1(out)\n", 613 | " \n", 614 | " # Max pool 1\n", 615 | " out = self.maxpool1(out)\n", 616 | " \n", 617 | " # Convolution 2\n", 618 | " out = self.cnn2(out)\n", 619 | " out = self.relu2(out)\n", 620 | " \n", 621 | " # Max pool 2\n", 622 | " out = self.maxpool2(out)\n", 623 | " \n", 624 | " # Resize\n", 625 | " # Original size: (100, 32, 7, 7)\n", 626 | " # out.size(0): 100\n", 627 | " # New out size: (100, 32*7*7)\n", 628 | " out = out.view(out.size(0), -1)\n", 629 | " \n", 630 | " # Linear function (readout)\n", 631 | " out = self.fc1(out)\n", 632 | " \n", 633 | " return out\n", 634 | "\n", 635 | "'''\n", 636 | "STEP 4: INSTANTIATE MODEL CLASS\n", 637 | "'''\n", 638 | "\n", 639 | "model = CNNModel()\n", 640 | "\n", 641 | "'''\n", 642 | "STEP 5: INSTANTIATE LOSS CLASS\n", 643 | "'''\n", 644 | "criterion = nn.CrossEntropyLoss()\n", 645 | "\n", 646 | "\n", 647 | "'''\n", 648 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 649 | "'''\n", 650 | "learning_rate = 0.01\n", 651 | "\n", 652 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 653 | "\n", 654 | "'''\n", 655 | "STEP 7: TRAIN THE MODEL\n", 656 | "'''\n", 657 | "iter = 0\n", 658 | "for epoch in range(num_epochs):\n", 659 | " for i, (images, labels) in enumerate(train_loader):\n", 660 | " # Load images as Variable\n", 661 | " images = Variable(images)\n", 662 | " labels = Variable(labels)\n", 663 | " \n", 664 | " # Clear gradients w.r.t. parameters\n", 665 | " optimizer.zero_grad()\n", 666 | " \n", 667 | " # Forward pass to get output/logits\n", 668 | " outputs = model(images)\n", 669 | " \n", 670 | " # Calculate Loss: softmax --> cross entropy loss\n", 671 | " loss = criterion(outputs, labels)\n", 672 | " \n", 673 | " # Getting gradients w.r.t. parameters\n", 674 | " loss.backward()\n", 675 | " \n", 676 | " # Updating parameters\n", 677 | " optimizer.step()\n", 678 | " \n", 679 | " iter += 1\n", 680 | " \n", 681 | " if iter % 500 == 0:\n", 682 | " # Calculate Accuracy\n", 683 | " correct = 0\n", 684 | " total = 0\n", 685 | " # Iterate through test dataset\n", 686 | " for images, labels in test_loader:\n", 687 | " # Load images to a Torch Variable\n", 688 | " images = Variable(images)\n", 689 | " \n", 690 | " # Forward pass only to get logits/output\n", 691 | " outputs = model(images)\n", 692 | " \n", 693 | " # Get predictions from the maximum value\n", 694 | " _, predicted = torch.max(outputs.data, 1)\n", 695 | " \n", 696 | " # Total number of labels\n", 697 | " total += labels.size(0)\n", 698 | " \n", 699 | " # Total correct predictions\n", 700 | " correct += (predicted == labels).sum()\n", 701 | " \n", 702 | " accuracy = 100 * correct / total\n", 703 | " \n", 704 | " # Print Loss\n", 705 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 706 | ] 707 | }, 708 | { 709 | "cell_type": "code", 710 | "execution_count": null, 711 | "metadata": {}, 712 | "outputs": [], 713 | "source": [] 714 | } 715 | ], 716 | "metadata": { 717 | "kernelspec": { 718 | "display_name": "Python 3", 719 | "language": "python", 720 | "name": "python3" 721 | }, 722 | "language_info": { 723 | "codemirror_mode": { 724 | "name": "ipython", 725 | "version": 3 726 | }, 727 | "file_extension": ".py", 728 | "mimetype": "text/x-python", 729 | "name": "python", 730 | "nbconvert_exporter": "python", 731 | "pygments_lexer": "ipython3", 732 | "version": "3.6.5" 733 | } 734 | }, 735 | "nbformat": 4, 736 | "nbformat_minor": 2 737 | } 738 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/10. Long Short-Term Memory Networks (LSTM)-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# LSTM in PyTorch" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import torch\n", 17 | "import torch.nn as nn\n", 18 | "import torchvision.transforms as transforms\n", 19 | "import torchvision.datasets as dsets\n", 20 | "from torch.autograd import Variable" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "train_dataset = dsets.MNIST(root='./data',\n", 30 | " train=True,\n", 31 | " transform=transforms.ToTensor(),\n", 32 | " download=True)\n", 33 | "\n", 34 | "test_dataset = dsets.MNIST(root='./data',\n", 35 | " train=False,\n", 36 | " transform=transforms.ToTensor())" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 3, 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "torch.Size([60000, 28, 28])\n" 49 | ] 50 | } 51 | ], 52 | "source": [ 53 | "print(train_dataset.train_data.size())" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 4, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "name": "stdout", 63 | "output_type": "stream", 64 | "text": [ 65 | "torch.Size([60000])\n" 66 | ] 67 | } 68 | ], 69 | "source": [ 70 | "print(train_dataset.train_labels.size())" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 5, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "torch.Size([10000, 28, 28])\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "print(test_dataset.test_data.size())" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 6, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "torch.Size([10000])\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "print(test_dataset.test_labels.size())" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 7, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "batch_size = 100\n", 114 | "n_iters = 3000\n", 115 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 116 | "num_epochs = int(num_epochs)\n", 117 | "\n", 118 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 119 | " batch_size=batch_size,\n", 120 | " shuffle=True)\n", 121 | "\n", 122 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 123 | " batch_size=batch_size,\n", 124 | " shuffle=False)" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 19, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "class LSTMModel(nn.Module):\n", 134 | " def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n", 135 | " super(LSTMModel, self).__init__()\n", 136 | " # Hidden dimensions\n", 137 | " self.hidden_dim = hidden_dim\n", 138 | " \n", 139 | " # Number of hidden layers\n", 140 | " self.layer_dim = layer_dim\n", 141 | " \n", 142 | " # Building your LSTM\n", 143 | " # batch_first=True causes input/output tensors to be of shape\n", 144 | " # (batch_dim, seq_dim, feature_dim)\n", 145 | " self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n", 146 | " \n", 147 | " # Readout layer\n", 148 | " self.fc = nn.Linear(hidden_dim, output_dim)\n", 149 | " \n", 150 | " def forward(self, x):\n", 151 | " # Initialize hidden state with zeros\n", 152 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 153 | " \n", 154 | " # Initialize cell state\n", 155 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 156 | " \n", 157 | " # 28 time steps\n", 158 | " out, (hn, cn) = self.lstm(x, (h0,c0))\n", 159 | " \n", 160 | " # Index hidden state of last time step\n", 161 | " # out.size() --> 100, 28, 100\n", 162 | " # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!\n", 163 | " out = self.fc(out[:, -1, :])\n", 164 | " # out.size() --> 100, 10\n", 165 | " return out" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 20, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "input_dim = 28\n", 175 | "hidden_dim = 100\n", 176 | "layer_dim = 1\n", 177 | "output_dim = 10" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 21, 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 22, 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "criterion = nn.CrossEntropyLoss()" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 23, 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "learning_rate = 0.1\n", 205 | "\n", 206 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 24, 212 | "metadata": {}, 213 | "outputs": [ 214 | { 215 | "data": { 216 | "text/plain": [ 217 | "6" 218 | ] 219 | }, 220 | "execution_count": 24, 221 | "metadata": {}, 222 | "output_type": "execute_result" 223 | } 224 | ], 225 | "source": [ 226 | "len(list(model.parameters()))" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": 25, 232 | "metadata": {}, 233 | "outputs": [ 234 | { 235 | "name": "stdout", 236 | "output_type": "stream", 237 | "text": [ 238 | "torch.Size([400, 28])\n", 239 | "torch.Size([400, 100])\n", 240 | "torch.Size([400])\n", 241 | "torch.Size([400])\n", 242 | "torch.Size([10, 100])\n", 243 | "torch.Size([10])\n" 244 | ] 245 | } 246 | ], 247 | "source": [ 248 | "for i in range(len(list(model.parameters()))):\n", 249 | " print(list(model.parameters())[i].size())" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 26, 255 | "metadata": {}, 256 | "outputs": [ 257 | { 258 | "name": "stderr", 259 | "output_type": "stream", 260 | "text": [ 261 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:53: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 262 | ] 263 | }, 264 | { 265 | "name": "stdout", 266 | "output_type": "stream", 267 | "text": [ 268 | "Iteration: 500. Loss: 2.2820582389831543. Accuracy: 19\n", 269 | "Iteration: 1000. Loss: 1.035079836845398. Accuracy: 64\n", 270 | "Iteration: 1500. Loss: 0.5265297889709473. Accuracy: 83\n", 271 | "Iteration: 2000. Loss: 0.24867504835128784. Accuracy: 92\n", 272 | "Iteration: 2500. Loss: 0.3007533550262451. Accuracy: 93\n", 273 | "Iteration: 3000. Loss: 0.13653035461902618. Accuracy: 95\n" 274 | ] 275 | } 276 | ], 277 | "source": [ 278 | "# Number of steps to unroll\n", 279 | "seq_dim = 28\n", 280 | "\n", 281 | "iter = 0\n", 282 | "for epoch in range(num_epochs):\n", 283 | " for i, (images, labels) in enumerate(train_loader):\n", 284 | " # Load images as Variable\n", 285 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 286 | " labels = Variable(labels)\n", 287 | " \n", 288 | " # Clear gradients w.r.t. parameters\n", 289 | " optimizer.zero_grad()\n", 290 | " \n", 291 | " # Forward pass to get output/logits\n", 292 | " # outputs.size() --> 100, 10\n", 293 | " outputs = model(images)\n", 294 | " \n", 295 | " # Calculate Loss: softmax --> cross entropy loss\n", 296 | " loss = criterion(outputs, labels)\n", 297 | " \n", 298 | " # Getting gradients w.r.t. parameters\n", 299 | " loss.backward()\n", 300 | " \n", 301 | " # Updating parameters\n", 302 | " optimizer.step()\n", 303 | " \n", 304 | " iter += 1\n", 305 | " \n", 306 | " if iter % 500 == 0:\n", 307 | " # Calculate Accuracy\n", 308 | " correct = 0\n", 309 | " total = 0\n", 310 | " # Iterate through test dataset\n", 311 | " for images, labels in test_loader:\n", 312 | " # Load images to a Torch Variable\n", 313 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 314 | " \n", 315 | " # Forward pass only to get logits/output\n", 316 | " outputs = model(images)\n", 317 | " \n", 318 | " # Get predictions from the maximum value\n", 319 | " _, predicted = torch.max(outputs.data, 1)\n", 320 | " \n", 321 | " # Total number of labels\n", 322 | " total += labels.size(0)\n", 323 | " \n", 324 | " # Total correct predictions\n", 325 | " correct += (predicted == labels).sum()\n", 326 | " \n", 327 | " accuracy = 100 * correct / total\n", 328 | " \n", 329 | " # Print Loss\n", 330 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 331 | ] 332 | }, 333 | { 334 | "cell_type": "markdown", 335 | "metadata": {}, 336 | "source": [ 337 | "# More LSTM Models in PyTorch" 338 | ] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": 27, 343 | "metadata": {}, 344 | "outputs": [ 345 | { 346 | "name": "stdout", 347 | "output_type": "stream", 348 | "text": [ 349 | "LSTMModel(\n", 350 | " (lstm): LSTM(28, 100, num_layers=2, batch_first=True)\n", 351 | " (fc): Linear(in_features=100, out_features=10, bias=True)\n", 352 | ")\n", 353 | "10\n", 354 | "torch.Size([400, 28])\n", 355 | "torch.Size([400, 100])\n", 356 | "torch.Size([400])\n", 357 | "torch.Size([400])\n", 358 | "torch.Size([400, 100])\n", 359 | "torch.Size([400, 100])\n", 360 | "torch.Size([400])\n", 361 | "torch.Size([400])\n", 362 | "torch.Size([10, 100])\n", 363 | "torch.Size([10])\n" 364 | ] 365 | }, 366 | { 367 | "name": "stderr", 368 | "output_type": "stream", 369 | "text": [ 370 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:158: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 371 | ] 372 | }, 373 | { 374 | "name": "stdout", 375 | "output_type": "stream", 376 | "text": [ 377 | "Iteration: 500. Loss: 2.2961795330047607. Accuracy: 11\n", 378 | "Iteration: 1000. Loss: 2.0218091011047363. Accuracy: 24\n", 379 | "Iteration: 1500. Loss: 0.77486252784729. Accuracy: 78\n", 380 | "Iteration: 2000. Loss: 0.22417789697647095. Accuracy: 91\n", 381 | "Iteration: 2500. Loss: 0.29147088527679443. Accuracy: 94\n", 382 | "Iteration: 3000. Loss: 0.11946485191583633. Accuracy: 95\n" 383 | ] 384 | } 385 | ], 386 | "source": [ 387 | "import torch\n", 388 | "import torch.nn as nn\n", 389 | "import torchvision.transforms as transforms\n", 390 | "import torchvision.datasets as dsets\n", 391 | "from torch.autograd import Variable\n", 392 | "\n", 393 | "'''\n", 394 | "STEP 1: LOADING DATASET\n", 395 | "'''\n", 396 | "train_dataset = dsets.MNIST(root='./data',\n", 397 | " train=True,\n", 398 | " transform=transforms.ToTensor(),\n", 399 | " download=True)\n", 400 | "\n", 401 | "test_dataset = dsets.MNIST(root='./data',\n", 402 | " train=False,\n", 403 | " transform=transforms.ToTensor())\n", 404 | "\n", 405 | "'''\n", 406 | "STEP 2: MAKING DATASET ITERABLE\n", 407 | "'''\n", 408 | "\n", 409 | "batch_size = 100\n", 410 | "n_iters = 3000\n", 411 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 412 | "num_epochs = int(num_epochs)\n", 413 | "\n", 414 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 415 | " batch_size=batch_size,\n", 416 | " shuffle=True)\n", 417 | "\n", 418 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 419 | " batch_size=batch_size,\n", 420 | " shuffle=False)\n", 421 | "\n", 422 | "'''\n", 423 | "STEP 3: CREATE MODEL CLASS\n", 424 | "'''\n", 425 | "\n", 426 | "class LSTMModel(nn.Module):\n", 427 | " def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n", 428 | " super(LSTMModel, self).__init__()\n", 429 | " # Hidden dimensions\n", 430 | " self.hidden_dim = hidden_dim\n", 431 | " \n", 432 | " # Number of hidden layers\n", 433 | " self.layer_dim = layer_dim\n", 434 | " \n", 435 | " # Building your LSTM\n", 436 | " # batch_first=True causes input/output tensors to be of shape\n", 437 | " # (batch_dim, seq_dim, feature_dim)\n", 438 | " self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n", 439 | " \n", 440 | " # Readout layer\n", 441 | " self.fc = nn.Linear(hidden_dim, output_dim)\n", 442 | " \n", 443 | " def forward(self, x):\n", 444 | " # Initialize hidden state with zeros\n", 445 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 446 | " \n", 447 | " # Initialize cell state\n", 448 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 449 | " \n", 450 | " # One time step\n", 451 | " out, (hn, cn) = self.lstm(x, (h0,c0))\n", 452 | " \n", 453 | " # Index hidden state of last time step\n", 454 | " # out.size() --> 100, 28, 100\n", 455 | " # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!\n", 456 | " out = self.fc(out[:, -1, :])\n", 457 | " # out.size() --> 100, 10\n", 458 | " return out\n", 459 | "\n", 460 | "'''\n", 461 | "STEP 4: INSTANTIATE MODEL CLASS\n", 462 | "'''\n", 463 | "input_dim = 28\n", 464 | "hidden_dim = 100\n", 465 | "layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\n", 466 | "output_dim = 10\n", 467 | "\n", 468 | "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n", 469 | "\n", 470 | "# JUST PRINTING MODEL & PARAMETERS\n", 471 | "print(model)\n", 472 | "print(len(list(model.parameters())))\n", 473 | "for i in range(len(list(model.parameters()))):\n", 474 | " print(list(model.parameters())[i].size())\n", 475 | "\n", 476 | "'''\n", 477 | "STEP 5: INSTANTIATE LOSS CLASS\n", 478 | "'''\n", 479 | "criterion = nn.CrossEntropyLoss()\n", 480 | "\n", 481 | "'''\n", 482 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 483 | "'''\n", 484 | "learning_rate = 0.1\n", 485 | "\n", 486 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 487 | "\n", 488 | "'''\n", 489 | "STEP 7: TRAIN THE MODEL\n", 490 | "'''\n", 491 | "\n", 492 | "# Number of steps to unroll\n", 493 | "seq_dim = 28\n", 494 | "\n", 495 | "iter = 0\n", 496 | "for epoch in range(num_epochs):\n", 497 | " for i, (images, labels) in enumerate(train_loader):\n", 498 | " # Load images as Variable\n", 499 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 500 | " labels = Variable(labels)\n", 501 | " \n", 502 | " # Clear gradients w.r.t. parameters\n", 503 | " optimizer.zero_grad()\n", 504 | " \n", 505 | " # Forward pass to get output/logits\n", 506 | " # outputs.size() --> 100, 10\n", 507 | " outputs = model(images)\n", 508 | " \n", 509 | " # Calculate Loss: softmax --> cross entropy loss\n", 510 | " loss = criterion(outputs, labels)\n", 511 | " \n", 512 | " # Getting gradients w.r.t. parameters\n", 513 | " loss.backward()\n", 514 | " \n", 515 | " # Updating parameters\n", 516 | " optimizer.step()\n", 517 | " \n", 518 | " iter += 1\n", 519 | " \n", 520 | " if iter % 500 == 0:\n", 521 | " # Calculate Accuracy\n", 522 | " correct = 0\n", 523 | " total = 0\n", 524 | " # Iterate through test dataset\n", 525 | " for images, labels in test_loader:\n", 526 | " # Load images to a Torch Variable\n", 527 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 528 | " \n", 529 | " # Forward pass only to get logits/output\n", 530 | " outputs = model(images)\n", 531 | " \n", 532 | " # Get predictions from the maximum value\n", 533 | " _, predicted = torch.max(outputs.data, 1)\n", 534 | " \n", 535 | " # Total number of labels\n", 536 | " total += labels.size(0)\n", 537 | " \n", 538 | " # Total correct predictions\n", 539 | " correct += (predicted == labels).sum()\n", 540 | " \n", 541 | " accuracy = 100 * correct / total\n", 542 | " \n", 543 | " # Print Loss\n", 544 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 545 | ] 546 | }, 547 | { 548 | "cell_type": "code", 549 | "execution_count": 30, 550 | "metadata": {}, 551 | "outputs": [ 552 | { 553 | "name": "stdout", 554 | "output_type": "stream", 555 | "text": [ 556 | "LSTMModel(\n", 557 | " (lstm): LSTM(28, 100, num_layers=3, batch_first=True)\n", 558 | " (fc): Linear(in_features=100, out_features=10, bias=True)\n", 559 | ")\n", 560 | "14\n", 561 | "torch.Size([400, 28])\n", 562 | "torch.Size([400, 100])\n", 563 | "torch.Size([400])\n", 564 | "torch.Size([400])\n", 565 | "torch.Size([400, 100])\n", 566 | "torch.Size([400, 100])\n", 567 | "torch.Size([400])\n", 568 | "torch.Size([400])\n", 569 | "torch.Size([400, 100])\n", 570 | "torch.Size([400, 100])\n", 571 | "torch.Size([400])\n", 572 | "torch.Size([400])\n", 573 | "torch.Size([10, 100])\n", 574 | "torch.Size([10])\n" 575 | ] 576 | }, 577 | { 578 | "name": "stderr", 579 | "output_type": "stream", 580 | "text": [ 581 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:158: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 582 | ] 583 | }, 584 | { 585 | "name": "stdout", 586 | "output_type": "stream", 587 | "text": [ 588 | "Iteration: 500. Loss: 2.3062610626220703. Accuracy: 11\n", 589 | "Iteration: 1000. Loss: 2.294701099395752. Accuracy: 11\n", 590 | "Iteration: 1500. Loss: 2.0946013927459717. Accuracy: 23\n", 591 | "Iteration: 2000. Loss: 0.9512524604797363. Accuracy: 62\n", 592 | "Iteration: 2500. Loss: 0.6940139532089233. Accuracy: 79\n", 593 | "Iteration: 3000. Loss: 0.39204394817352295. Accuracy: 82\n" 594 | ] 595 | } 596 | ], 597 | "source": [ 598 | "import torch\n", 599 | "import torch.nn as nn\n", 600 | "import torchvision.transforms as transforms\n", 601 | "import torchvision.datasets as dsets\n", 602 | "from torch.autograd import Variable\n", 603 | "\n", 604 | "'''\n", 605 | "STEP 1: LOADING DATASET\n", 606 | "'''\n", 607 | "train_dataset = dsets.MNIST(root='./data',\n", 608 | " train=True,\n", 609 | " transform=transforms.ToTensor(),\n", 610 | " download=True)\n", 611 | "\n", 612 | "test_dataset = dsets.MNIST(root='./data',\n", 613 | " train=False,\n", 614 | " transform=transforms.ToTensor())\n", 615 | "\n", 616 | "'''\n", 617 | "STEP 2: MAKING DATASET ITERABLE\n", 618 | "'''\n", 619 | "\n", 620 | "batch_size = 100\n", 621 | "n_iters = 3000\n", 622 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 623 | "num_epochs = int(num_epochs)\n", 624 | "\n", 625 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 626 | " batch_size=batch_size,\n", 627 | " shuffle=True)\n", 628 | "\n", 629 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 630 | " batch_size=batch_size,\n", 631 | " shuffle=False)\n", 632 | "\n", 633 | "'''\n", 634 | "STEP 3: CREATE MODEL CLASS\n", 635 | "'''\n", 636 | "\n", 637 | "class LSTMModel(nn.Module):\n", 638 | " def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n", 639 | " super(LSTMModel, self).__init__()\n", 640 | " # Hidden dimensions\n", 641 | " self.hidden_dim = hidden_dim\n", 642 | " \n", 643 | " # Number of hidden layers\n", 644 | " self.layer_dim = layer_dim\n", 645 | " \n", 646 | " # Building your LSTM\n", 647 | " # batch_first=True causes input/output tensors to be of shape\n", 648 | " # (batch_dim, seq_dim, feature_dim)\n", 649 | " self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n", 650 | " \n", 651 | " # Readout layer\n", 652 | " self.fc = nn.Linear(hidden_dim, output_dim)\n", 653 | " \n", 654 | " def forward(self, x):\n", 655 | " # Initialize hidden state with zeros\n", 656 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 657 | " \n", 658 | " # Initialize cell state\n", 659 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 660 | " \n", 661 | " # One time step\n", 662 | " out, (hn, cn) = self.lstm(x, (h0,c0))\n", 663 | " \n", 664 | " # Index hidden state of last time step\n", 665 | " # out.size() --> 100, 28, 100\n", 666 | " # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!\n", 667 | " out = self.fc(out[:, -1, :])\n", 668 | " # out.size() --> 100, 10\n", 669 | " return out\n", 670 | "\n", 671 | "'''\n", 672 | "STEP 4: INSTANTIATE MODEL CLASS\n", 673 | "'''\n", 674 | "input_dim = 28\n", 675 | "hidden_dim = 100\n", 676 | "layer_dim = 3 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\n", 677 | "output_dim = 10\n", 678 | "\n", 679 | "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n", 680 | "\n", 681 | "# JUST PRINTING MODEL & PARAMETERS\n", 682 | "print(model)\n", 683 | "print(len(list(model.parameters())))\n", 684 | "for i in range(len(list(model.parameters()))):\n", 685 | " print(list(model.parameters())[i].size())\n", 686 | "\n", 687 | "'''\n", 688 | "STEP 5: INSTANTIATE LOSS CLASS\n", 689 | "'''\n", 690 | "criterion = nn.CrossEntropyLoss()\n", 691 | "\n", 692 | "'''\n", 693 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 694 | "'''\n", 695 | "learning_rate = 0.1\n", 696 | "\n", 697 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 698 | "\n", 699 | "'''\n", 700 | "STEP 7: TRAIN THE MODEL\n", 701 | "'''\n", 702 | "\n", 703 | "# Number of steps to unroll\n", 704 | "seq_dim = 28\n", 705 | "\n", 706 | "iter = 0\n", 707 | "for epoch in range(num_epochs):\n", 708 | " for i, (images, labels) in enumerate(train_loader):\n", 709 | " # Load images as Variable\n", 710 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 711 | " labels = Variable(labels)\n", 712 | " \n", 713 | " # Clear gradients w.r.t. parameters\n", 714 | " optimizer.zero_grad()\n", 715 | " \n", 716 | " # Forward pass to get output/logits\n", 717 | " # outputs.size() --> 100, 10\n", 718 | " outputs = model(images)\n", 719 | " \n", 720 | " # Calculate Loss: softmax --> cross entropy loss\n", 721 | " loss = criterion(outputs, labels)\n", 722 | " \n", 723 | " # Getting gradients w.r.t. parameters\n", 724 | " loss.backward()\n", 725 | " \n", 726 | " # Updating parameters\n", 727 | " optimizer.step()\n", 728 | " \n", 729 | " iter += 1\n", 730 | " \n", 731 | " if iter % 500 == 0:\n", 732 | " # Calculate Accuracy\n", 733 | " correct = 0\n", 734 | " total = 0\n", 735 | " # Iterate through test dataset\n", 736 | " for images, labels in test_loader:\n", 737 | " # Load images to a Torch Variable\n", 738 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 739 | " \n", 740 | " # Forward pass only to get logits/output\n", 741 | " outputs = model(images)\n", 742 | " \n", 743 | " # Get predictions from the maximum value\n", 744 | " _, predicted = torch.max(outputs.data, 1)\n", 745 | " \n", 746 | " # Total number of labels\n", 747 | " total += labels.size(0)\n", 748 | " \n", 749 | " # Total correct predictions\n", 750 | " correct += (predicted == labels).sum()\n", 751 | " \n", 752 | " accuracy = 100 * correct / total\n", 753 | " \n", 754 | " # Print Loss\n", 755 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 756 | ] 757 | }, 758 | { 759 | "cell_type": "markdown", 760 | "metadata": {}, 761 | "source": [ 762 | "# LSTM From CPU to GPU in PyTorch" 763 | ] 764 | }, 765 | { 766 | "cell_type": "code", 767 | "execution_count": 31, 768 | "metadata": {}, 769 | "outputs": [ 770 | { 771 | "name": "stderr", 772 | "output_type": "stream", 773 | "text": [ 774 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:187: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 775 | ] 776 | }, 777 | { 778 | "name": "stdout", 779 | "output_type": "stream", 780 | "text": [ 781 | "Iteration: 500. Loss: 2.306875705718994. Accuracy: 11\n", 782 | "Iteration: 1000. Loss: 2.3024940490722656. Accuracy: 11\n", 783 | "Iteration: 1500. Loss: 2.241243839263916. Accuracy: 19\n", 784 | "Iteration: 2000. Loss: 1.0950016975402832. Accuracy: 62\n", 785 | "Iteration: 2500. Loss: 0.5713849067687988. Accuracy: 85\n", 786 | "Iteration: 3000. Loss: 0.4543074369430542. Accuracy: 91\n" 787 | ] 788 | } 789 | ], 790 | "source": [ 791 | "import torch\n", 792 | "import torch.nn as nn\n", 793 | "import torchvision.transforms as transforms\n", 794 | "import torchvision.datasets as dsets\n", 795 | "from torch.autograd import Variable\n", 796 | "\n", 797 | "'''\n", 798 | "STEP 1: LOADING DATASET\n", 799 | "'''\n", 800 | "train_dataset = dsets.MNIST(root='./data',\n", 801 | " train=True,\n", 802 | " transform=transforms.ToTensor(),\n", 803 | " download=True)\n", 804 | "\n", 805 | "test_dataset = dsets.MNIST(root='./data',\n", 806 | " train=False,\n", 807 | " transform=transforms.ToTensor())\n", 808 | "\n", 809 | "'''\n", 810 | "STEP 2: MAKING DATASET ITERABLE\n", 811 | "'''\n", 812 | "\n", 813 | "batch_size = 100\n", 814 | "n_iters = 3000\n", 815 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 816 | "num_epochs = int(num_epochs)\n", 817 | "\n", 818 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 819 | " batch_size=batch_size,\n", 820 | " shuffle=True)\n", 821 | "\n", 822 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 823 | " batch_size=batch_size,\n", 824 | " shuffle=False)\n", 825 | "\n", 826 | "'''\n", 827 | "STEP 3: CREATE MODEL CLASS\n", 828 | "'''\n", 829 | "\n", 830 | "class LSTMModel(nn.Module):\n", 831 | " def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n", 832 | " super(LSTMModel, self).__init__()\n", 833 | " # Hidden dimensions\n", 834 | " self.hidden_dim = hidden_dim\n", 835 | " \n", 836 | " # Number of hidden layers\n", 837 | " self.layer_dim = layer_dim\n", 838 | " \n", 839 | " # Building your LSTM\n", 840 | " # batch_first=True causes input/output tensors to be of shape\n", 841 | " # (batch_dim, seq_dim, feature_dim)\n", 842 | " self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n", 843 | " \n", 844 | " # Readout layer\n", 845 | " self.fc = nn.Linear(hidden_dim, output_dim)\n", 846 | " \n", 847 | " def forward(self, x):\n", 848 | " # Initialize hidden state with zeros\n", 849 | " #######################\n", 850 | " # USE GPU FOR MODEL #\n", 851 | " #######################\n", 852 | " \n", 853 | " if torch.cuda.is_available():\n", 854 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n", 855 | " else:\n", 856 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 857 | " \n", 858 | " # Initialize cell state\n", 859 | " if torch.cuda.is_available():\n", 860 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n", 861 | " else:\n", 862 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 863 | " \n", 864 | " # One time step\n", 865 | " out, (hn, cn) = self.lstm(x, (h0,c0))\n", 866 | " \n", 867 | " # Index hidden state of last time step\n", 868 | " # out.size() --> 100, 28, 100\n", 869 | " # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!\n", 870 | " out = self.fc(out[:, -1, :])\n", 871 | " # out.size() --> 100, 10\n", 872 | " return out\n", 873 | "\n", 874 | "'''\n", 875 | "STEP 4: INSTANTIATE MODEL CLASS\n", 876 | "'''\n", 877 | "input_dim = 28\n", 878 | "hidden_dim = 100\n", 879 | "layer_dim = 3 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\n", 880 | "output_dim = 10\n", 881 | "\n", 882 | "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n", 883 | "\n", 884 | "#######################\n", 885 | "# USE GPU FOR MODEL #\n", 886 | "#######################\n", 887 | "\n", 888 | "if torch.cuda.is_available():\n", 889 | " model.cuda()\n", 890 | "\n", 891 | "'''\n", 892 | "STEP 5: INSTANTIATE LOSS CLASS\n", 893 | "'''\n", 894 | "criterion = nn.CrossEntropyLoss()\n", 895 | "\n", 896 | "'''\n", 897 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 898 | "'''\n", 899 | "learning_rate = 0.1\n", 900 | "\n", 901 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 902 | "\n", 903 | "'''\n", 904 | "STEP 7: TRAIN THE MODEL\n", 905 | "'''\n", 906 | "\n", 907 | "# Number of steps to unroll\n", 908 | "seq_dim = 28\n", 909 | "\n", 910 | "iter = 0\n", 911 | "for epoch in range(num_epochs):\n", 912 | " for i, (images, labels) in enumerate(train_loader):\n", 913 | " # Load images as Variable\n", 914 | " #######################\n", 915 | " # USE GPU FOR MODEL #\n", 916 | " #######################\n", 917 | " if torch.cuda.is_available():\n", 918 | " images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n", 919 | " labels = Variable(labels.cuda())\n", 920 | " else:\n", 921 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 922 | " labels = Variable(labels)\n", 923 | " \n", 924 | " # Clear gradients w.r.t. parameters\n", 925 | " optimizer.zero_grad()\n", 926 | " \n", 927 | " # Forward pass to get output/logits\n", 928 | " # outputs.size() --> 100, 10\n", 929 | " outputs = model(images)\n", 930 | " \n", 931 | " # Calculate Loss: softmax --> cross entropy loss\n", 932 | " loss = criterion(outputs, labels)\n", 933 | " \n", 934 | " # Getting gradients w.r.t. parameters\n", 935 | " loss.backward()\n", 936 | " \n", 937 | " # Updating parameters\n", 938 | " optimizer.step()\n", 939 | " \n", 940 | " iter += 1\n", 941 | " \n", 942 | " if iter % 500 == 0:\n", 943 | " # Calculate Accuracy\n", 944 | " correct = 0\n", 945 | " total = 0\n", 946 | " # Iterate through test dataset\n", 947 | " for images, labels in test_loader:\n", 948 | " #######################\n", 949 | " # USE GPU FOR MODEL #\n", 950 | " #######################\n", 951 | " if torch.cuda.is_available():\n", 952 | " images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n", 953 | " else:\n", 954 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 955 | " \n", 956 | " # Forward pass only to get logits/output\n", 957 | " outputs = model(images)\n", 958 | " \n", 959 | " # Get predictions from the maximum value\n", 960 | " _, predicted = torch.max(outputs.data, 1)\n", 961 | " \n", 962 | " # Total number of labels\n", 963 | " total += labels.size(0)\n", 964 | " \n", 965 | " # Total correct predictions\n", 966 | " #######################\n", 967 | " # USE GPU FOR MODEL #\n", 968 | " #######################\n", 969 | " if torch.cuda.is_available():\n", 970 | " correct += (predicted.cpu() == labels.cpu()),sum()\n", 971 | " else:\n", 972 | " correct += (predicted == labels).sum()\n", 973 | " \n", 974 | " accuracy = 100 * correct / total\n", 975 | " \n", 976 | " # Print Loss\n", 977 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 978 | ] 979 | }, 980 | { 981 | "cell_type": "code", 982 | "execution_count": null, 983 | "metadata": {}, 984 | "outputs": [], 985 | "source": [] 986 | } 987 | ], 988 | "metadata": { 989 | "kernelspec": { 990 | "display_name": "Python 3", 991 | "language": "python", 992 | "name": "python3" 993 | }, 994 | "language_info": { 995 | "codemirror_mode": { 996 | "name": "ipython", 997 | "version": 3 998 | }, 999 | "file_extension": ".py", 1000 | "mimetype": "text/x-python", 1001 | "name": "python", 1002 | "nbconvert_exporter": "python", 1003 | "pygments_lexer": "ipython3", 1004 | "version": "3.6.5" 1005 | } 1006 | }, 1007 | "nbformat": 4, 1008 | "nbformat_minor": 2 1009 | } 1010 | -------------------------------------------------------------------------------- /03. PyTorch Fundamentals - Matrices.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Matrix Basics" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 2, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 3, 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | "[[1, 2], [3, 4]]\n" 29 | ] 30 | } 31 | ], 32 | "source": [ 33 | "# Creating a 2x2 array\n", 34 | "arr = [[1, 2], [3, 4]]\n", 35 | "print(arr)" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 4, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "data": { 45 | "text/plain": [ 46 | "array([[1, 2],\n", 47 | " [3, 4]])" 48 | ] 49 | }, 50 | "execution_count": 4, 51 | "metadata": {}, 52 | "output_type": "execute_result" 53 | } 54 | ], 55 | "source": [ 56 | "# Convert to NumPy\n", 57 | "np.array(arr)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 5, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "import torch" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 6, 72 | "metadata": {}, 73 | "outputs": [ 74 | { 75 | "data": { 76 | "text/plain": [ 77 | "tensor([[ 1., 2.],\n", 78 | " [ 3., 4.]])" 79 | ] 80 | }, 81 | "execution_count": 6, 82 | "metadata": {}, 83 | "output_type": "execute_result" 84 | } 85 | ], 86 | "source": [ 87 | "# Convert to PyTorch Tensor\n", 88 | "torch.Tensor(arr)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 7, 94 | "metadata": {}, 95 | "outputs": [ 96 | { 97 | "data": { 98 | "text/plain": [ 99 | "array([[1., 1.],\n", 100 | " [1., 1.]])" 101 | ] 102 | }, 103 | "execution_count": 7, 104 | "metadata": {}, 105 | "output_type": "execute_result" 106 | } 107 | ], 108 | "source": [ 109 | "np.ones((2, 2))" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 8, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "data": { 119 | "text/plain": [ 120 | "tensor([[ 1., 1.],\n", 121 | " [ 1., 1.]])" 122 | ] 123 | }, 124 | "execution_count": 8, 125 | "metadata": {}, 126 | "output_type": "execute_result" 127 | } 128 | ], 129 | "source": [ 130 | "torch.ones((2, 2))" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 9, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "data": { 140 | "text/plain": [ 141 | "array([[0.54314946, 0.32265673],\n", 142 | " [0.76294125, 0.58130214]])" 143 | ] 144 | }, 145 | "execution_count": 9, 146 | "metadata": {}, 147 | "output_type": "execute_result" 148 | } 149 | ], 150 | "source": [ 151 | "np.random.rand(2, 2)" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 10, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "data": { 161 | "text/plain": [ 162 | "tensor([[ 0.1709, 0.6083],\n", 163 | " [ 0.6287, 0.5718]])" 164 | ] 165 | }, 166 | "execution_count": 10, 167 | "metadata": {}, 168 | "output_type": "execute_result" 169 | } 170 | ], 171 | "source": [ 172 | "torch.rand(2, 2)" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": {}, 178 | "source": [ 179 | "# Seed for Reproducibility" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 11, 185 | "metadata": {}, 186 | "outputs": [ 187 | { 188 | "data": { 189 | "text/plain": [ 190 | "array([[0.5488135 , 0.71518937],\n", 191 | " [0.60276338, 0.54488318]])" 192 | ] 193 | }, 194 | "execution_count": 11, 195 | "metadata": {}, 196 | "output_type": "execute_result" 197 | } 198 | ], 199 | "source": [ 200 | "# Seed\n", 201 | "np.random.seed(0)\n", 202 | "np.random.rand(2, 2)" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 12, 208 | "metadata": {}, 209 | "outputs": [ 210 | { 211 | "data": { 212 | "text/plain": [ 213 | "array([[0.5488135 , 0.71518937],\n", 214 | " [0.60276338, 0.54488318]])" 215 | ] 216 | }, 217 | "execution_count": 12, 218 | "metadata": {}, 219 | "output_type": "execute_result" 220 | } 221 | ], 222 | "source": [ 223 | "# Seed\n", 224 | "np.random.seed(0)\n", 225 | "np.random.rand(2, 2)" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 13, 231 | "metadata": {}, 232 | "outputs": [ 233 | { 234 | "data": { 235 | "text/plain": [ 236 | "array([[0.4236548 , 0.64589411],\n", 237 | " [0.43758721, 0.891773 ]])" 238 | ] 239 | }, 240 | "execution_count": 13, 241 | "metadata": {}, 242 | "output_type": "execute_result" 243 | } 244 | ], 245 | "source": [ 246 | "# No seed\n", 247 | "np.random.rand(2, 2)" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 14, 253 | "metadata": {}, 254 | "outputs": [ 255 | { 256 | "data": { 257 | "text/plain": [ 258 | "array([[0.96366276, 0.38344152],\n", 259 | " [0.79172504, 0.52889492]])" 260 | ] 261 | }, 262 | "execution_count": 14, 263 | "metadata": {}, 264 | "output_type": "execute_result" 265 | } 266 | ], 267 | "source": [ 268 | "# No seed\n", 269 | "np.random.rand(2, 2)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 15, 275 | "metadata": {}, 276 | "outputs": [ 277 | { 278 | "data": { 279 | "text/plain": [ 280 | "tensor([[ 0.4963, 0.7682],\n", 281 | " [ 0.0885, 0.1320]])" 282 | ] 283 | }, 284 | "execution_count": 15, 285 | "metadata": {}, 286 | "output_type": "execute_result" 287 | } 288 | ], 289 | "source": [ 290 | "# Torch Seed\n", 291 | "torch.manual_seed(0)\n", 292 | "torch.rand(2, 2)" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": 16, 298 | "metadata": {}, 299 | "outputs": [ 300 | { 301 | "data": { 302 | "text/plain": [ 303 | "tensor([[ 0.4963, 0.7682],\n", 304 | " [ 0.0885, 0.1320]])" 305 | ] 306 | }, 307 | "execution_count": 16, 308 | "metadata": {}, 309 | "output_type": "execute_result" 310 | } 311 | ], 312 | "source": [ 313 | "# Torch Seed\n", 314 | "torch.manual_seed(0)\n", 315 | "torch.rand(2, 2)" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": 17, 321 | "metadata": {}, 322 | "outputs": [ 323 | { 324 | "data": { 325 | "text/plain": [ 326 | "tensor([[ 0.3074, 0.6341],\n", 327 | " [ 0.4901, 0.8964]])" 328 | ] 329 | }, 330 | "execution_count": 17, 331 | "metadata": {}, 332 | "output_type": "execute_result" 333 | } 334 | ], 335 | "source": [ 336 | "# Torch No Seed\n", 337 | "torch.rand(2, 2)" 338 | ] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": 18, 343 | "metadata": {}, 344 | "outputs": [ 345 | { 346 | "data": { 347 | "text/plain": [ 348 | "tensor([[ 0.4556, 0.6323],\n", 349 | " [ 0.3489, 0.4017]])" 350 | ] 351 | }, 352 | "execution_count": 18, 353 | "metadata": {}, 354 | "output_type": "execute_result" 355 | } 356 | ], 357 | "source": [ 358 | "# Torch No Seed\n", 359 | "torch.rand(2, 2)" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": 19, 365 | "metadata": {}, 366 | "outputs": [ 367 | { 368 | "data": { 369 | "text/plain": [ 370 | "False" 371 | ] 372 | }, 373 | "execution_count": 19, 374 | "metadata": {}, 375 | "output_type": "execute_result" 376 | } 377 | ], 378 | "source": [ 379 | "torch.cuda.is_available()" 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": 20, 385 | "metadata": {}, 386 | "outputs": [], 387 | "source": [ 388 | "if torch.cuda.is_available():\n", 389 | " torch.cuda.manual_seed_all(0)" 390 | ] 391 | }, 392 | { 393 | "cell_type": "markdown", 394 | "metadata": {}, 395 | "source": [ 396 | "# Torch to NumPy Bridge" 397 | ] 398 | }, 399 | { 400 | "cell_type": "code", 401 | "execution_count": 21, 402 | "metadata": {}, 403 | "outputs": [], 404 | "source": [ 405 | "# Numpy array\n", 406 | "np_array = np.ones((2, 2))" 407 | ] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": 22, 412 | "metadata": {}, 413 | "outputs": [ 414 | { 415 | "name": "stdout", 416 | "output_type": "stream", 417 | "text": [ 418 | "[[1. 1.]\n", 419 | " [1. 1.]]\n" 420 | ] 421 | } 422 | ], 423 | "source": [ 424 | "print(np_array)" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": 23, 430 | "metadata": {}, 431 | "outputs": [ 432 | { 433 | "name": "stdout", 434 | "output_type": "stream", 435 | "text": [ 436 | "\n" 437 | ] 438 | } 439 | ], 440 | "source": [ 441 | "print(type(np_array))" 442 | ] 443 | }, 444 | { 445 | "cell_type": "code", 446 | "execution_count": 24, 447 | "metadata": {}, 448 | "outputs": [], 449 | "source": [ 450 | "# Convert to Torch Tensor\n", 451 | "torch_tensor = torch.from_numpy(np_array)" 452 | ] 453 | }, 454 | { 455 | "cell_type": "code", 456 | "execution_count": 25, 457 | "metadata": {}, 458 | "outputs": [ 459 | { 460 | "name": "stdout", 461 | "output_type": "stream", 462 | "text": [ 463 | "tensor([[ 1., 1.],\n", 464 | " [ 1., 1.]], dtype=torch.float64)\n" 465 | ] 466 | } 467 | ], 468 | "source": [ 469 | "print(torch_tensor)" 470 | ] 471 | }, 472 | { 473 | "cell_type": "code", 474 | "execution_count": 26, 475 | "metadata": {}, 476 | "outputs": [ 477 | { 478 | "name": "stdout", 479 | "output_type": "stream", 480 | "text": [ 481 | "\n" 482 | ] 483 | } 484 | ], 485 | "source": [ 486 | "print(type(torch_tensor))" 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 27, 492 | "metadata": {}, 493 | "outputs": [ 494 | { 495 | "ename": "TypeError", 496 | "evalue": "can't convert np.ndarray of type numpy.int8. The only supported types are: double, float, float16, int64, int32, and uint8.", 497 | "output_type": "error", 498 | "traceback": [ 499 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 500 | "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", 501 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Data types matter: intentional error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mnp_array_new\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp_array_new\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 502 | "\u001b[0;31mTypeError\u001b[0m: can't convert np.ndarray of type numpy.int8. The only supported types are: double, float, float16, int64, int32, and uint8." 503 | ] 504 | } 505 | ], 506 | "source": [ 507 | "# Data types matter: intentional error\n", 508 | "np_array_new = np.ones((2, 2), dtype=np.int8)\n", 509 | "torch.from_numpy(np_array_new)" 510 | ] 511 | }, 512 | { 513 | "cell_type": "code", 514 | "execution_count": 30, 515 | "metadata": {}, 516 | "outputs": [ 517 | { 518 | "data": { 519 | "text/plain": [ 520 | "tensor([[ 1, 1],\n", 521 | " [ 1, 1]])" 522 | ] 523 | }, 524 | "execution_count": 30, 525 | "metadata": {}, 526 | "output_type": "execute_result" 527 | } 528 | ], 529 | "source": [ 530 | "# Data types matter\n", 531 | "np_array_new = np.ones((2, 2), dtype=np.int64)\n", 532 | "torch.from_numpy(np_array_new)" 533 | ] 534 | }, 535 | { 536 | "cell_type": "code", 537 | "execution_count": 33, 538 | "metadata": {}, 539 | "outputs": [ 540 | { 541 | "data": { 542 | "text/plain": [ 543 | "tensor([[ 1, 1],\n", 544 | " [ 1, 1]], dtype=torch.int32)" 545 | ] 546 | }, 547 | "execution_count": 33, 548 | "metadata": {}, 549 | "output_type": "execute_result" 550 | } 551 | ], 552 | "source": [ 553 | "# Data types matter\n", 554 | "np_array_new = np.ones((2, 2), dtype=np.int32)\n", 555 | "torch.from_numpy(np_array_new)" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": 36, 561 | "metadata": {}, 562 | "outputs": [ 563 | { 564 | "data": { 565 | "text/plain": [ 566 | "tensor([[ 1, 1],\n", 567 | " [ 1, 1]], dtype=torch.uint8)" 568 | ] 569 | }, 570 | "execution_count": 36, 571 | "metadata": {}, 572 | "output_type": "execute_result" 573 | } 574 | ], 575 | "source": [ 576 | "# Data types matter\n", 577 | "np_array_new = np.ones((2, 2), dtype=np.uint8)\n", 578 | "torch.from_numpy(np_array_new)" 579 | ] 580 | }, 581 | { 582 | "cell_type": "code", 583 | "execution_count": 39, 584 | "metadata": {}, 585 | "outputs": [ 586 | { 587 | "data": { 588 | "text/plain": [ 589 | "tensor([[ 1., 1.],\n", 590 | " [ 1., 1.]], dtype=torch.float64)" 591 | ] 592 | }, 593 | "execution_count": 39, 594 | "metadata": {}, 595 | "output_type": "execute_result" 596 | } 597 | ], 598 | "source": [ 599 | "# Data types matter\n", 600 | "np_array_new = np.ones((2, 2), dtype=np.float64)\n", 601 | "torch.from_numpy(np_array_new)" 602 | ] 603 | }, 604 | { 605 | "cell_type": "code", 606 | "execution_count": 42, 607 | "metadata": {}, 608 | "outputs": [ 609 | { 610 | "data": { 611 | "text/plain": [ 612 | "tensor([[ 1., 1.],\n", 613 | " [ 1., 1.]])" 614 | ] 615 | }, 616 | "execution_count": 42, 617 | "metadata": {}, 618 | "output_type": "execute_result" 619 | } 620 | ], 621 | "source": [ 622 | "# Data types matter\n", 623 | "np_array_new = np.ones((2, 2), dtype=np.float32)\n", 624 | "torch.from_numpy(np_array_new)" 625 | ] 626 | }, 627 | { 628 | "cell_type": "code", 629 | "execution_count": 45, 630 | "metadata": {}, 631 | "outputs": [ 632 | { 633 | "data": { 634 | "text/plain": [ 635 | "tensor([[ 1., 1.],\n", 636 | " [ 1., 1.]], dtype=torch.float64)" 637 | ] 638 | }, 639 | "execution_count": 45, 640 | "metadata": {}, 641 | "output_type": "execute_result" 642 | } 643 | ], 644 | "source": [ 645 | "# Data types matter\n", 646 | "np_array_new = np.ones((2, 2), dtype=np.double)\n", 647 | "torch.from_numpy(np_array_new)" 648 | ] 649 | }, 650 | { 651 | "cell_type": "markdown", 652 | "metadata": {}, 653 | "source": [ 654 | "# NumPy to Torch Bridge" 655 | ] 656 | }, 657 | { 658 | "cell_type": "code", 659 | "execution_count": 46, 660 | "metadata": {}, 661 | "outputs": [], 662 | "source": [ 663 | "torch_tensor = torch.ones(2, 2)" 664 | ] 665 | }, 666 | { 667 | "cell_type": "code", 668 | "execution_count": 47, 669 | "metadata": {}, 670 | "outputs": [ 671 | { 672 | "data": { 673 | "text/plain": [ 674 | "torch.Tensor" 675 | ] 676 | }, 677 | "execution_count": 47, 678 | "metadata": {}, 679 | "output_type": "execute_result" 680 | } 681 | ], 682 | "source": [ 683 | "type(torch_tensor)" 684 | ] 685 | }, 686 | { 687 | "cell_type": "code", 688 | "execution_count": 48, 689 | "metadata": {}, 690 | "outputs": [], 691 | "source": [ 692 | "torch_to_numpy = torch_tensor.numpy()" 693 | ] 694 | }, 695 | { 696 | "cell_type": "code", 697 | "execution_count": 49, 698 | "metadata": { 699 | "scrolled": true 700 | }, 701 | "outputs": [ 702 | { 703 | "data": { 704 | "text/plain": [ 705 | "numpy.ndarray" 706 | ] 707 | }, 708 | "execution_count": 49, 709 | "metadata": {}, 710 | "output_type": "execute_result" 711 | } 712 | ], 713 | "source": [ 714 | "type(torch_to_numpy)" 715 | ] 716 | }, 717 | { 718 | "cell_type": "markdown", 719 | "metadata": {}, 720 | "source": [ 721 | "# GPU and CPU Toggling" 722 | ] 723 | }, 724 | { 725 | "cell_type": "code", 726 | "execution_count": 51, 727 | "metadata": {}, 728 | "outputs": [], 729 | "source": [ 730 | "# CPU\n", 731 | "tensor_cpu = torch.ones(2, 2)" 732 | ] 733 | }, 734 | { 735 | "cell_type": "code", 736 | "execution_count": 54, 737 | "metadata": {}, 738 | "outputs": [], 739 | "source": [ 740 | "# CPU to GPU\n", 741 | "if torch.cuda.is_available():\n", 742 | " tensor_cpu.cuda()" 743 | ] 744 | }, 745 | { 746 | "cell_type": "code", 747 | "execution_count": 55, 748 | "metadata": { 749 | "scrolled": true 750 | }, 751 | "outputs": [ 752 | { 753 | "data": { 754 | "text/plain": [ 755 | "tensor([[ 1., 1.],\n", 756 | " [ 1., 1.]])" 757 | ] 758 | }, 759 | "execution_count": 55, 760 | "metadata": {}, 761 | "output_type": "execute_result" 762 | } 763 | ], 764 | "source": [ 765 | "# GPU to CPU\n", 766 | "tensor_cpu.cpu()" 767 | ] 768 | }, 769 | { 770 | "cell_type": "markdown", 771 | "metadata": {}, 772 | "source": [ 773 | "# Basic Mathematical Tensor Operations" 774 | ] 775 | }, 776 | { 777 | "cell_type": "code", 778 | "execution_count": 56, 779 | "metadata": {}, 780 | "outputs": [ 781 | { 782 | "name": "stdout", 783 | "output_type": "stream", 784 | "text": [ 785 | "tensor([[ 1., 1.],\n", 786 | " [ 1., 1.]])\n" 787 | ] 788 | } 789 | ], 790 | "source": [ 791 | "a = torch.ones(2, 2)\n", 792 | "print(a)" 793 | ] 794 | }, 795 | { 796 | "cell_type": "code", 797 | "execution_count": 57, 798 | "metadata": {}, 799 | "outputs": [ 800 | { 801 | "name": "stdout", 802 | "output_type": "stream", 803 | "text": [ 804 | "torch.Size([2, 2])\n" 805 | ] 806 | } 807 | ], 808 | "source": [ 809 | "print(a.size())" 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": 58, 815 | "metadata": {}, 816 | "outputs": [ 817 | { 818 | "data": { 819 | "text/plain": [ 820 | "tensor([ 1., 1., 1., 1.])" 821 | ] 822 | }, 823 | "execution_count": 58, 824 | "metadata": {}, 825 | "output_type": "execute_result" 826 | } 827 | ], 828 | "source": [ 829 | "a.view(4)" 830 | ] 831 | }, 832 | { 833 | "cell_type": "code", 834 | "execution_count": 59, 835 | "metadata": {}, 836 | "outputs": [ 837 | { 838 | "data": { 839 | "text/plain": [ 840 | "torch.Size([4])" 841 | ] 842 | }, 843 | "execution_count": 59, 844 | "metadata": {}, 845 | "output_type": "execute_result" 846 | } 847 | ], 848 | "source": [ 849 | "a.view(4).size()" 850 | ] 851 | }, 852 | { 853 | "cell_type": "code", 854 | "execution_count": 60, 855 | "metadata": {}, 856 | "outputs": [ 857 | { 858 | "name": "stdout", 859 | "output_type": "stream", 860 | "text": [ 861 | "tensor([[ 1., 1.],\n", 862 | " [ 1., 1.]])\n" 863 | ] 864 | } 865 | ], 866 | "source": [ 867 | "a = torch.ones(2, 2)\n", 868 | "print(a)" 869 | ] 870 | }, 871 | { 872 | "cell_type": "code", 873 | "execution_count": 61, 874 | "metadata": {}, 875 | "outputs": [ 876 | { 877 | "name": "stdout", 878 | "output_type": "stream", 879 | "text": [ 880 | "tensor([[ 1., 1.],\n", 881 | " [ 1., 1.]])\n" 882 | ] 883 | } 884 | ], 885 | "source": [ 886 | "b = torch.ones(2, 2)\n", 887 | "print(b)" 888 | ] 889 | }, 890 | { 891 | "cell_type": "code", 892 | "execution_count": 62, 893 | "metadata": {}, 894 | "outputs": [ 895 | { 896 | "name": "stdout", 897 | "output_type": "stream", 898 | "text": [ 899 | "tensor([[ 2., 2.],\n", 900 | " [ 2., 2.]])\n" 901 | ] 902 | } 903 | ], 904 | "source": [ 905 | "# Element-wise addition\n", 906 | "c = a + b\n", 907 | "print(c)" 908 | ] 909 | }, 910 | { 911 | "cell_type": "code", 912 | "execution_count": 63, 913 | "metadata": {}, 914 | "outputs": [ 915 | { 916 | "name": "stdout", 917 | "output_type": "stream", 918 | "text": [ 919 | "tensor([[ 2., 2.],\n", 920 | " [ 2., 2.]])\n" 921 | ] 922 | } 923 | ], 924 | "source": [ 925 | "# Element-wise addition\n", 926 | "c = torch.add(a, b)\n", 927 | "print(c)" 928 | ] 929 | }, 930 | { 931 | "cell_type": "code", 932 | "execution_count": 64, 933 | "metadata": {}, 934 | "outputs": [ 935 | { 936 | "name": "stdout", 937 | "output_type": "stream", 938 | "text": [ 939 | "Old c tensor\n", 940 | "tensor([[ 2., 2.],\n", 941 | " [ 2., 2.]])\n", 942 | "------------------------------------------------------------\n", 943 | "New c tensor\n", 944 | "tensor([[ 3., 3.],\n", 945 | " [ 3., 3.]])\n" 946 | ] 947 | } 948 | ], 949 | "source": [ 950 | "# In-place addition\n", 951 | "print('Old c tensor')\n", 952 | "print(c)\n", 953 | "\n", 954 | "c.add_(a)\n", 955 | "\n", 956 | "print('-'*60)\n", 957 | "print('New c tensor')\n", 958 | "print(c)" 959 | ] 960 | }, 961 | { 962 | "cell_type": "code", 963 | "execution_count": 65, 964 | "metadata": {}, 965 | "outputs": [ 966 | { 967 | "name": "stdout", 968 | "output_type": "stream", 969 | "text": [ 970 | "tensor([[ 1., 1.],\n", 971 | " [ 1., 1.]])\n", 972 | "tensor([[ 1., 1.],\n", 973 | " [ 1., 1.]])\n" 974 | ] 975 | } 976 | ], 977 | "source": [ 978 | "print(a)\n", 979 | "print(b)" 980 | ] 981 | }, 982 | { 983 | "cell_type": "code", 984 | "execution_count": 66, 985 | "metadata": {}, 986 | "outputs": [ 987 | { 988 | "data": { 989 | "text/plain": [ 990 | "tensor([[ 0., 0.],\n", 991 | " [ 0., 0.]])" 992 | ] 993 | }, 994 | "execution_count": 66, 995 | "metadata": {}, 996 | "output_type": "execute_result" 997 | } 998 | ], 999 | "source": [ 1000 | "a - b" 1001 | ] 1002 | }, 1003 | { 1004 | "cell_type": "code", 1005 | "execution_count": 67, 1006 | "metadata": {}, 1007 | "outputs": [ 1008 | { 1009 | "name": "stdout", 1010 | "output_type": "stream", 1011 | "text": [ 1012 | "tensor([[ 0., 0.],\n", 1013 | " [ 0., 0.]])\n", 1014 | "tensor([[ 1., 1.],\n", 1015 | " [ 1., 1.]])\n" 1016 | ] 1017 | } 1018 | ], 1019 | "source": [ 1020 | "# Not in-place\n", 1021 | "print(a.sub(b))\n", 1022 | "print(a)" 1023 | ] 1024 | }, 1025 | { 1026 | "cell_type": "code", 1027 | "execution_count": 68, 1028 | "metadata": {}, 1029 | "outputs": [ 1030 | { 1031 | "name": "stdout", 1032 | "output_type": "stream", 1033 | "text": [ 1034 | "tensor([[ 0., 0.],\n", 1035 | " [ 0., 0.]])\n", 1036 | "tensor([[ 0., 0.],\n", 1037 | " [ 0., 0.]])\n" 1038 | ] 1039 | } 1040 | ], 1041 | "source": [ 1042 | "# Inplace\n", 1043 | "print(a.sub_(b))\n", 1044 | "print(a)" 1045 | ] 1046 | }, 1047 | { 1048 | "cell_type": "code", 1049 | "execution_count": 69, 1050 | "metadata": {}, 1051 | "outputs": [ 1052 | { 1053 | "name": "stdout", 1054 | "output_type": "stream", 1055 | "text": [ 1056 | "tensor([[ 1., 1.],\n", 1057 | " [ 1., 1.]])\n", 1058 | "tensor([[ 0., 0.],\n", 1059 | " [ 0., 0.]])\n" 1060 | ] 1061 | } 1062 | ], 1063 | "source": [ 1064 | "a = torch.ones(2, 2)\n", 1065 | "print(a)\n", 1066 | "b = torch.zeros(2, 2)\n", 1067 | "print(b)" 1068 | ] 1069 | }, 1070 | { 1071 | "cell_type": "code", 1072 | "execution_count": 70, 1073 | "metadata": {}, 1074 | "outputs": [ 1075 | { 1076 | "data": { 1077 | "text/plain": [ 1078 | "tensor([[ 0., 0.],\n", 1079 | " [ 0., 0.]])" 1080 | ] 1081 | }, 1082 | "execution_count": 70, 1083 | "metadata": {}, 1084 | "output_type": "execute_result" 1085 | } 1086 | ], 1087 | "source": [ 1088 | "a * b" 1089 | ] 1090 | }, 1091 | { 1092 | "cell_type": "code", 1093 | "execution_count": 71, 1094 | "metadata": {}, 1095 | "outputs": [ 1096 | { 1097 | "name": "stdout", 1098 | "output_type": "stream", 1099 | "text": [ 1100 | "tensor([[ 0., 0.],\n", 1101 | " [ 0., 0.]])\n", 1102 | "tensor([[ 1., 1.],\n", 1103 | " [ 1., 1.]])\n" 1104 | ] 1105 | } 1106 | ], 1107 | "source": [ 1108 | "# Not in-place\n", 1109 | "print(torch.mul(a, b))\n", 1110 | "print(a)" 1111 | ] 1112 | }, 1113 | { 1114 | "cell_type": "code", 1115 | "execution_count": 72, 1116 | "metadata": {}, 1117 | "outputs": [ 1118 | { 1119 | "name": "stdout", 1120 | "output_type": "stream", 1121 | "text": [ 1122 | "tensor([[ 0., 0.],\n", 1123 | " [ 0., 0.]])\n", 1124 | "tensor([[ 0., 0.],\n", 1125 | " [ 0., 0.]])\n" 1126 | ] 1127 | } 1128 | ], 1129 | "source": [ 1130 | "# In-place\n", 1131 | "print(a.mul_(b))\n", 1132 | "print(a)" 1133 | ] 1134 | }, 1135 | { 1136 | "cell_type": "code", 1137 | "execution_count": 73, 1138 | "metadata": {}, 1139 | "outputs": [], 1140 | "source": [ 1141 | "a = torch.ones(2, 2)\n", 1142 | "b = torch.zeros(2, 2)" 1143 | ] 1144 | }, 1145 | { 1146 | "cell_type": "code", 1147 | "execution_count": 74, 1148 | "metadata": {}, 1149 | "outputs": [ 1150 | { 1151 | "data": { 1152 | "text/plain": [ 1153 | "tensor([[ 0., 0.],\n", 1154 | " [ 0., 0.]])" 1155 | ] 1156 | }, 1157 | "execution_count": 74, 1158 | "metadata": {}, 1159 | "output_type": "execute_result" 1160 | } 1161 | ], 1162 | "source": [ 1163 | "b / a" 1164 | ] 1165 | }, 1166 | { 1167 | "cell_type": "code", 1168 | "execution_count": 75, 1169 | "metadata": {}, 1170 | "outputs": [ 1171 | { 1172 | "data": { 1173 | "text/plain": [ 1174 | "tensor([[ 0., 0.],\n", 1175 | " [ 0., 0.]])" 1176 | ] 1177 | }, 1178 | "execution_count": 75, 1179 | "metadata": {}, 1180 | "output_type": "execute_result" 1181 | } 1182 | ], 1183 | "source": [ 1184 | "torch.div(b, a)" 1185 | ] 1186 | }, 1187 | { 1188 | "cell_type": "code", 1189 | "execution_count": 76, 1190 | "metadata": {}, 1191 | "outputs": [ 1192 | { 1193 | "data": { 1194 | "text/plain": [ 1195 | "tensor([[ 0., 0.],\n", 1196 | " [ 0., 0.]])" 1197 | ] 1198 | }, 1199 | "execution_count": 76, 1200 | "metadata": {}, 1201 | "output_type": "execute_result" 1202 | } 1203 | ], 1204 | "source": [ 1205 | "# Inplace\n", 1206 | "b.div_(a)" 1207 | ] 1208 | }, 1209 | { 1210 | "cell_type": "code", 1211 | "execution_count": 77, 1212 | "metadata": {}, 1213 | "outputs": [ 1214 | { 1215 | "data": { 1216 | "text/plain": [ 1217 | "torch.Size([10])" 1218 | ] 1219 | }, 1220 | "execution_count": 77, 1221 | "metadata": {}, 1222 | "output_type": "execute_result" 1223 | } 1224 | ], 1225 | "source": [ 1226 | "a = torch.Tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n", 1227 | "a.size()" 1228 | ] 1229 | }, 1230 | { 1231 | "cell_type": "code", 1232 | "execution_count": 81, 1233 | "metadata": {}, 1234 | "outputs": [ 1235 | { 1236 | "data": { 1237 | "text/plain": [ 1238 | "tensor(5.5000)" 1239 | ] 1240 | }, 1241 | "execution_count": 81, 1242 | "metadata": {}, 1243 | "output_type": "execute_result" 1244 | } 1245 | ], 1246 | "source": [ 1247 | "a.mean(dim=0)" 1248 | ] 1249 | }, 1250 | { 1251 | "cell_type": "code", 1252 | "execution_count": 82, 1253 | "metadata": {}, 1254 | "outputs": [ 1255 | { 1256 | "ename": "RuntimeError", 1257 | "evalue": "dimension out of range (expected to be in range of [-1, 0], but got 1)", 1258 | "output_type": "error", 1259 | "traceback": [ 1260 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 1261 | "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", 1262 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 1263 | "\u001b[0;31mRuntimeError\u001b[0m: dimension out of range (expected to be in range of [-1, 0], but got 1)" 1264 | ] 1265 | } 1266 | ], 1267 | "source": [ 1268 | "a.mean(dim=1)" 1269 | ] 1270 | }, 1271 | { 1272 | "cell_type": "code", 1273 | "execution_count": 83, 1274 | "metadata": {}, 1275 | "outputs": [], 1276 | "source": [ 1277 | "a = torch.Tensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]])" 1278 | ] 1279 | }, 1280 | { 1281 | "cell_type": "code", 1282 | "execution_count": 84, 1283 | "metadata": {}, 1284 | "outputs": [ 1285 | { 1286 | "data": { 1287 | "text/plain": [ 1288 | "torch.Size([2, 10])" 1289 | ] 1290 | }, 1291 | "execution_count": 84, 1292 | "metadata": {}, 1293 | "output_type": "execute_result" 1294 | } 1295 | ], 1296 | "source": [ 1297 | "a.size()" 1298 | ] 1299 | }, 1300 | { 1301 | "cell_type": "code", 1302 | "execution_count": 87, 1303 | "metadata": {}, 1304 | "outputs": [ 1305 | { 1306 | "data": { 1307 | "text/plain": [ 1308 | "tensor([ 5.5000, 5.5000])" 1309 | ] 1310 | }, 1311 | "execution_count": 87, 1312 | "metadata": {}, 1313 | "output_type": "execute_result" 1314 | } 1315 | ], 1316 | "source": [ 1317 | "a.mean(dim=1)" 1318 | ] 1319 | }, 1320 | { 1321 | "cell_type": "code", 1322 | "execution_count": 88, 1323 | "metadata": {}, 1324 | "outputs": [ 1325 | { 1326 | "data": { 1327 | "text/plain": [ 1328 | "tensor(3.0277)" 1329 | ] 1330 | }, 1331 | "execution_count": 88, 1332 | "metadata": {}, 1333 | "output_type": "execute_result" 1334 | } 1335 | ], 1336 | "source": [ 1337 | "a = torch.Tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n", 1338 | "a.std(dim=0)" 1339 | ] 1340 | }, 1341 | { 1342 | "cell_type": "code", 1343 | "execution_count": null, 1344 | "metadata": {}, 1345 | "outputs": [], 1346 | "source": [] 1347 | } 1348 | ], 1349 | "metadata": { 1350 | "kernelspec": { 1351 | "display_name": "Python 3", 1352 | "language": "python", 1353 | "name": "python3" 1354 | }, 1355 | "language_info": { 1356 | "codemirror_mode": { 1357 | "name": "ipython", 1358 | "version": 3 1359 | }, 1360 | "file_extension": ".py", 1361 | "mimetype": "text/x-python", 1362 | "name": "python", 1363 | "nbconvert_exporter": "python", 1364 | "pygments_lexer": "ipython3", 1365 | "version": "3.6.5" 1366 | } 1367 | }, 1368 | "nbformat": 4, 1369 | "nbformat_minor": 2 1370 | } 1371 | -------------------------------------------------------------------------------- /04. PyTorch Fundamentals - Variables and Gradients.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Variables" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import torch\n", 17 | "from torch.autograd import Variable" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [ 25 | { 26 | "data": { 27 | "text/plain": [ 28 | "tensor([[ 1., 1.],\n", 29 | " [ 1., 1.]])" 30 | ] 31 | }, 32 | "execution_count": 2, 33 | "metadata": {}, 34 | "output_type": "execute_result" 35 | } 36 | ], 37 | "source": [ 38 | "a = Variable(torch.ones(2, 2), requires_grad=True)\n", 39 | "a" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 3, 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/plain": [ 50 | "tensor([[ 1., 1., 1.],\n", 51 | " [ 1., 1., 1.]])" 52 | ] 53 | }, 54 | "execution_count": 3, 55 | "metadata": {}, 56 | "output_type": "execute_result" 57 | } 58 | ], 59 | "source": [ 60 | "# Not a variable\n", 61 | "torch.ones(2, 3)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 4, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "name": "stdout", 71 | "output_type": "stream", 72 | "text": [ 73 | "tensor([[ 2., 2.],\n", 74 | " [ 2., 2.]])\n", 75 | "tensor([[ 2., 2.],\n", 76 | " [ 2., 2.]])\n" 77 | ] 78 | } 79 | ], 80 | "source": [ 81 | "# Behaves similarly to tensors\n", 82 | "b = Variable(torch.ones(2, 2), requires_grad=True)\n", 83 | "print(a + b)\n", 84 | "print(torch.add(a, b))" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 5, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "name": "stdout", 94 | "output_type": "stream", 95 | "text": [ 96 | "tensor([[ 1., 1.],\n", 97 | " [ 1., 1.]])\n", 98 | "tensor([[ 1., 1.],\n", 99 | " [ 1., 1.]])\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "print(a * b)\n", 105 | "print(torch.mul(a, b))" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "# Gradients" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 17, 118 | "metadata": {}, 119 | "outputs": [ 120 | { 121 | "data": { 122 | "text/plain": [ 123 | "tensor([ 1., 1.])" 124 | ] 125 | }, 126 | "execution_count": 17, 127 | "metadata": {}, 128 | "output_type": "execute_result" 129 | } 130 | ], 131 | "source": [ 132 | "x = Variable(torch.ones(2), requires_grad=True)\n", 133 | "x" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 18, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "data": { 143 | "text/plain": [ 144 | "tensor([ 20., 20.])" 145 | ] 146 | }, 147 | "execution_count": 18, 148 | "metadata": {}, 149 | "output_type": "execute_result" 150 | } 151 | ], 152 | "source": [ 153 | "y = 5 * (x + 1) ** 2\n", 154 | "y" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": 19, 160 | "metadata": {}, 161 | "outputs": [ 162 | { 163 | "data": { 164 | "text/plain": [ 165 | "tensor(20.)" 166 | ] 167 | }, 168 | "execution_count": 19, 169 | "metadata": {}, 170 | "output_type": "execute_result" 171 | } 172 | ], 173 | "source": [ 174 | "o = (1/2) * torch.sum(y)\n", 175 | "o" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 13, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "o.backward()" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 14, 190 | "metadata": {}, 191 | "outputs": [ 192 | { 193 | "data": { 194 | "text/plain": [ 195 | "tensor([ 10., 10.])" 196 | ] 197 | }, 198 | "execution_count": 14, 199 | "metadata": {}, 200 | "output_type": "execute_result" 201 | } 202 | ], 203 | "source": [ 204 | "x.grad" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 20, 210 | "metadata": {}, 211 | "outputs": [ 212 | { 213 | "data": { 214 | "text/plain": [ 215 | "tensor([ 10., 10.])" 216 | ] 217 | }, 218 | "execution_count": 20, 219 | "metadata": {}, 220 | "output_type": "execute_result" 221 | } 222 | ], 223 | "source": [ 224 | "# backward in detail\n", 225 | "o.backward(torch.FloatTensor([1.0, 1.0]))\n", 226 | "x.grad" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": null, 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [] 235 | } 236 | ], 237 | "metadata": { 238 | "kernelspec": { 239 | "display_name": "Python 3", 240 | "language": "python", 241 | "name": "python3" 242 | }, 243 | "language_info": { 244 | "codemirror_mode": { 245 | "name": "ipython", 246 | "version": 3 247 | }, 248 | "file_extension": ".py", 249 | "mimetype": "text/x-python", 250 | "name": "python", 251 | "nbconvert_exporter": "python", 252 | "pygments_lexer": "ipython3", 253 | "version": "3.6.5" 254 | } 255 | }, 256 | "nbformat": 4, 257 | "nbformat_minor": 2 258 | } 259 | -------------------------------------------------------------------------------- /08. Convolutional Neural Network (CNN) with PyTorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# CNN in PyTorch" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import torch\n", 17 | "import torch.nn as nn\n", 18 | "import torchvision.transforms as transforms\n", 19 | "import torchvision.datasets as dsets\n", 20 | "from torch.autograd import Variable" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "train_dataset = dsets.MNIST(root='./data',\n", 30 | " train=True,\n", 31 | " transform=transforms.ToTensor(),\n", 32 | " download=True)\n", 33 | "\n", 34 | "test_dataset = dsets.MNIST(root='./data',\n", 35 | " train=False,\n", 36 | " transform=transforms.ToTensor())" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 3, 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "torch.Size([60000, 28, 28])\n" 49 | ] 50 | } 51 | ], 52 | "source": [ 53 | "print(train_dataset.train_data.size())" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 4, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "name": "stdout", 63 | "output_type": "stream", 64 | "text": [ 65 | "torch.Size([60000])\n" 66 | ] 67 | } 68 | ], 69 | "source": [ 70 | "print(train_dataset.train_labels.size())" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 5, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "torch.Size([10000, 28, 28])\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "print(test_dataset.test_data.size())" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 6, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "torch.Size([10000])\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "print(test_dataset.test_labels.size())" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 7, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "batch_size = 100\n", 114 | "n_iters = 3000\n", 115 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 116 | "num_epochs = int(num_epochs)\n", 117 | "\n", 118 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 119 | " batch_size=batch_size,\n", 120 | " shuffle=True)\n", 121 | "\n", 122 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 123 | " batch_size=batch_size,\n", 124 | " shuffle=False)" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 10, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "class CNNModel(nn.Module):\n", 134 | " def __init__(self):\n", 135 | " super(CNNModel, self).__init__()\n", 136 | " \n", 137 | " # Convolution 1\n", 138 | " self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=2)\n", 139 | " self.relu1 = nn.ReLU()\n", 140 | " \n", 141 | " # Max pool 1\n", 142 | " self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n", 143 | " \n", 144 | " # Convolution 2\n", 145 | " self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)\n", 146 | " self.relu2 = nn.ReLU()\n", 147 | " \n", 148 | " # Max pool 2\n", 149 | " self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n", 150 | " \n", 151 | " # Fully connected 1 (readout)\n", 152 | " self.fc1 = nn.Linear(32 * 7 * 7, 10)\n", 153 | " \n", 154 | " def forward(self, x):\n", 155 | " # Convolution 1\n", 156 | " out = self.cnn1(x)\n", 157 | " out = self.relu1(out)\n", 158 | " \n", 159 | " # Max pool 1\n", 160 | " out = self.maxpool1(out)\n", 161 | " \n", 162 | " # Convolution 2\n", 163 | " out = self.cnn2(out)\n", 164 | " out = self.relu2(out)\n", 165 | " \n", 166 | " # Max pool 2\n", 167 | " out = self.maxpool2(out)\n", 168 | " \n", 169 | " # Resize\n", 170 | " # Original size: (100, 32, 7, 7)\n", 171 | " # out.size(0): 100\n", 172 | " # New out size: (100, 32*7*7)\n", 173 | " out = out.view(out.size(0), -1)\n", 174 | " \n", 175 | " # Linear function (readout)\n", 176 | " out = self.fc1(out)\n", 177 | " \n", 178 | " return out" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 11, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "model = CNNModel()" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 12, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "criterion = nn.CrossEntropyLoss()" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 13, 202 | "metadata": {}, 203 | "outputs": [], 204 | "source": [ 205 | "learning_rate = 0.01\n", 206 | "\n", 207 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 14, 213 | "metadata": {}, 214 | "outputs": [ 215 | { 216 | "name": "stdout", 217 | "output_type": "stream", 218 | "text": [ 219 | "\n", 220 | "6\n", 221 | "torch.Size([16, 1, 5, 5])\n", 222 | "torch.Size([16])\n", 223 | "torch.Size([32, 16, 5, 5])\n", 224 | "torch.Size([32])\n", 225 | "torch.Size([10, 1568])\n", 226 | "torch.Size([10])\n" 227 | ] 228 | } 229 | ], 230 | "source": [ 231 | "print(model.parameters())\n", 232 | "\n", 233 | "print(len(list(model.parameters())))\n", 234 | "\n", 235 | "# Convolution 1: 16 Kernels\n", 236 | "print(list(model.parameters())[0].size())\n", 237 | "\n", 238 | "# Convolution 1 Bias: 16 Kernels\n", 239 | "print(list(model.parameters())[1].size())\n", 240 | "\n", 241 | "# Convolution 2: 32 Kernels with depth = 16\n", 242 | "print(list(model.parameters())[2].size())\n", 243 | "\n", 244 | "# Convolution 2 Bias: 32 Kernels with depth = 16\n", 245 | "print(list(model.parameters())[3].size())\n", 246 | "\n", 247 | "# Fully Connected Layer 1\n", 248 | "print(list(model.parameters())[4].size())\n", 249 | "\n", 250 | "# Fully Connected Layer Bias\n", 251 | "print(list(model.parameters())[5].size())" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": 15, 257 | "metadata": {}, 258 | "outputs": [ 259 | { 260 | "name": "stderr", 261 | "output_type": "stream", 262 | "text": [ 263 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:49: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 264 | ] 265 | }, 266 | { 267 | "name": "stdout", 268 | "output_type": "stream", 269 | "text": [ 270 | "Iteration: 500. Loss: 0.22800475358963013. Accuracy: 91\n", 271 | "Iteration: 1000. Loss: 0.16966554522514343. Accuracy: 93\n", 272 | "Iteration: 1500. Loss: 0.12278220057487488. Accuracy: 94\n", 273 | "Iteration: 2000. Loss: 0.21044640243053436. Accuracy: 95\n", 274 | "Iteration: 2500. Loss: 0.09591661393642426. Accuracy: 96\n", 275 | "Iteration: 3000. Loss: 0.08583936095237732. Accuracy: 96\n" 276 | ] 277 | } 278 | ], 279 | "source": [ 280 | "iter = 0\n", 281 | "for epoch in range(num_epochs):\n", 282 | " for i, (images, labels) in enumerate(train_loader):\n", 283 | " # Load images as Variable\n", 284 | " images = Variable(images)\n", 285 | " labels = Variable(labels)\n", 286 | " \n", 287 | " # Clear gradients w.r.t. parameters\n", 288 | " optimizer.zero_grad()\n", 289 | " \n", 290 | " # Forward pass to get output/logits\n", 291 | " outputs = model(images)\n", 292 | " \n", 293 | " # Calculate Loss: softmax --> cross entropy loss\n", 294 | " loss = criterion(outputs, labels)\n", 295 | " \n", 296 | " # Getting gradients w.r.t. parameters\n", 297 | " loss.backward()\n", 298 | " \n", 299 | " # Updating parameters\n", 300 | " optimizer.step()\n", 301 | " \n", 302 | " iter += 1\n", 303 | " \n", 304 | " if iter % 500 == 0:\n", 305 | " # Calculate Accuracy\n", 306 | " correct = 0\n", 307 | " total = 0\n", 308 | " # Iterate through test dataset\n", 309 | " for images, labels in test_loader:\n", 310 | " # Load images to a Torch Variable\n", 311 | " images = Variable(images)\n", 312 | " \n", 313 | " # Forward pass only to get logits/output\n", 314 | " outputs = model(images)\n", 315 | " \n", 316 | " # Get predictions from the maximum value\n", 317 | " _, predicted = torch.max(outputs.data, 1)\n", 318 | " \n", 319 | " # Total number of labels\n", 320 | " total += labels.size(0)\n", 321 | " \n", 322 | " # Total correct predictions\n", 323 | " correct += (predicted == labels).sum()\n", 324 | " \n", 325 | " accuracy = 100 * correct / total\n", 326 | " \n", 327 | " # Print Loss\n", 328 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 329 | ] 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "metadata": {}, 334 | "source": [ 335 | "# More CNN Models in PyTorch" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": 19, 341 | "metadata": {}, 342 | "outputs": [ 343 | { 344 | "name": "stderr", 345 | "output_type": "stream", 346 | "text": [ 347 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:157: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 348 | ] 349 | }, 350 | { 351 | "name": "stdout", 352 | "output_type": "stream", 353 | "text": [ 354 | "Iteration: 500. Loss: 0.5118485689163208. Accuracy: 86\n", 355 | "Iteration: 1000. Loss: 0.29064464569091797. Accuracy: 89\n", 356 | "Iteration: 1500. Loss: 0.373428076505661. Accuracy: 90\n", 357 | "Iteration: 2000. Loss: 0.448984831571579. Accuracy: 91\n", 358 | "Iteration: 2500. Loss: 0.20878082513809204. Accuracy: 91\n", 359 | "Iteration: 3000. Loss: 0.16859890520572662. Accuracy: 92\n" 360 | ] 361 | } 362 | ], 363 | "source": [ 364 | "import torch\n", 365 | "import torch.nn as nn\n", 366 | "import torchvision.transforms as transforms\n", 367 | "import torchvision.datasets as dsets\n", 368 | "from torch.autograd import Variable\n", 369 | "\n", 370 | "'''\n", 371 | "STEP 1: LOADING DATASET\n", 372 | "'''\n", 373 | "\n", 374 | "train_dataset = dsets.MNIST(root='./data',\n", 375 | " train=True,\n", 376 | " transform=transforms.ToTensor(),\n", 377 | " download=True)\n", 378 | "\n", 379 | "test_dataset = dsets.MNIST(root='./data',\n", 380 | " train=False,\n", 381 | " transform=transforms.ToTensor())\n", 382 | "\n", 383 | "'''\n", 384 | "STEP 2: MAKING DATASET ITERABLE\n", 385 | "'''\n", 386 | "\n", 387 | "batch_size = 100\n", 388 | "n_iters = 3000\n", 389 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 390 | "num_epochs = int(num_epochs)\n", 391 | "\n", 392 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 393 | " batch_size=batch_size,\n", 394 | " shuffle=True)\n", 395 | "\n", 396 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 397 | " batch_size=batch_size,\n", 398 | " shuffle=False)\n", 399 | "\n", 400 | "'''\n", 401 | "STEP 3: CREATE MODEL CLASS\n", 402 | "'''\n", 403 | "class CNNModel(nn.Module):\n", 404 | " def __init__(self):\n", 405 | " super(CNNModel, self).__init__()\n", 406 | " \n", 407 | " # Convolution 1\n", 408 | " self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=2)\n", 409 | " self.relu1 = nn.ReLU()\n", 410 | " \n", 411 | " # Average pool 1\n", 412 | " self.avgpool1 = nn.AvgPool2d(kernel_size=2)\n", 413 | " \n", 414 | " # Convolution 2\n", 415 | " self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)\n", 416 | " self.relu2 = nn.ReLU()\n", 417 | " \n", 418 | " # Average pool 2\n", 419 | " self.avgpool2 = nn.AvgPool2d(kernel_size=2)\n", 420 | " \n", 421 | " # Fully connected 1 (readout)\n", 422 | " self.fc1 = nn.Linear(32 * 7 * 7, 10)\n", 423 | " \n", 424 | " def forward(self, x):\n", 425 | " # Convolution 1\n", 426 | " out = self.cnn1(x)\n", 427 | " out = self.relu1(out)\n", 428 | " \n", 429 | " # Average pool 1\n", 430 | " out = self.avgpool1(out)\n", 431 | " \n", 432 | " # Convolution 2\n", 433 | " out = self.cnn2(out)\n", 434 | " out = self.relu2(out)\n", 435 | " \n", 436 | " # Max pool 2\n", 437 | " out = self.avgpool2(out)\n", 438 | " \n", 439 | " # Resize\n", 440 | " # Original size: (100, 32, 7, 7)\n", 441 | " # out.size(0): 100\n", 442 | " # New out size: (100, 32*7*7)\n", 443 | " out = out.view(out.size(0), -1)\n", 444 | " \n", 445 | " # Linear function (readout)\n", 446 | " out = self.fc1(out)\n", 447 | " \n", 448 | " return out\n", 449 | " \n", 450 | "'''\n", 451 | "STEP 4: INSTANTIATE MODEL CLASS\n", 452 | "'''\n", 453 | "\n", 454 | "model = CNNModel()\n", 455 | "\n", 456 | "'''\n", 457 | "STEP 5: INSTANTIATE LOSS CLASS\n", 458 | "'''\n", 459 | "criterion = nn.CrossEntropyLoss()\n", 460 | "\n", 461 | "\n", 462 | "'''\n", 463 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 464 | "'''\n", 465 | "learning_rate = 0.01\n", 466 | "\n", 467 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 468 | "\n", 469 | "'''\n", 470 | "STEP 7: TRAIN THE MODEL\n", 471 | "'''\n", 472 | "iter = 0\n", 473 | "for epoch in range(num_epochs):\n", 474 | " for i, (images, labels) in enumerate(train_loader):\n", 475 | " # Load images as Variable\n", 476 | " images = Variable(images)\n", 477 | " labels = Variable(labels)\n", 478 | " \n", 479 | " # Clear gradients w.r.t. parameters\n", 480 | " optimizer.zero_grad()\n", 481 | " \n", 482 | " # Forward pass to get output/logits\n", 483 | " outputs = model(images)\n", 484 | " \n", 485 | " # Calculate Loss: softmax --> cross entropy loss\n", 486 | " loss = criterion(outputs, labels)\n", 487 | " \n", 488 | " # Getting gradients w.r.t. parameters\n", 489 | " loss.backward()\n", 490 | " \n", 491 | " # Updating parameters\n", 492 | " optimizer.step()\n", 493 | " \n", 494 | " iter += 1\n", 495 | " \n", 496 | " if iter % 500 == 0:\n", 497 | " # Calculate Accuracy\n", 498 | " correct = 0\n", 499 | " total = 0\n", 500 | " # Iterate through test dataset\n", 501 | " for images, labels in test_loader:\n", 502 | " # Load images to a Torch Variable\n", 503 | " images = Variable(images)\n", 504 | " \n", 505 | " # Forward pass only to get logits/output\n", 506 | " outputs = model(images)\n", 507 | " \n", 508 | " # Get predictions from the maximum value\n", 509 | " _, predicted = torch.max(outputs.data, 1)\n", 510 | " \n", 511 | " # Total number of labels\n", 512 | " total += labels.size(0)\n", 513 | " \n", 514 | " # Total correct predictions\n", 515 | " correct += (predicted == labels).sum()\n", 516 | " \n", 517 | " accuracy = 100 * correct / total\n", 518 | " \n", 519 | " # Print Loss\n", 520 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 521 | ] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "execution_count": 20, 526 | "metadata": {}, 527 | "outputs": [ 528 | { 529 | "name": "stderr", 530 | "output_type": "stream", 531 | "text": [ 532 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:157: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 533 | ] 534 | }, 535 | { 536 | "name": "stdout", 537 | "output_type": "stream", 538 | "text": [ 539 | "Iteration: 500. Loss: 0.23876222968101501. Accuracy: 91\n", 540 | "Iteration: 1000. Loss: 0.22581394016742706. Accuracy: 93\n", 541 | "Iteration: 1500. Loss: 0.18389980494976044. Accuracy: 95\n", 542 | "Iteration: 2000. Loss: 0.09902214258909225. Accuracy: 96\n", 543 | "Iteration: 2500. Loss: 0.14246611297130585. Accuracy: 96\n", 544 | "Iteration: 3000. Loss: 0.17715296149253845. Accuracy: 96\n" 545 | ] 546 | } 547 | ], 548 | "source": [ 549 | "import torch\n", 550 | "import torch.nn as nn\n", 551 | "import torchvision.transforms as transforms\n", 552 | "import torchvision.datasets as dsets\n", 553 | "from torch.autograd import Variable\n", 554 | "\n", 555 | "'''\n", 556 | "STEP 1: LOADING DATASET\n", 557 | "'''\n", 558 | "\n", 559 | "train_dataset = dsets.MNIST(root='./data',\n", 560 | " train=True,\n", 561 | " transform=transforms.ToTensor(),\n", 562 | " download=True)\n", 563 | "\n", 564 | "test_dataset = dsets.MNIST(root='./data',\n", 565 | " train=False,\n", 566 | " transform=transforms.ToTensor())\n", 567 | "\n", 568 | "'''\n", 569 | "STEP 2: MAKING DATASET ITERABLE\n", 570 | "'''\n", 571 | "\n", 572 | "batch_size = 100\n", 573 | "n_iters = 3000\n", 574 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 575 | "num_epochs = int(num_epochs)\n", 576 | "\n", 577 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 578 | " batch_size=batch_size,\n", 579 | " shuffle=True)\n", 580 | "\n", 581 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 582 | " batch_size=batch_size,\n", 583 | " shuffle=False)\n", 584 | "\n", 585 | "'''\n", 586 | "STEP 3: CREATE MODEL CLASS\n", 587 | "'''\n", 588 | "class CNNModel(nn.Module):\n", 589 | " def __init__(self):\n", 590 | " super(CNNModel, self).__init__()\n", 591 | " \n", 592 | " # Convolution 1\n", 593 | " self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)\n", 594 | " self.relu1 = nn.ReLU()\n", 595 | " \n", 596 | " # Max pool 1\n", 597 | " self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n", 598 | " \n", 599 | " # Convolution 2\n", 600 | " self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)\n", 601 | " self.relu2 = nn.ReLU()\n", 602 | " \n", 603 | " # Max pool 2\n", 604 | " self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n", 605 | " \n", 606 | " # Fully connected 1 (readout)\n", 607 | " self.fc1 = nn.Linear(32 * 4 * 4, 10)\n", 608 | " \n", 609 | " def forward(self, x):\n", 610 | " # Convolution 1\n", 611 | " out = self.cnn1(x)\n", 612 | " out = self.relu1(out)\n", 613 | " \n", 614 | " # Max pool 1\n", 615 | " out = self.maxpool1(out)\n", 616 | " \n", 617 | " # Convolution 2\n", 618 | " out = self.cnn2(out)\n", 619 | " out = self.relu2(out)\n", 620 | " \n", 621 | " # Max pool 2\n", 622 | " out = self.maxpool2(out)\n", 623 | " \n", 624 | " # Resize\n", 625 | " # Original size: (100, 32, 7, 7)\n", 626 | " # out.size(0): 100\n", 627 | " # New out size: (100, 32*7*7)\n", 628 | " out = out.view(out.size(0), -1)\n", 629 | " \n", 630 | " # Linear function (readout)\n", 631 | " out = self.fc1(out)\n", 632 | " \n", 633 | " return out\n", 634 | "\n", 635 | "'''\n", 636 | "STEP 4: INSTANTIATE MODEL CLASS\n", 637 | "'''\n", 638 | "\n", 639 | "model = CNNModel()\n", 640 | "\n", 641 | "'''\n", 642 | "STEP 5: INSTANTIATE LOSS CLASS\n", 643 | "'''\n", 644 | "criterion = nn.CrossEntropyLoss()\n", 645 | "\n", 646 | "\n", 647 | "'''\n", 648 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 649 | "'''\n", 650 | "learning_rate = 0.01\n", 651 | "\n", 652 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 653 | "\n", 654 | "'''\n", 655 | "STEP 7: TRAIN THE MODEL\n", 656 | "'''\n", 657 | "iter = 0\n", 658 | "for epoch in range(num_epochs):\n", 659 | " for i, (images, labels) in enumerate(train_loader):\n", 660 | " # Load images as Variable\n", 661 | " images = Variable(images)\n", 662 | " labels = Variable(labels)\n", 663 | " \n", 664 | " # Clear gradients w.r.t. parameters\n", 665 | " optimizer.zero_grad()\n", 666 | " \n", 667 | " # Forward pass to get output/logits\n", 668 | " outputs = model(images)\n", 669 | " \n", 670 | " # Calculate Loss: softmax --> cross entropy loss\n", 671 | " loss = criterion(outputs, labels)\n", 672 | " \n", 673 | " # Getting gradients w.r.t. parameters\n", 674 | " loss.backward()\n", 675 | " \n", 676 | " # Updating parameters\n", 677 | " optimizer.step()\n", 678 | " \n", 679 | " iter += 1\n", 680 | " \n", 681 | " if iter % 500 == 0:\n", 682 | " # Calculate Accuracy\n", 683 | " correct = 0\n", 684 | " total = 0\n", 685 | " # Iterate through test dataset\n", 686 | " for images, labels in test_loader:\n", 687 | " # Load images to a Torch Variable\n", 688 | " images = Variable(images)\n", 689 | " \n", 690 | " # Forward pass only to get logits/output\n", 691 | " outputs = model(images)\n", 692 | " \n", 693 | " # Get predictions from the maximum value\n", 694 | " _, predicted = torch.max(outputs.data, 1)\n", 695 | " \n", 696 | " # Total number of labels\n", 697 | " total += labels.size(0)\n", 698 | " \n", 699 | " # Total correct predictions\n", 700 | " correct += (predicted == labels).sum()\n", 701 | " \n", 702 | " accuracy = 100 * correct / total\n", 703 | " \n", 704 | " # Print Loss\n", 705 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 706 | ] 707 | }, 708 | { 709 | "cell_type": "markdown", 710 | "metadata": {}, 711 | "source": [ 712 | "# CNN From CPU to GPU in PyTorch" 713 | ] 714 | }, 715 | { 716 | "cell_type": "code", 717 | "execution_count": 21, 718 | "metadata": {}, 719 | "outputs": [ 720 | { 721 | "name": "stderr", 722 | "output_type": "stream", 723 | "text": [ 724 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:182: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 725 | ] 726 | }, 727 | { 728 | "name": "stdout", 729 | "output_type": "stream", 730 | "text": [ 731 | "Iteration: 500. Loss: 0.3236732482910156. Accuracy: 87\n", 732 | "Iteration: 1000. Loss: 0.2088351994752884. Accuracy: 92\n", 733 | "Iteration: 1500. Loss: 0.20040953159332275. Accuracy: 93\n", 734 | "Iteration: 2000. Loss: 0.18269087374210358. Accuracy: 95\n", 735 | "Iteration: 2500. Loss: 0.12317320704460144. Accuracy: 95\n", 736 | "Iteration: 3000. Loss: 0.11328031867742538. Accuracy: 96\n" 737 | ] 738 | } 739 | ], 740 | "source": [ 741 | "import torch\n", 742 | "import torch.nn as nn\n", 743 | "import torchvision.transforms as transforms\n", 744 | "import torchvision.datasets as dsets\n", 745 | "from torch.autograd import Variable\n", 746 | "\n", 747 | "'''\n", 748 | "STEP 1: LOADING DATASET\n", 749 | "'''\n", 750 | "\n", 751 | "train_dataset = dsets.MNIST(root='./data',\n", 752 | " train=True,\n", 753 | " transform=transforms.ToTensor(),\n", 754 | " download=True)\n", 755 | "\n", 756 | "test_dataset = dsets.MNIST(root='./data',\n", 757 | " train=False,\n", 758 | " transform=transforms.ToTensor())\n", 759 | "\n", 760 | "'''\n", 761 | "STEP 2: MAKING DATASET ITERABLE\n", 762 | "'''\n", 763 | "\n", 764 | "batch_size = 100\n", 765 | "n_iters = 3000\n", 766 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 767 | "num_epochs = int(num_epochs)\n", 768 | "\n", 769 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 770 | " batch_size=batch_size,\n", 771 | " shuffle=True)\n", 772 | "\n", 773 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 774 | " batch_size=batch_size,\n", 775 | " shuffle=False)\n", 776 | "\n", 777 | "'''\n", 778 | "STEP 3: CREATE MODEL CLASS\n", 779 | "'''\n", 780 | "class CNNModel(nn.Module):\n", 781 | " def __init__(self):\n", 782 | " super(CNNModel, self).__init__()\n", 783 | " \n", 784 | " # Convolution 1\n", 785 | " self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)\n", 786 | " self.relu1 = nn.ReLU()\n", 787 | " \n", 788 | " # Max pool 1\n", 789 | " self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n", 790 | " \n", 791 | " # Convolution 2\n", 792 | " self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)\n", 793 | " self.relu2 = nn.ReLU()\n", 794 | " \n", 795 | " # Max pool 2\n", 796 | " self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n", 797 | " \n", 798 | " # Fully connected 1 (readout)\n", 799 | " self.fc1 = nn.Linear(32 * 4 * 4, 10)\n", 800 | " \n", 801 | " def forward(self, x):\n", 802 | " # Convolution 1\n", 803 | " out = self.cnn1(x)\n", 804 | " out = self.relu1(out)\n", 805 | " \n", 806 | " # Max pool 1\n", 807 | " out = self.maxpool1(out)\n", 808 | " \n", 809 | " # Convolution 2\n", 810 | " out = self.cnn2(out)\n", 811 | " out = self.relu2(out)\n", 812 | " \n", 813 | " # Max pool 2\n", 814 | " out = self.maxpool2(out)\n", 815 | " \n", 816 | " # Resize\n", 817 | " # Original size: (100, 32, 7, 7)\n", 818 | " # out.size(0): 100\n", 819 | " # New out size: (100, 32*7*7)\n", 820 | " out = out.view(out.size(0), -1)\n", 821 | " \n", 822 | " # Linear function (readout)\n", 823 | " out = self.fc1(out)\n", 824 | " \n", 825 | " return out\n", 826 | " \n", 827 | "'''\n", 828 | "STEP 4: INSTANTIATE MODEL CLASS\n", 829 | "'''\n", 830 | "\n", 831 | "model = CNNModel()\n", 832 | "\n", 833 | "#######################\n", 834 | "# USE GPU FOR MODEL #\n", 835 | "#######################\n", 836 | "\n", 837 | "if torch.cuda.is_available():\n", 838 | " model.cuda()\n", 839 | " \n", 840 | "'''\n", 841 | "STEP 5: INSTANTIATE LOSS CLASS\n", 842 | "'''\n", 843 | "criterion = nn.CrossEntropyLoss()\n", 844 | "\n", 845 | "\n", 846 | "'''\n", 847 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 848 | "'''\n", 849 | "learning_rate = 0.01\n", 850 | "\n", 851 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 852 | "\n", 853 | "'''\n", 854 | "STEP 7: TRAIN THE MODEL\n", 855 | "'''\n", 856 | "iter = 0\n", 857 | "for epoch in range(num_epochs):\n", 858 | " for i, (images, labels) in enumerate(train_loader):\n", 859 | " \n", 860 | " #######################\n", 861 | " # USE GPU FOR MODEL #\n", 862 | " #######################\n", 863 | " if torch.cuda.is_available():\n", 864 | " images = Variable(images.cuda())\n", 865 | " labels = Variable(labels.cuda())\n", 866 | " else:\n", 867 | " images = Variable(images)\n", 868 | " labels = Variable(labels)\n", 869 | " \n", 870 | " # Clear gradients w.r.t. parameters\n", 871 | " optimizer.zero_grad()\n", 872 | " \n", 873 | " # Forward pass to get output/logits\n", 874 | " outputs = model(images)\n", 875 | " \n", 876 | " # Calculate Loss: softmax --> cross entropy loss\n", 877 | " loss = criterion(outputs, labels)\n", 878 | " \n", 879 | " # Getting gradients w.r.t. parameters\n", 880 | " loss.backward()\n", 881 | " \n", 882 | " # Updating parameters\n", 883 | " optimizer.step()\n", 884 | " \n", 885 | " iter += 1\n", 886 | " \n", 887 | " if iter % 500 == 0:\n", 888 | " # Calculate Accuracy\n", 889 | " correct = 0\n", 890 | " total = 0\n", 891 | " # Iterate through test dataset\n", 892 | " for images, labels in test_loader:\n", 893 | " #######################\n", 894 | " # USE GPU FOR MODEL #\n", 895 | " #######################\n", 896 | " if torch.cuda.is_available():\n", 897 | " images = Variable(images.cuda())\n", 898 | " else:\n", 899 | " images = Variable(images)\n", 900 | " \n", 901 | " # Forward pass only to get logits/output\n", 902 | " outputs = model(images)\n", 903 | " \n", 904 | " # Get predictions from the maximum value\n", 905 | " _, predicted = torch.max(outputs.data, 1)\n", 906 | " \n", 907 | " # Total number of labels\n", 908 | " total += labels.size(0)\n", 909 | " \n", 910 | " #######################\n", 911 | " # USE GPU FOR MODEL #\n", 912 | " #######################\n", 913 | " # Total correct predictions\n", 914 | " if torch.cuda.is_available():\n", 915 | " correct += (predicted.cpu() == labels.cpu()).sum()\n", 916 | " else:\n", 917 | " correct += (predicted == labels).sum()\n", 918 | " \n", 919 | " accuracy = 100 * correct / total\n", 920 | " \n", 921 | " # Print Loss\n", 922 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 923 | ] 924 | }, 925 | { 926 | "cell_type": "code", 927 | "execution_count": null, 928 | "metadata": {}, 929 | "outputs": [], 930 | "source": [] 931 | } 932 | ], 933 | "metadata": { 934 | "kernelspec": { 935 | "display_name": "Python 3", 936 | "language": "python", 937 | "name": "python3" 938 | }, 939 | "language_info": { 940 | "codemirror_mode": { 941 | "name": "ipython", 942 | "version": 3 943 | }, 944 | "file_extension": ".py", 945 | "mimetype": "text/x-python", 946 | "name": "python", 947 | "nbconvert_exporter": "python", 948 | "pygments_lexer": "ipython3", 949 | "version": "3.6.5" 950 | } 951 | }, 952 | "nbformat": 4, 953 | "nbformat_minor": 2 954 | } 955 | -------------------------------------------------------------------------------- /10. Long Short-Term Memory Networks (LSTM).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# LSTM in PyTorch" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import torch\n", 17 | "import torch.nn as nn\n", 18 | "import torchvision.transforms as transforms\n", 19 | "import torchvision.datasets as dsets\n", 20 | "from torch.autograd import Variable" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "train_dataset = dsets.MNIST(root='./data',\n", 30 | " train=True,\n", 31 | " transform=transforms.ToTensor(),\n", 32 | " download=True)\n", 33 | "\n", 34 | "test_dataset = dsets.MNIST(root='./data',\n", 35 | " train=False,\n", 36 | " transform=transforms.ToTensor())" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 3, 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "torch.Size([60000, 28, 28])\n" 49 | ] 50 | } 51 | ], 52 | "source": [ 53 | "print(train_dataset.train_data.size())" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 4, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "name": "stdout", 63 | "output_type": "stream", 64 | "text": [ 65 | "torch.Size([60000])\n" 66 | ] 67 | } 68 | ], 69 | "source": [ 70 | "print(train_dataset.train_labels.size())" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 5, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "torch.Size([10000, 28, 28])\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "print(test_dataset.test_data.size())" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 6, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "torch.Size([10000])\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "print(test_dataset.test_labels.size())" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 7, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "batch_size = 100\n", 114 | "n_iters = 3000\n", 115 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 116 | "num_epochs = int(num_epochs)\n", 117 | "\n", 118 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 119 | " batch_size=batch_size,\n", 120 | " shuffle=True)\n", 121 | "\n", 122 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 123 | " batch_size=batch_size,\n", 124 | " shuffle=False)" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 19, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "class LSTMModel(nn.Module):\n", 134 | " def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n", 135 | " super(LSTMModel, self).__init__()\n", 136 | " # Hidden dimensions\n", 137 | " self.hidden_dim = hidden_dim\n", 138 | " \n", 139 | " # Number of hidden layers\n", 140 | " self.layer_dim = layer_dim\n", 141 | " \n", 142 | " # Building your LSTM\n", 143 | " # batch_first=True causes input/output tensors to be of shape\n", 144 | " # (batch_dim, seq_dim, feature_dim)\n", 145 | " self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n", 146 | " \n", 147 | " # Readout layer\n", 148 | " self.fc = nn.Linear(hidden_dim, output_dim)\n", 149 | " \n", 150 | " def forward(self, x):\n", 151 | " # Initialize hidden state with zeros\n", 152 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 153 | " \n", 154 | " # Initialize cell state\n", 155 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 156 | " \n", 157 | " # 28 time steps\n", 158 | " out, (hn, cn) = self.lstm(x, (h0,c0))\n", 159 | " \n", 160 | " # Index hidden state of last time step\n", 161 | " # out.size() --> 100, 28, 100\n", 162 | " # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!\n", 163 | " out = self.fc(out[:, -1, :])\n", 164 | " # out.size() --> 100, 10\n", 165 | " return out" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 20, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "input_dim = 28\n", 175 | "hidden_dim = 100\n", 176 | "layer_dim = 1\n", 177 | "output_dim = 10" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 21, 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 22, 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "criterion = nn.CrossEntropyLoss()" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 23, 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "learning_rate = 0.1\n", 205 | "\n", 206 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 24, 212 | "metadata": {}, 213 | "outputs": [ 214 | { 215 | "data": { 216 | "text/plain": [ 217 | "6" 218 | ] 219 | }, 220 | "execution_count": 24, 221 | "metadata": {}, 222 | "output_type": "execute_result" 223 | } 224 | ], 225 | "source": [ 226 | "len(list(model.parameters()))" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": 25, 232 | "metadata": {}, 233 | "outputs": [ 234 | { 235 | "name": "stdout", 236 | "output_type": "stream", 237 | "text": [ 238 | "torch.Size([400, 28])\n", 239 | "torch.Size([400, 100])\n", 240 | "torch.Size([400])\n", 241 | "torch.Size([400])\n", 242 | "torch.Size([10, 100])\n", 243 | "torch.Size([10])\n" 244 | ] 245 | } 246 | ], 247 | "source": [ 248 | "for i in range(len(list(model.parameters()))):\n", 249 | " print(list(model.parameters())[i].size())" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 26, 255 | "metadata": {}, 256 | "outputs": [ 257 | { 258 | "name": "stderr", 259 | "output_type": "stream", 260 | "text": [ 261 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:53: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 262 | ] 263 | }, 264 | { 265 | "name": "stdout", 266 | "output_type": "stream", 267 | "text": [ 268 | "Iteration: 500. Loss: 2.2820582389831543. Accuracy: 19\n", 269 | "Iteration: 1000. Loss: 1.035079836845398. Accuracy: 64\n", 270 | "Iteration: 1500. Loss: 0.5265297889709473. Accuracy: 83\n", 271 | "Iteration: 2000. Loss: 0.24867504835128784. Accuracy: 92\n", 272 | "Iteration: 2500. Loss: 0.3007533550262451. Accuracy: 93\n", 273 | "Iteration: 3000. Loss: 0.13653035461902618. Accuracy: 95\n" 274 | ] 275 | } 276 | ], 277 | "source": [ 278 | "# Number of steps to unroll\n", 279 | "seq_dim = 28\n", 280 | "\n", 281 | "iter = 0\n", 282 | "for epoch in range(num_epochs):\n", 283 | " for i, (images, labels) in enumerate(train_loader):\n", 284 | " # Load images as Variable\n", 285 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 286 | " labels = Variable(labels)\n", 287 | " \n", 288 | " # Clear gradients w.r.t. parameters\n", 289 | " optimizer.zero_grad()\n", 290 | " \n", 291 | " # Forward pass to get output/logits\n", 292 | " # outputs.size() --> 100, 10\n", 293 | " outputs = model(images)\n", 294 | " \n", 295 | " # Calculate Loss: softmax --> cross entropy loss\n", 296 | " loss = criterion(outputs, labels)\n", 297 | " \n", 298 | " # Getting gradients w.r.t. parameters\n", 299 | " loss.backward()\n", 300 | " \n", 301 | " # Updating parameters\n", 302 | " optimizer.step()\n", 303 | " \n", 304 | " iter += 1\n", 305 | " \n", 306 | " if iter % 500 == 0:\n", 307 | " # Calculate Accuracy\n", 308 | " correct = 0\n", 309 | " total = 0\n", 310 | " # Iterate through test dataset\n", 311 | " for images, labels in test_loader:\n", 312 | " # Load images to a Torch Variable\n", 313 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 314 | " \n", 315 | " # Forward pass only to get logits/output\n", 316 | " outputs = model(images)\n", 317 | " \n", 318 | " # Get predictions from the maximum value\n", 319 | " _, predicted = torch.max(outputs.data, 1)\n", 320 | " \n", 321 | " # Total number of labels\n", 322 | " total += labels.size(0)\n", 323 | " \n", 324 | " # Total correct predictions\n", 325 | " correct += (predicted == labels).sum()\n", 326 | " \n", 327 | " accuracy = 100 * correct / total\n", 328 | " \n", 329 | " # Print Loss\n", 330 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 331 | ] 332 | }, 333 | { 334 | "cell_type": "markdown", 335 | "metadata": {}, 336 | "source": [ 337 | "# More LSTM Models in PyTorch" 338 | ] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": 27, 343 | "metadata": {}, 344 | "outputs": [ 345 | { 346 | "name": "stdout", 347 | "output_type": "stream", 348 | "text": [ 349 | "LSTMModel(\n", 350 | " (lstm): LSTM(28, 100, num_layers=2, batch_first=True)\n", 351 | " (fc): Linear(in_features=100, out_features=10, bias=True)\n", 352 | ")\n", 353 | "10\n", 354 | "torch.Size([400, 28])\n", 355 | "torch.Size([400, 100])\n", 356 | "torch.Size([400])\n", 357 | "torch.Size([400])\n", 358 | "torch.Size([400, 100])\n", 359 | "torch.Size([400, 100])\n", 360 | "torch.Size([400])\n", 361 | "torch.Size([400])\n", 362 | "torch.Size([10, 100])\n", 363 | "torch.Size([10])\n" 364 | ] 365 | }, 366 | { 367 | "name": "stderr", 368 | "output_type": "stream", 369 | "text": [ 370 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:158: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 371 | ] 372 | }, 373 | { 374 | "name": "stdout", 375 | "output_type": "stream", 376 | "text": [ 377 | "Iteration: 500. Loss: 2.2961795330047607. Accuracy: 11\n", 378 | "Iteration: 1000. Loss: 2.0218091011047363. Accuracy: 24\n", 379 | "Iteration: 1500. Loss: 0.77486252784729. Accuracy: 78\n", 380 | "Iteration: 2000. Loss: 0.22417789697647095. Accuracy: 91\n", 381 | "Iteration: 2500. Loss: 0.29147088527679443. Accuracy: 94\n", 382 | "Iteration: 3000. Loss: 0.11946485191583633. Accuracy: 95\n" 383 | ] 384 | } 385 | ], 386 | "source": [ 387 | "import torch\n", 388 | "import torch.nn as nn\n", 389 | "import torchvision.transforms as transforms\n", 390 | "import torchvision.datasets as dsets\n", 391 | "from torch.autograd import Variable\n", 392 | "\n", 393 | "'''\n", 394 | "STEP 1: LOADING DATASET\n", 395 | "'''\n", 396 | "train_dataset = dsets.MNIST(root='./data',\n", 397 | " train=True,\n", 398 | " transform=transforms.ToTensor(),\n", 399 | " download=True)\n", 400 | "\n", 401 | "test_dataset = dsets.MNIST(root='./data',\n", 402 | " train=False,\n", 403 | " transform=transforms.ToTensor())\n", 404 | "\n", 405 | "'''\n", 406 | "STEP 2: MAKING DATASET ITERABLE\n", 407 | "'''\n", 408 | "\n", 409 | "batch_size = 100\n", 410 | "n_iters = 3000\n", 411 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 412 | "num_epochs = int(num_epochs)\n", 413 | "\n", 414 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 415 | " batch_size=batch_size,\n", 416 | " shuffle=True)\n", 417 | "\n", 418 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 419 | " batch_size=batch_size,\n", 420 | " shuffle=False)\n", 421 | "\n", 422 | "'''\n", 423 | "STEP 3: CREATE MODEL CLASS\n", 424 | "'''\n", 425 | "\n", 426 | "class LSTMModel(nn.Module):\n", 427 | " def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n", 428 | " super(LSTMModel, self).__init__()\n", 429 | " # Hidden dimensions\n", 430 | " self.hidden_dim = hidden_dim\n", 431 | " \n", 432 | " # Number of hidden layers\n", 433 | " self.layer_dim = layer_dim\n", 434 | " \n", 435 | " # Building your LSTM\n", 436 | " # batch_first=True causes input/output tensors to be of shape\n", 437 | " # (batch_dim, seq_dim, feature_dim)\n", 438 | " self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n", 439 | " \n", 440 | " # Readout layer\n", 441 | " self.fc = nn.Linear(hidden_dim, output_dim)\n", 442 | " \n", 443 | " def forward(self, x):\n", 444 | " # Initialize hidden state with zeros\n", 445 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 446 | " \n", 447 | " # Initialize cell state\n", 448 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 449 | " \n", 450 | " # One time step\n", 451 | " out, (hn, cn) = self.lstm(x, (h0,c0))\n", 452 | " \n", 453 | " # Index hidden state of last time step\n", 454 | " # out.size() --> 100, 28, 100\n", 455 | " # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!\n", 456 | " out = self.fc(out[:, -1, :])\n", 457 | " # out.size() --> 100, 10\n", 458 | " return out\n", 459 | "\n", 460 | "'''\n", 461 | "STEP 4: INSTANTIATE MODEL CLASS\n", 462 | "'''\n", 463 | "input_dim = 28\n", 464 | "hidden_dim = 100\n", 465 | "layer_dim = 2 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\n", 466 | "output_dim = 10\n", 467 | "\n", 468 | "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n", 469 | "\n", 470 | "# JUST PRINTING MODEL & PARAMETERS\n", 471 | "print(model)\n", 472 | "print(len(list(model.parameters())))\n", 473 | "for i in range(len(list(model.parameters()))):\n", 474 | " print(list(model.parameters())[i].size())\n", 475 | "\n", 476 | "'''\n", 477 | "STEP 5: INSTANTIATE LOSS CLASS\n", 478 | "'''\n", 479 | "criterion = nn.CrossEntropyLoss()\n", 480 | "\n", 481 | "'''\n", 482 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 483 | "'''\n", 484 | "learning_rate = 0.1\n", 485 | "\n", 486 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 487 | "\n", 488 | "'''\n", 489 | "STEP 7: TRAIN THE MODEL\n", 490 | "'''\n", 491 | "\n", 492 | "# Number of steps to unroll\n", 493 | "seq_dim = 28\n", 494 | "\n", 495 | "iter = 0\n", 496 | "for epoch in range(num_epochs):\n", 497 | " for i, (images, labels) in enumerate(train_loader):\n", 498 | " # Load images as Variable\n", 499 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 500 | " labels = Variable(labels)\n", 501 | " \n", 502 | " # Clear gradients w.r.t. parameters\n", 503 | " optimizer.zero_grad()\n", 504 | " \n", 505 | " # Forward pass to get output/logits\n", 506 | " # outputs.size() --> 100, 10\n", 507 | " outputs = model(images)\n", 508 | " \n", 509 | " # Calculate Loss: softmax --> cross entropy loss\n", 510 | " loss = criterion(outputs, labels)\n", 511 | " \n", 512 | " # Getting gradients w.r.t. parameters\n", 513 | " loss.backward()\n", 514 | " \n", 515 | " # Updating parameters\n", 516 | " optimizer.step()\n", 517 | " \n", 518 | " iter += 1\n", 519 | " \n", 520 | " if iter % 500 == 0:\n", 521 | " # Calculate Accuracy\n", 522 | " correct = 0\n", 523 | " total = 0\n", 524 | " # Iterate through test dataset\n", 525 | " for images, labels in test_loader:\n", 526 | " # Load images to a Torch Variable\n", 527 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 528 | " \n", 529 | " # Forward pass only to get logits/output\n", 530 | " outputs = model(images)\n", 531 | " \n", 532 | " # Get predictions from the maximum value\n", 533 | " _, predicted = torch.max(outputs.data, 1)\n", 534 | " \n", 535 | " # Total number of labels\n", 536 | " total += labels.size(0)\n", 537 | " \n", 538 | " # Total correct predictions\n", 539 | " correct += (predicted == labels).sum()\n", 540 | " \n", 541 | " accuracy = 100 * correct / total\n", 542 | " \n", 543 | " # Print Loss\n", 544 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 545 | ] 546 | }, 547 | { 548 | "cell_type": "code", 549 | "execution_count": 30, 550 | "metadata": {}, 551 | "outputs": [ 552 | { 553 | "name": "stdout", 554 | "output_type": "stream", 555 | "text": [ 556 | "LSTMModel(\n", 557 | " (lstm): LSTM(28, 100, num_layers=3, batch_first=True)\n", 558 | " (fc): Linear(in_features=100, out_features=10, bias=True)\n", 559 | ")\n", 560 | "14\n", 561 | "torch.Size([400, 28])\n", 562 | "torch.Size([400, 100])\n", 563 | "torch.Size([400])\n", 564 | "torch.Size([400])\n", 565 | "torch.Size([400, 100])\n", 566 | "torch.Size([400, 100])\n", 567 | "torch.Size([400])\n", 568 | "torch.Size([400])\n", 569 | "torch.Size([400, 100])\n", 570 | "torch.Size([400, 100])\n", 571 | "torch.Size([400])\n", 572 | "torch.Size([400])\n", 573 | "torch.Size([10, 100])\n", 574 | "torch.Size([10])\n" 575 | ] 576 | }, 577 | { 578 | "name": "stderr", 579 | "output_type": "stream", 580 | "text": [ 581 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:158: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 582 | ] 583 | }, 584 | { 585 | "name": "stdout", 586 | "output_type": "stream", 587 | "text": [ 588 | "Iteration: 500. Loss: 2.3062610626220703. Accuracy: 11\n", 589 | "Iteration: 1000. Loss: 2.294701099395752. Accuracy: 11\n", 590 | "Iteration: 1500. Loss: 2.0946013927459717. Accuracy: 23\n", 591 | "Iteration: 2000. Loss: 0.9512524604797363. Accuracy: 62\n", 592 | "Iteration: 2500. Loss: 0.6940139532089233. Accuracy: 79\n", 593 | "Iteration: 3000. Loss: 0.39204394817352295. Accuracy: 82\n" 594 | ] 595 | } 596 | ], 597 | "source": [ 598 | "import torch\n", 599 | "import torch.nn as nn\n", 600 | "import torchvision.transforms as transforms\n", 601 | "import torchvision.datasets as dsets\n", 602 | "from torch.autograd import Variable\n", 603 | "\n", 604 | "'''\n", 605 | "STEP 1: LOADING DATASET\n", 606 | "'''\n", 607 | "train_dataset = dsets.MNIST(root='./data',\n", 608 | " train=True,\n", 609 | " transform=transforms.ToTensor(),\n", 610 | " download=True)\n", 611 | "\n", 612 | "test_dataset = dsets.MNIST(root='./data',\n", 613 | " train=False,\n", 614 | " transform=transforms.ToTensor())\n", 615 | "\n", 616 | "'''\n", 617 | "STEP 2: MAKING DATASET ITERABLE\n", 618 | "'''\n", 619 | "\n", 620 | "batch_size = 100\n", 621 | "n_iters = 3000\n", 622 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 623 | "num_epochs = int(num_epochs)\n", 624 | "\n", 625 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 626 | " batch_size=batch_size,\n", 627 | " shuffle=True)\n", 628 | "\n", 629 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 630 | " batch_size=batch_size,\n", 631 | " shuffle=False)\n", 632 | "\n", 633 | "'''\n", 634 | "STEP 3: CREATE MODEL CLASS\n", 635 | "'''\n", 636 | "\n", 637 | "class LSTMModel(nn.Module):\n", 638 | " def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n", 639 | " super(LSTMModel, self).__init__()\n", 640 | " # Hidden dimensions\n", 641 | " self.hidden_dim = hidden_dim\n", 642 | " \n", 643 | " # Number of hidden layers\n", 644 | " self.layer_dim = layer_dim\n", 645 | " \n", 646 | " # Building your LSTM\n", 647 | " # batch_first=True causes input/output tensors to be of shape\n", 648 | " # (batch_dim, seq_dim, feature_dim)\n", 649 | " self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n", 650 | " \n", 651 | " # Readout layer\n", 652 | " self.fc = nn.Linear(hidden_dim, output_dim)\n", 653 | " \n", 654 | " def forward(self, x):\n", 655 | " # Initialize hidden state with zeros\n", 656 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 657 | " \n", 658 | " # Initialize cell state\n", 659 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 660 | " \n", 661 | " # One time step\n", 662 | " out, (hn, cn) = self.lstm(x, (h0,c0))\n", 663 | " \n", 664 | " # Index hidden state of last time step\n", 665 | " # out.size() --> 100, 28, 100\n", 666 | " # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!\n", 667 | " out = self.fc(out[:, -1, :])\n", 668 | " # out.size() --> 100, 10\n", 669 | " return out\n", 670 | "\n", 671 | "'''\n", 672 | "STEP 4: INSTANTIATE MODEL CLASS\n", 673 | "'''\n", 674 | "input_dim = 28\n", 675 | "hidden_dim = 100\n", 676 | "layer_dim = 3 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\n", 677 | "output_dim = 10\n", 678 | "\n", 679 | "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n", 680 | "\n", 681 | "# JUST PRINTING MODEL & PARAMETERS\n", 682 | "print(model)\n", 683 | "print(len(list(model.parameters())))\n", 684 | "for i in range(len(list(model.parameters()))):\n", 685 | " print(list(model.parameters())[i].size())\n", 686 | "\n", 687 | "'''\n", 688 | "STEP 5: INSTANTIATE LOSS CLASS\n", 689 | "'''\n", 690 | "criterion = nn.CrossEntropyLoss()\n", 691 | "\n", 692 | "'''\n", 693 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 694 | "'''\n", 695 | "learning_rate = 0.1\n", 696 | "\n", 697 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 698 | "\n", 699 | "'''\n", 700 | "STEP 7: TRAIN THE MODEL\n", 701 | "'''\n", 702 | "\n", 703 | "# Number of steps to unroll\n", 704 | "seq_dim = 28\n", 705 | "\n", 706 | "iter = 0\n", 707 | "for epoch in range(num_epochs):\n", 708 | " for i, (images, labels) in enumerate(train_loader):\n", 709 | " # Load images as Variable\n", 710 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 711 | " labels = Variable(labels)\n", 712 | " \n", 713 | " # Clear gradients w.r.t. parameters\n", 714 | " optimizer.zero_grad()\n", 715 | " \n", 716 | " # Forward pass to get output/logits\n", 717 | " # outputs.size() --> 100, 10\n", 718 | " outputs = model(images)\n", 719 | " \n", 720 | " # Calculate Loss: softmax --> cross entropy loss\n", 721 | " loss = criterion(outputs, labels)\n", 722 | " \n", 723 | " # Getting gradients w.r.t. parameters\n", 724 | " loss.backward()\n", 725 | " \n", 726 | " # Updating parameters\n", 727 | " optimizer.step()\n", 728 | " \n", 729 | " iter += 1\n", 730 | " \n", 731 | " if iter % 500 == 0:\n", 732 | " # Calculate Accuracy\n", 733 | " correct = 0\n", 734 | " total = 0\n", 735 | " # Iterate through test dataset\n", 736 | " for images, labels in test_loader:\n", 737 | " # Load images to a Torch Variable\n", 738 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 739 | " \n", 740 | " # Forward pass only to get logits/output\n", 741 | " outputs = model(images)\n", 742 | " \n", 743 | " # Get predictions from the maximum value\n", 744 | " _, predicted = torch.max(outputs.data, 1)\n", 745 | " \n", 746 | " # Total number of labels\n", 747 | " total += labels.size(0)\n", 748 | " \n", 749 | " # Total correct predictions\n", 750 | " correct += (predicted == labels).sum()\n", 751 | " \n", 752 | " accuracy = 100 * correct / total\n", 753 | " \n", 754 | " # Print Loss\n", 755 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 756 | ] 757 | }, 758 | { 759 | "cell_type": "markdown", 760 | "metadata": {}, 761 | "source": [ 762 | "# LSTM From CPU to GPU in PyTorch" 763 | ] 764 | }, 765 | { 766 | "cell_type": "code", 767 | "execution_count": 31, 768 | "metadata": {}, 769 | "outputs": [ 770 | { 771 | "name": "stderr", 772 | "output_type": "stream", 773 | "text": [ 774 | "/Users/robertlowe/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:187: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n" 775 | ] 776 | }, 777 | { 778 | "name": "stdout", 779 | "output_type": "stream", 780 | "text": [ 781 | "Iteration: 500. Loss: 2.306875705718994. Accuracy: 11\n", 782 | "Iteration: 1000. Loss: 2.3024940490722656. Accuracy: 11\n", 783 | "Iteration: 1500. Loss: 2.241243839263916. Accuracy: 19\n", 784 | "Iteration: 2000. Loss: 1.0950016975402832. Accuracy: 62\n", 785 | "Iteration: 2500. Loss: 0.5713849067687988. Accuracy: 85\n", 786 | "Iteration: 3000. Loss: 0.4543074369430542. Accuracy: 91\n" 787 | ] 788 | } 789 | ], 790 | "source": [ 791 | "import torch\n", 792 | "import torch.nn as nn\n", 793 | "import torchvision.transforms as transforms\n", 794 | "import torchvision.datasets as dsets\n", 795 | "from torch.autograd import Variable\n", 796 | "\n", 797 | "'''\n", 798 | "STEP 1: LOADING DATASET\n", 799 | "'''\n", 800 | "train_dataset = dsets.MNIST(root='./data',\n", 801 | " train=True,\n", 802 | " transform=transforms.ToTensor(),\n", 803 | " download=True)\n", 804 | "\n", 805 | "test_dataset = dsets.MNIST(root='./data',\n", 806 | " train=False,\n", 807 | " transform=transforms.ToTensor())\n", 808 | "\n", 809 | "'''\n", 810 | "STEP 2: MAKING DATASET ITERABLE\n", 811 | "'''\n", 812 | "\n", 813 | "batch_size = 100\n", 814 | "n_iters = 3000\n", 815 | "num_epochs = n_iters / (len(train_dataset) / batch_size)\n", 816 | "num_epochs = int(num_epochs)\n", 817 | "\n", 818 | "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n", 819 | " batch_size=batch_size,\n", 820 | " shuffle=True)\n", 821 | "\n", 822 | "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n", 823 | " batch_size=batch_size,\n", 824 | " shuffle=False)\n", 825 | "\n", 826 | "'''\n", 827 | "STEP 3: CREATE MODEL CLASS\n", 828 | "'''\n", 829 | "\n", 830 | "class LSTMModel(nn.Module):\n", 831 | " def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n", 832 | " super(LSTMModel, self).__init__()\n", 833 | " # Hidden dimensions\n", 834 | " self.hidden_dim = hidden_dim\n", 835 | " \n", 836 | " # Number of hidden layers\n", 837 | " self.layer_dim = layer_dim\n", 838 | " \n", 839 | " # Building your LSTM\n", 840 | " # batch_first=True causes input/output tensors to be of shape\n", 841 | " # (batch_dim, seq_dim, feature_dim)\n", 842 | " self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n", 843 | " \n", 844 | " # Readout layer\n", 845 | " self.fc = nn.Linear(hidden_dim, output_dim)\n", 846 | " \n", 847 | " def forward(self, x):\n", 848 | " # Initialize hidden state with zeros\n", 849 | " #######################\n", 850 | " # USE GPU FOR MODEL #\n", 851 | " #######################\n", 852 | " \n", 853 | " if torch.cuda.is_available():\n", 854 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n", 855 | " else:\n", 856 | " h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 857 | " \n", 858 | " # Initialize cell state\n", 859 | " if torch.cuda.is_available():\n", 860 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n", 861 | " else:\n", 862 | " c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n", 863 | " \n", 864 | " # One time step\n", 865 | " out, (hn, cn) = self.lstm(x, (h0,c0))\n", 866 | " \n", 867 | " # Index hidden state of last time step\n", 868 | " # out.size() --> 100, 28, 100\n", 869 | " # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!\n", 870 | " out = self.fc(out[:, -1, :])\n", 871 | " # out.size() --> 100, 10\n", 872 | " return out\n", 873 | "\n", 874 | "'''\n", 875 | "STEP 4: INSTANTIATE MODEL CLASS\n", 876 | "'''\n", 877 | "input_dim = 28\n", 878 | "hidden_dim = 100\n", 879 | "layer_dim = 3 # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\n", 880 | "output_dim = 10\n", 881 | "\n", 882 | "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n", 883 | "\n", 884 | "#######################\n", 885 | "# USE GPU FOR MODEL #\n", 886 | "#######################\n", 887 | "\n", 888 | "if torch.cuda.is_available():\n", 889 | " model.cuda()\n", 890 | "\n", 891 | "'''\n", 892 | "STEP 5: INSTANTIATE LOSS CLASS\n", 893 | "'''\n", 894 | "criterion = nn.CrossEntropyLoss()\n", 895 | "\n", 896 | "'''\n", 897 | "STEP 6: INSTANTIATE OPTIMIZER CLASS\n", 898 | "'''\n", 899 | "learning_rate = 0.1\n", 900 | "\n", 901 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 902 | "\n", 903 | "'''\n", 904 | "STEP 7: TRAIN THE MODEL\n", 905 | "'''\n", 906 | "\n", 907 | "# Number of steps to unroll\n", 908 | "seq_dim = 28\n", 909 | "\n", 910 | "iter = 0\n", 911 | "for epoch in range(num_epochs):\n", 912 | " for i, (images, labels) in enumerate(train_loader):\n", 913 | " # Load images as Variable\n", 914 | " #######################\n", 915 | " # USE GPU FOR MODEL #\n", 916 | " #######################\n", 917 | " if torch.cuda.is_available():\n", 918 | " images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n", 919 | " labels = Variable(labels.cuda())\n", 920 | " else:\n", 921 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 922 | " labels = Variable(labels)\n", 923 | " \n", 924 | " # Clear gradients w.r.t. parameters\n", 925 | " optimizer.zero_grad()\n", 926 | " \n", 927 | " # Forward pass to get output/logits\n", 928 | " # outputs.size() --> 100, 10\n", 929 | " outputs = model(images)\n", 930 | " \n", 931 | " # Calculate Loss: softmax --> cross entropy loss\n", 932 | " loss = criterion(outputs, labels)\n", 933 | " \n", 934 | " # Getting gradients w.r.t. parameters\n", 935 | " loss.backward()\n", 936 | " \n", 937 | " # Updating parameters\n", 938 | " optimizer.step()\n", 939 | " \n", 940 | " iter += 1\n", 941 | " \n", 942 | " if iter % 500 == 0:\n", 943 | " # Calculate Accuracy\n", 944 | " correct = 0\n", 945 | " total = 0\n", 946 | " # Iterate through test dataset\n", 947 | " for images, labels in test_loader:\n", 948 | " #######################\n", 949 | " # USE GPU FOR MODEL #\n", 950 | " #######################\n", 951 | " if torch.cuda.is_available():\n", 952 | " images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n", 953 | " else:\n", 954 | " images = Variable(images.view(-1, seq_dim, input_dim))\n", 955 | " \n", 956 | " # Forward pass only to get logits/output\n", 957 | " outputs = model(images)\n", 958 | " \n", 959 | " # Get predictions from the maximum value\n", 960 | " _, predicted = torch.max(outputs.data, 1)\n", 961 | " \n", 962 | " # Total number of labels\n", 963 | " total += labels.size(0)\n", 964 | " \n", 965 | " # Total correct predictions\n", 966 | " #######################\n", 967 | " # USE GPU FOR MODEL #\n", 968 | " #######################\n", 969 | " if torch.cuda.is_available():\n", 970 | " correct += (predicted.cpu() == labels.cpu()),sum()\n", 971 | " else:\n", 972 | " correct += (predicted == labels).sum()\n", 973 | " \n", 974 | " accuracy = 100 * correct / total\n", 975 | " \n", 976 | " # Print Loss\n", 977 | " print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))" 978 | ] 979 | }, 980 | { 981 | "cell_type": "code", 982 | "execution_count": null, 983 | "metadata": {}, 984 | "outputs": [], 985 | "source": [] 986 | } 987 | ], 988 | "metadata": { 989 | "kernelspec": { 990 | "display_name": "Python 3", 991 | "language": "python", 992 | "name": "python3" 993 | }, 994 | "language_info": { 995 | "codemirror_mode": { 996 | "name": "ipython", 997 | "version": 3 998 | }, 999 | "file_extension": ".py", 1000 | "mimetype": "text/x-python", 1001 | "name": "python", 1002 | "nbconvert_exporter": "python", 1003 | "pygments_lexer": "ipython3", 1004 | "version": "3.6.5" 1005 | } 1006 | }, 1007 | "nbformat": 4, 1008 | "nbformat_minor": 2 1009 | } 1010 | -------------------------------------------------------------------------------- /data/processed/test.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmlowe/practical-deep-learning-with-pytorch/34ffaf164b0ced615d55605f25d6dfef1a58516f/data/processed/test.pt -------------------------------------------------------------------------------- /data/processed/training.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmlowe/practical-deep-learning-with-pytorch/34ffaf164b0ced615d55605f25d6dfef1a58516f/data/processed/training.pt -------------------------------------------------------------------------------- /data/raw/t10k-images-idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmlowe/practical-deep-learning-with-pytorch/34ffaf164b0ced615d55605f25d6dfef1a58516f/data/raw/t10k-images-idx3-ubyte -------------------------------------------------------------------------------- /data/raw/t10k-labels-idx1-ubyte: -------------------------------------------------------------------------------- 1 | '                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             -------------------------------------------------------------------------------- /data/raw/train-images-idx3-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmlowe/practical-deep-learning-with-pytorch/34ffaf164b0ced615d55605f25d6dfef1a58516f/data/raw/train-images-idx3-ubyte -------------------------------------------------------------------------------- /data/raw/train-labels-idx1-ubyte: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rmlowe/practical-deep-learning-with-pytorch/34ffaf164b0ced615d55605f25d6dfef1a58516f/data/raw/train-labels-idx1-ubyte --------------------------------------------------------------------------------