├── 9781484230954 ├── Chapter2.ipynb ├── Chapter3.ipynb ├── Chapter4.ipynb ├── Chapter5.ipynb └── Chapter6.ipynb ├── LICENSE.txt ├── README.md └── contributing.md /9781484230954/Chapter2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Listing 2-1. Import TensorFlow and numpy Library" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "metadata": { 17 | "collapsed": true, 18 | "deletable": true, 19 | "editable": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "import tensorflow as tf\n", 24 | "import numpy as np\n" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": { 30 | "deletable": true, 31 | "editable": true 32 | }, 33 | "source": [ 34 | "# Listing 2-2.Activate a TensorFlow Interactive Session " 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "metadata": { 41 | "collapsed": false, 42 | "deletable": true, 43 | "editable": true 44 | }, 45 | "outputs": [ 46 | { 47 | "data": { 48 | "text/plain": [ 49 | "" 50 | ] 51 | }, 52 | "execution_count": 2, 53 | "metadata": {}, 54 | "output_type": "execute_result" 55 | } 56 | ], 57 | "source": [ 58 | "tf.InteractiveSession() " 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": { 64 | "deletable": true, 65 | "editable": true 66 | }, 67 | "source": [ 68 | "# Listing 2-3. Defining Tensors " 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": { 75 | "collapsed": true, 76 | "deletable": true, 77 | "editable": true 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "a = tf.zeros((2,2));\n", 82 | "b = tf.ones((2,2)) " 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": { 88 | "deletable": true, 89 | "editable": true 90 | }, 91 | "source": [ 92 | "# Listing 2-4. Sum the elements of the matrix (2D Tensor) across the horizontal axis " 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 4, 98 | "metadata": { 99 | "collapsed": false, 100 | "deletable": true, 101 | "editable": true 102 | }, 103 | "outputs": [ 104 | { 105 | "data": { 106 | "text/plain": [ 107 | "array([ 2., 2.], dtype=float32)" 108 | ] 109 | }, 110 | "execution_count": 4, 111 | "metadata": {}, 112 | "output_type": "execute_result" 113 | } 114 | ], 115 | "source": [ 116 | "tf.reduce_sum(b,reduction_indices = 1).eval() " 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": { 122 | "deletable": true, 123 | "editable": true 124 | }, 125 | "source": [ 126 | "# Listing 2-5. Check the shape of the Tensor " 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 5, 132 | "metadata": { 133 | "collapsed": false, 134 | "deletable": true, 135 | "editable": true 136 | }, 137 | "outputs": [ 138 | { 139 | "data": { 140 | "text/plain": [ 141 | "TensorShape([Dimension(2), Dimension(2)])" 142 | ] 143 | }, 144 | "execution_count": 5, 145 | "metadata": {}, 146 | "output_type": "execute_result" 147 | } 148 | ], 149 | "source": [ 150 | "a.get_shape() " 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": { 156 | "deletable": true, 157 | "editable": true 158 | }, 159 | "source": [ 160 | "# Listing 2-6. Reshaping a Tensor " 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 6, 166 | "metadata": { 167 | "collapsed": false, 168 | "deletable": true, 169 | "editable": true 170 | }, 171 | "outputs": [ 172 | { 173 | "data": { 174 | "text/plain": [ 175 | "array([[ 0., 0., 0., 0.]], dtype=float32)" 176 | ] 177 | }, 178 | "execution_count": 6, 179 | "metadata": {}, 180 | "output_type": "execute_result" 181 | } 182 | ], 183 | "source": [ 184 | "tf.reshape(a,(1,4)).eval() " 185 | ] 186 | }, 187 | { 188 | "cell_type": "markdown", 189 | "metadata": { 190 | "deletable": true, 191 | "editable": true 192 | }, 193 | "source": [ 194 | "# Listing 2-7. Explicit evaluation in TensorFlow and difference with Numpy" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 7, 200 | "metadata": { 201 | "collapsed": false, 202 | "deletable": true, 203 | "editable": true 204 | }, 205 | "outputs": [ 206 | { 207 | "name": "stdout", 208 | "output_type": "stream", 209 | "text": [ 210 | "Tensor(\"zeros_1:0\", shape=(2, 2), dtype=float32)\n" 211 | ] 212 | } 213 | ], 214 | "source": [ 215 | "ta = tf.zeros((2,2))\n", 216 | "print(ta) " 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 8, 222 | "metadata": { 223 | "collapsed": false, 224 | "deletable": true, 225 | "editable": true 226 | }, 227 | "outputs": [ 228 | { 229 | "name": "stdout", 230 | "output_type": "stream", 231 | "text": [ 232 | "[[ 0. 0.]\n", 233 | " [ 0. 0.]]\n" 234 | ] 235 | } 236 | ], 237 | "source": [ 238 | "print(ta.eval()) " 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": 9, 244 | "metadata": { 245 | "collapsed": false, 246 | "deletable": true, 247 | "editable": true 248 | }, 249 | "outputs": [ 250 | { 251 | "name": "stdout", 252 | "output_type": "stream", 253 | "text": [ 254 | "[[ 0. 0.]\n", 255 | " [ 0. 0.]]\n" 256 | ] 257 | } 258 | ], 259 | "source": [ 260 | "a = np.zeros((2,2))\n", 261 | "print(a) " 262 | ] 263 | }, 264 | { 265 | "cell_type": "markdown", 266 | "metadata": { 267 | "deletable": true, 268 | "editable": true 269 | }, 270 | "source": [ 271 | "# Listing 2-8.Defining TensorFlow Constants " 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": 10, 277 | "metadata": { 278 | "collapsed": true, 279 | "deletable": true, 280 | "editable": true 281 | }, 282 | "outputs": [], 283 | "source": [ 284 | "a = tf.constant(1)\n", 285 | "b = tf.constant(5)\n", 286 | "c= a*b " 287 | ] 288 | }, 289 | { 290 | "cell_type": "markdown", 291 | "metadata": { 292 | "deletable": true, 293 | "editable": true 294 | }, 295 | "source": [ 296 | "# Listing 2-9. TensorFlow Sesssion for executive of the Commands through Run and Eval " 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 12, 302 | "metadata": { 303 | "collapsed": false, 304 | "deletable": true, 305 | "editable": true 306 | }, 307 | "outputs": [ 308 | { 309 | "name": "stdout", 310 | "output_type": "stream", 311 | "text": [ 312 | "5\n", 313 | "5\n" 314 | ] 315 | } 316 | ], 317 | "source": [ 318 | "with tf.Session() as sess:\n", 319 | " print(c.eval())\n", 320 | " print(sess.run(c)) " 321 | ] 322 | }, 323 | { 324 | "cell_type": "markdown", 325 | "metadata": { 326 | "deletable": true, 327 | "editable": true 328 | }, 329 | "source": [ 330 | "# Listing 2-10a.Defining TensorFlow variables " 331 | ] 332 | }, 333 | { 334 | "cell_type": "code", 335 | "execution_count": 29, 336 | "metadata": { 337 | "collapsed": false, 338 | "deletable": true, 339 | "editable": true 340 | }, 341 | "outputs": [], 342 | "source": [ 343 | "w = tf.ones((2,2),name='weights')\n" 344 | ] 345 | }, 346 | { 347 | "cell_type": "markdown", 348 | "metadata": { 349 | "deletable": true, 350 | "editable": true 351 | }, 352 | "source": [ 353 | "# Listing 2-10b.Initializing the variables after invoking the Session " 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": 30, 359 | "metadata": { 360 | "collapsed": false, 361 | "deletable": true, 362 | "editable": true 363 | }, 364 | "outputs": [ 365 | { 366 | "name": "stdout", 367 | "output_type": "stream", 368 | "text": [ 369 | "[[ 1. 1.]\n", 370 | " [ 1. 1.]]\n" 371 | ] 372 | } 373 | ], 374 | "source": [ 375 | "with tf.Session() as sess:\n", 376 | " sess.run(tf.global_variables_initializer())\n", 377 | " print(sess.run(w)) " 378 | ] 379 | }, 380 | { 381 | "cell_type": "markdown", 382 | "metadata": { 383 | "deletable": true, 384 | "editable": true 385 | }, 386 | "source": [ 387 | "# Listing 2-11a.Defining the TensorFlow Variable with random initial values from Standard Normal Distribution " 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": 32, 393 | "metadata": { 394 | "collapsed": true, 395 | "deletable": true, 396 | "editable": true 397 | }, 398 | "outputs": [], 399 | "source": [ 400 | "rw = tf.Variable(tf.random_normal((2,2)),name='random_weights') " 401 | ] 402 | }, 403 | { 404 | "cell_type": "markdown", 405 | "metadata": { 406 | "deletable": true, 407 | "editable": true 408 | }, 409 | "source": [ 410 | "# Listing 2-11b.Invoking Session and displaying the Initial State of the variable " 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": 33, 416 | "metadata": { 417 | "collapsed": false, 418 | "deletable": true, 419 | "editable": true 420 | }, 421 | "outputs": [ 422 | { 423 | "name": "stdout", 424 | "output_type": "stream", 425 | "text": [ 426 | "[[ 1.01376367 2.30987906]\n", 427 | " [ 0.0666444 -0.29094756]]\n" 428 | ] 429 | } 430 | ], 431 | "source": [ 432 | "with tf.Session()as sess:\n", 433 | " sess.run(tf.global_variables_initializer())\n", 434 | " print(sess.run(rw)) " 435 | ] 436 | }, 437 | { 438 | "cell_type": "markdown", 439 | "metadata": { 440 | "deletable": true, 441 | "editable": true 442 | }, 443 | "source": [ 444 | "# Listing 2-12.TensorFlow Variable State Update " 445 | ] 446 | }, 447 | { 448 | "cell_type": "code", 449 | "execution_count": 35, 450 | "metadata": { 451 | "collapsed": false, 452 | "deletable": true, 453 | "editable": true 454 | }, 455 | "outputs": [ 456 | { 457 | "name": "stdout", 458 | "output_type": "stream", 459 | "text": [ 460 | "1\n", 461 | "2\n", 462 | "3\n", 463 | "4\n", 464 | "5\n" 465 | ] 466 | } 467 | ], 468 | "source": [ 469 | "var_1 = tf.Variable(0,name='var_1')\n", 470 | "add_op = tf.add(var_1,tf.constant(1))\n", 471 | "upd_op = tf.assign(var_1,add_op)\n", 472 | "\n", 473 | "with tf.Session() as sess:\n", 474 | " sess.run(tf.global_variables_initializer())\n", 475 | " for i in xrange(5):\n", 476 | " print(sess.run(upd_op)) " 477 | ] 478 | }, 479 | { 480 | "cell_type": "markdown", 481 | "metadata": { 482 | "deletable": true, 483 | "editable": true 484 | }, 485 | "source": [ 486 | "# Listing 2-13. Displaying the TensorFlow Variable State " 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 36, 492 | "metadata": { 493 | "collapsed": false, 494 | "deletable": true, 495 | "editable": true 496 | }, 497 | "outputs": [ 498 | { 499 | "name": "stdout", 500 | "output_type": "stream", 501 | "text": [ 502 | "[5, 12]\n" 503 | ] 504 | } 505 | ], 506 | "source": [ 507 | "x = tf.constant(1)\n", 508 | "y = tf.constant(5)\n", 509 | "z = tf.constant(7)\n", 510 | "mul_x_y = x*y\n", 511 | "final_op = mul_x_y + z\n", 512 | "with tf.Session() as sess:\n", 513 | " print(sess.run([mul_x_y,final_op])) " 514 | ] 515 | }, 516 | { 517 | "cell_type": "markdown", 518 | "metadata": { 519 | "deletable": true, 520 | "editable": true 521 | }, 522 | "source": [ 523 | "# Listing 2-14.Converting a Numpy array to Tensor " 524 | ] 525 | }, 526 | { 527 | "cell_type": "code", 528 | "execution_count": 37, 529 | "metadata": { 530 | "collapsed": false, 531 | "deletable": true, 532 | "editable": true 533 | }, 534 | "outputs": [ 535 | { 536 | "name": "stdout", 537 | "output_type": "stream", 538 | "text": [ 539 | "[[ 1. 1. 1.]\n", 540 | " [ 1. 1. 1.]\n", 541 | " [ 1. 1. 1.]]\n" 542 | ] 543 | } 544 | ], 545 | "source": [ 546 | "a = np.ones((3,3))\n", 547 | "b = tf.convert_to_tensor(a)\n", 548 | "with tf.Session() as sess:\n", 549 | " print(sess.run(b)) " 550 | ] 551 | }, 552 | { 553 | "cell_type": "markdown", 554 | "metadata": { 555 | "deletable": true, 556 | "editable": true 557 | }, 558 | "source": [ 559 | "# Listing 2-15.PlaceHolders and Feed Dictionary " 560 | ] 561 | }, 562 | { 563 | "cell_type": "code", 564 | "execution_count": 38, 565 | "metadata": { 566 | "collapsed": false, 567 | "deletable": true, 568 | "editable": true 569 | }, 570 | "outputs": [ 571 | { 572 | "name": "stdout", 573 | "output_type": "stream", 574 | "text": [ 575 | "[array([[ 10.]], dtype=float32)]\n" 576 | ] 577 | } 578 | ], 579 | "source": [ 580 | "inp1 = tf.placeholder(tf.float32,shape=(1,2))\n", 581 | "inp2 = tf.placeholder(tf.float32,shape=(2,1))\n", 582 | "output = tf.matmul(inp1,inp2)\n", 583 | "with tf.Session() as sess:\n", 584 | " print(sess.run([output],feed_dict={inp1:[[1.,3.]],inp2:[[1],[3]]})) " 585 | ] 586 | }, 587 | { 588 | "cell_type": "markdown", 589 | "metadata": { 590 | "deletable": true, 591 | "editable": true 592 | }, 593 | "source": [ 594 | "# Listing 2-16 . XOR implementation with Hidden layers having sigmoid activation functions. " 595 | ] 596 | }, 597 | { 598 | "cell_type": "code", 599 | "execution_count": 4, 600 | "metadata": { 601 | "collapsed": false, 602 | "deletable": true, 603 | "editable": true 604 | }, 605 | "outputs": [ 606 | { 607 | "name": "stdout", 608 | "output_type": "stream", 609 | "text": [ 610 | "WARNING:tensorflow:From :39: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n", 611 | "Instructions for updating:\n", 612 | "Use `tf.global_variables_initializer` instead.\n", 613 | "WARNING:tensorflow:Passing a `GraphDef` to the SummaryWriter is deprecated. Pass a `Graph` object instead, such as `sess.graph`.\n", 614 | "('Epoch ', 0)\n", 615 | "('Prediction:', array([[ 0.64893764],\n", 616 | " [ 0.59689724],\n", 617 | " [ 0.65669155],\n", 618 | " [ 0.60471165]], dtype=float32))\n", 619 | "('Weights from input to hidden layer:', array([[ 0.33330926, -0.06977983],\n", 620 | " [-0.74415874, -0.76291597]], dtype=float32))\n", 621 | "('Bias in the hidden layer:', array([-0.00018968, -0.00019783], dtype=float32))\n", 622 | "('Weights from hidden layer to output layer:', array([[ 0.55683446],\n", 623 | " [ 0.67457557]], dtype=float32))\n", 624 | "('Bias in the output layer:', array([-0.00127236], dtype=float32))\n", 625 | "('Cost:', 0.72787058)\n", 626 | "('Epoch ', 10000)\n", 627 | "('Prediction:', array([[ 0.51114953],\n", 628 | " [ 0.47865298],\n", 629 | " [ 0.52110726],\n", 630 | " [ 0.48724094]], dtype=float32))\n", 631 | "('Weights from input to hidden layer:', array([[ 0.36421987, 0.07621444],\n", 632 | " [-0.80342209, -0.69109011]], dtype=float32))\n", 633 | "('Bias in the hidden layer:', array([-0.1754255 , -0.05907499], dtype=float32))\n", 634 | "('Weights from hidden layer to output layer:', array([[ 0.35651296],\n", 635 | " [ 0.39376989]], dtype=float32))\n", 636 | "('Bias in the output layer:', array([-0.3091267], dtype=float32))\n", 637 | "('Cost:', 0.69305658)\n", 638 | "('Epoch ', 20000)\n", 639 | "('Prediction:', array([[ 0.50909698],\n", 640 | " [ 0.47744894],\n", 641 | " [ 0.52366114],\n", 642 | " [ 0.48920777]], dtype=float32))\n", 643 | "('Weights from input to hidden layer:', array([[ 0.4604663 , 0.15454701],\n", 644 | " [-0.90493822, -0.66533172]], dtype=float32))\n", 645 | "('Bias in the hidden layer:', array([-0.3383978 , -0.10174778], dtype=float32))\n", 646 | "('Weights from hidden layer to output layer:', array([[ 0.40625799],\n", 647 | " [ 0.30809221]], dtype=float32))\n", 648 | "('Bias in the output layer:', array([-0.2789081], dtype=float32))\n", 649 | "('Cost:', 0.69237739)\n", 650 | "('Epoch ', 30000)\n", 651 | "('Prediction:', array([[ 0.50825125],\n", 652 | " [ 0.46836436],\n", 653 | " [ 0.53683698],\n", 654 | " [ 0.48732194]], dtype=float32))\n", 655 | "('Weights from input to hidden layer:', array([[ 0.68638313, 0.19565736],\n", 656 | " [-1.15333831, -0.65534836]], dtype=float32))\n", 657 | "('Bias in the hidden layer:', array([-0.61990702, -0.14916727], dtype=float32))\n", 658 | "('Weights from hidden layer to output layer:', array([[ 0.62740165],\n", 659 | " [ 0.20371079]], dtype=float32))\n", 660 | "('Bias in the output layer:', array([-0.28073126], dtype=float32))\n", 661 | "('Cost:', 0.68961602)\n", 662 | "('Epoch ', 40000)\n", 663 | "('Prediction:', array([[ 0.49339783],\n", 664 | " [ 0.4297123 ],\n", 665 | " [ 0.61139292],\n", 666 | " [ 0.46820232]], dtype=float32))\n", 667 | "('Weights from input to hidden layer:', array([[ 1.46895719, 0.2015698 ],\n", 668 | " [-1.96267307, -0.64456999]], dtype=float32))\n", 669 | "('Bias in the hidden layer:', array([-1.27286911, -0.17152636], dtype=float32))\n", 670 | "('Weights from hidden layer to output layer:', array([[ 1.46054983],\n", 671 | " [-0.05060644]], dtype=float32))\n", 672 | "('Bias in the output layer:', array([-0.32279116], dtype=float32))\n", 673 | "('Cost:', 0.66204411)\n", 674 | "('Epoch ', 50000)\n", 675 | "('Prediction:', array([[ 0.39733696],\n", 676 | " [ 0.37896886],\n", 677 | " [ 0.81074315],\n", 678 | " [ 0.39166152]], dtype=float32))\n", 679 | "('Weights from input to hidden layer:', array([[ 3.14588714, 0.58158773],\n", 680 | " [-3.63937855, -0.94115627]], dtype=float32))\n", 681 | "('Bias in the hidden layer:', array([-2.32414532, 0.01375323], dtype=float32))\n", 682 | "('Weights from hidden layer to output layer:', array([[ 3.3128767 ],\n", 683 | " [-0.95220023]], dtype=float32))\n", 684 | "('Bias in the output layer:', array([-0.23251912], dtype=float32))\n", 685 | "('Cost:', 0.54588151)\n", 686 | "('Epoch ', 60000)\n", 687 | "('Prediction:', array([[ 0.21082206],\n", 688 | " [ 0.65720528],\n", 689 | " [ 0.88564587],\n", 690 | " [ 0.17220463]], dtype=float32))\n", 691 | "('Weights from input to hidden layer:', array([[ 4.58157301, 3.03108788],\n", 692 | " [-4.87241077, -2.74053288]], dtype=float32))\n", 693 | "('Bias in the hidden layer:', array([-2.88855243, 1.18735385], dtype=float32))\n", 694 | "('Weights from hidden layer to output layer:', array([[ 5.3037591 ],\n", 695 | " [-3.79996419]], dtype=float32))\n", 696 | "('Bias in the output layer:', array([ 1.31218469], dtype=float32))\n", 697 | "('Cost:', 0.24173743)\n", 698 | "('Epoch ', 70000)\n", 699 | "('Prediction:', array([[ 0.08185078],\n", 700 | " [ 0.87771422],\n", 701 | " [ 0.94029832],\n", 702 | " [ 0.06675483]], dtype=float32))\n", 703 | "('Weights from input to hidden layer:', array([[ 5.31078911, 4.45055914],\n", 704 | " [-5.47133875, -4.14548302]], dtype=float32))\n", 705 | "('Bias in the hidden layer:', array([-3.06447935, 1.97060025], dtype=float32))\n", 706 | "('Weights from hidden layer to output layer:', array([[ 6.86798048],\n", 707 | " [-6.05090141]], dtype=float32))\n", 708 | "('Bias in the output layer:', array([ 2.5869782], dtype=float32))\n", 709 | "('Cost:', 0.086618751)\n", 710 | "('Epoch ', 80000)\n", 711 | "('Prediction:', array([[ 0.04654188],\n", 712 | " [ 0.93326318],\n", 713 | " [ 0.96261185],\n", 714 | " [ 0.03852965]], dtype=float32))\n", 715 | "('Weights from input to hidden layer:', array([[ 5.65764332, 5.05240822],\n", 716 | " [-5.79591513, -4.7506671 ]], dtype=float32))\n", 717 | "('Bias in the hidden layer:', array([-3.17089868, 2.30809426], dtype=float32))\n", 718 | "('Weights from hidden layer to output layer:', array([[ 7.83147526],\n", 719 | " [-7.19909239]], dtype=float32))\n", 720 | "('Bias in the output layer:', array([ 3.21273899], dtype=float32))\n", 721 | "('Cost:', 0.048531085)\n", 722 | "('Epoch ', 90000)\n", 723 | "('Prediction:', array([[ 0.03189324],\n", 724 | " [ 0.9551484 ],\n", 725 | " [ 0.97327334],\n", 726 | " [ 0.02663374]], dtype=float32))\n", 727 | "('Weights from input to hidden layer:', array([[ 5.87259769, 5.40335369],\n", 728 | " [-6.0087204 , -5.10436583]], dtype=float32))\n", 729 | "('Bias in the hidden layer:', array([-3.24681425, 2.50163841], dtype=float32))\n", 730 | "('Weights from hidden layer to output layer:', array([[ 8.49800205],\n", 731 | " [-7.93741798]], dtype=float32))\n", 732 | "('Bias in the output layer:', array([ 3.60508418], dtype=float32))\n", 733 | "('Cost:', 0.03309666)\n", 734 | "('Final Prediction', array([[ 0.02407609],\n", 735 | " [ 0.96649462],\n", 736 | " [ 0.97934979],\n", 737 | " [ 0.02021892]], dtype=float32))\n" 738 | ] 739 | } 740 | ], 741 | "source": [ 742 | "#----------------------------------------------------------------------------------------------\n", 743 | "#XOR implementation in Tensorflow with hidden layers being sigmoid to introduce Non-Linearity\n", 744 | "#----------------------------------------------------------------------------------------------\n", 745 | "import tensorflow as tf\n", 746 | "#----------------------------------------------------------------------------------------------\n", 747 | "# Create placeholders for training input and output labels\n", 748 | "#----------------------------------------------------------------------------------------------\n", 749 | "x_ = tf.placeholder(tf.float32, shape=[4,2], name=\"x-input\")\n", 750 | "y_ = tf.placeholder(tf.float32, shape=[4,1], name=\"y-input\")\n", 751 | "#----------------------------------------------------------------------------------------------\n", 752 | "#Define the weights to the hidden and output layerrespectively. \n", 753 | "#----------------------------------------------------------------------------------------------\n", 754 | "w1 = tf.Variable(tf.random_uniform([2,2], -1, 1), name=\"Weights1\")\n", 755 | "w2 = tf.Variable(tf.random_uniform([2,1], -1, 1), name=\"Weights2\")\n", 756 | "#----------------------------------------------------------------------------------------------\n", 757 | "# Define the bias to the hidden and output layers respectively\n", 758 | "#----------------------------------------------------------------------------------------------\n", 759 | "b1 = tf.Variable(tf.zeros([2]), name=\"Bias1\")\n", 760 | "b2 = tf.Variable(tf.zeros([1]), name=\"Bias2\")\n", 761 | "#----------------------------------------------------------------------------------------------\n", 762 | "# Define the final output through forward pass\n", 763 | "#----------------------------------------------------------------------------------------------\n", 764 | "z2 = tf.sigmoid(tf.matmul(x_, w1) + b1)\n", 765 | "pred = tf.sigmoid(tf.matmul(z2,w2) + b2)\n", 766 | "#----------------------------------------------------------------------------------------------\n", 767 | "#Define the Cross-entropy/Log-loss Cost function based on the output label y and the predicted \n", 768 | "#probability by the forward pass\n", 769 | "#----------------------------------------------------------------------------------------------\n", 770 | "cost = tf.reduce_mean(( (y_ * tf.log(pred)) + \n", 771 | " ((1 - y_) * tf.log(1.0 - pred)) ) * -1)\n", 772 | "learning_rate = 0.01\n", 773 | "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n", 774 | "#----------------------------------------------------------------------------------------------\n", 775 | "#Now that we have all that we need set up we will start the training\n", 776 | "#----------------------------------------------------------------------------------------------\n", 777 | "XOR_X = [[0,0],[0,1],[1,0],[1,1]]\n", 778 | "XOR_Y = [[0],[1],[1],[0]]\n", 779 | "\n", 780 | "init = tf.initialize_all_variables()\n", 781 | "sess = tf.Session()\n", 782 | "writer = tf.summary.FileWriter(\"./Downloads/XOR1_logs\", sess.graph_def)\n", 783 | "\n", 784 | "sess.run(init)\n", 785 | "for i in range(100000):\n", 786 | " sess.run(train_step, feed_dict={x_: XOR_X, y_: XOR_Y})\n", 787 | " if i % 10000 == 0:\n", 788 | " print('Epoch ', i)\n", 789 | " print('Prediction:', sess.run(pred,feed_dict={x_: XOR_X, y_: XOR_Y}))\n", 790 | " print('Weights from input to hidden layer:', sess.run(w1))\n", 791 | " print('Bias in the hidden layer:', sess.run(b1))\n", 792 | " print('Weights from hidden layer to output layer:', sess.run(w2))\n", 793 | " print('Bias in the output layer:', sess.run(b2))\n", 794 | " print('Cost:', sess.run(cost, feed_dict={x_: XOR_X, y_: XOR_Y}))\n", 795 | "#---------------------------------------------------------------------------------------------- \n", 796 | "print('Final Prediction', sess.run(pred, feed_dict={x_: XOR_X, y_: XOR_Y}))\n", 797 | "#----------------------------------------------------------------------------------------------" 798 | ] 799 | }, 800 | { 801 | "cell_type": "markdown", 802 | "metadata": { 803 | "deletable": true, 804 | "editable": true 805 | }, 806 | "source": [ 807 | "# Listing 2-17 . XOR implementation with linear activation functionsin hidden layer " 808 | ] 809 | }, 810 | { 811 | "cell_type": "code", 812 | "execution_count": 66, 813 | "metadata": { 814 | "collapsed": false, 815 | "deletable": true, 816 | "editable": true 817 | }, 818 | "outputs": [ 819 | { 820 | "name": "stdout", 821 | "output_type": "stream", 822 | "text": [ 823 | "WARNING:tensorflow:From :39: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n", 824 | "Instructions for updating:\n", 825 | "Use `tf.global_variables_initializer` instead.\n" 826 | ] 827 | }, 828 | { 829 | "name": "stderr", 830 | "output_type": "stream", 831 | "text": [ 832 | "WARNING:tensorflow:From :39: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n", 833 | "Instructions for updating:\n", 834 | "Use `tf.global_variables_initializer` instead.\n" 835 | ] 836 | }, 837 | { 838 | "name": "stdout", 839 | "output_type": "stream", 840 | "text": [ 841 | "('Epoch ', 0)\n", 842 | "('Prediction:', array([[ 0.49962392],\n", 843 | " [ 0.62373435],\n", 844 | " [ 0.56146997],\n", 845 | " [ 0.68006349]], dtype=float32))\n", 846 | "('Weights from input to hidden layer:', array([[-0.45627943, 0.23895949],\n", 847 | " [-0.85042536, 0.31715545]], dtype=float32))\n", 848 | "('Bias in the hidden layer:', array([ 0.00066458, 0.00031138], dtype=float32))\n", 849 | "('Weights from hidden layer to output layer:', array([[-0.72267991],\n", 850 | " [-0.33942884]], dtype=float32))\n", 851 | "('Bias in the output layer:', array([-0.00091842], dtype=float32))\n", 852 | "('Cost:', 0.72031391)\n", 853 | "('Epoch ', 10000)\n", 854 | "('Prediction:', array([[ 0.49985746],\n", 855 | " [ 0.50034904],\n", 856 | " [ 0.49959072],\n", 857 | " [ 0.50008225]], dtype=float32))\n", 858 | "('Weights from input to hidden layer:', array([[-0.37904808, 0.28653768],\n", 859 | " [-0.64646655, 0.48142514]], dtype=float32))\n", 860 | "('Bias in the hidden layer:', array([ 0.02285987, -0.0133572 ], dtype=float32))\n", 861 | "('Weights from hidden layer to output layer:', array([[-0.39112166],\n", 862 | " [-0.52112126]], dtype=float32))\n", 863 | "('Bias in the output layer:', array([ 0.00141016], dtype=float32))\n", 864 | "('Cost:', 0.6931473)\n", 865 | "('Epoch ', 20000)\n", 866 | "('Prediction:', array([[ 0.50000429],\n", 867 | " [ 0.50002593],\n", 868 | " [ 0.49997497],\n", 869 | " [ 0.49999654]], dtype=float32))\n", 870 | "('Weights from input to hidden layer:', array([[-0.38027543, 0.28490078],\n", 871 | " [-0.64534467, 0.4829419 ]], dtype=float32))\n", 872 | "('Bias in the hidden layer:', array([ 0.0227049 , -0.01356433], dtype=float32))\n", 873 | "('Weights from hidden layer to output layer:', array([[-0.39045283],\n", 874 | " [-0.5215745 ]], dtype=float32))\n", 875 | "('Bias in the output layer:', array([ 0.00180753], dtype=float32))\n", 876 | "('Cost:', 0.69314718)\n", 877 | "('Epoch ', 30000)\n", 878 | "('Prediction:', array([[ 0.5000003 ],\n", 879 | " [ 0.50001115],\n", 880 | " [ 0.49998885],\n", 881 | " [ 0.4999997 ]], dtype=float32))\n", 882 | "('Weights from input to hidden layer:', array([[-0.38033357, 0.28480721],\n", 883 | " [-0.64534467, 0.48302445]], dtype=float32))\n", 884 | "('Bias in the hidden layer:', array([ 0.02270905, -0.01355919], dtype=float32))\n", 885 | "('Weights from hidden layer to output layer:', array([[-0.39045283],\n", 886 | " [-0.5215745 ]], dtype=float32))\n", 887 | "('Bias in the output layer:', array([ 0.00179592], dtype=float32))\n", 888 | "('Cost:', 0.69314718)\n", 889 | "('Epoch ', 40000)\n", 890 | "('Prediction:', array([[ 0.5000003 ],\n", 891 | " [ 0.50001115],\n", 892 | " [ 0.49998885],\n", 893 | " [ 0.4999997 ]], dtype=float32))\n", 894 | "('Weights from input to hidden layer:', array([[-0.38033357, 0.28480721],\n", 895 | " [-0.64534467, 0.48302445]], dtype=float32))\n", 896 | "('Bias in the hidden layer:', array([ 0.02270905, -0.01355919], dtype=float32))\n", 897 | "('Weights from hidden layer to output layer:', array([[-0.39045283],\n", 898 | " [-0.5215745 ]], dtype=float32))\n", 899 | "('Bias in the output layer:', array([ 0.00179592], dtype=float32))\n", 900 | "('Cost:', 0.69314718)\n", 901 | "('Epoch ', 50000)\n", 902 | "('Prediction:', array([[ 0.5000003 ],\n", 903 | " [ 0.50001115],\n", 904 | " [ 0.49998885],\n", 905 | " [ 0.4999997 ]], dtype=float32))\n", 906 | "('Weights from input to hidden layer:', array([[-0.38033357, 0.28480721],\n", 907 | " [-0.64534467, 0.48302445]], dtype=float32))\n", 908 | "('Bias in the hidden layer:', array([ 0.02270905, -0.01355919], dtype=float32))\n", 909 | "('Weights from hidden layer to output layer:', array([[-0.39045283],\n", 910 | " [-0.5215745 ]], dtype=float32))\n", 911 | "('Bias in the output layer:', array([ 0.00179592], dtype=float32))\n", 912 | "('Cost:', 0.69314718)\n", 913 | "('Epoch ', 60000)\n", 914 | "('Prediction:', array([[ 0.5000003 ],\n", 915 | " [ 0.50001115],\n", 916 | " [ 0.49998885],\n", 917 | " [ 0.4999997 ]], dtype=float32))\n", 918 | "('Weights from input to hidden layer:', array([[-0.38033357, 0.28480721],\n", 919 | " [-0.64534467, 0.48302445]], dtype=float32))\n", 920 | "('Bias in the hidden layer:', array([ 0.02270905, -0.01355919], dtype=float32))\n", 921 | "('Weights from hidden layer to output layer:', array([[-0.39045283],\n", 922 | " [-0.5215745 ]], dtype=float32))\n", 923 | "('Bias in the output layer:', array([ 0.00179592], dtype=float32))\n", 924 | "('Cost:', 0.69314718)\n", 925 | "('Epoch ', 70000)\n", 926 | "('Prediction:', array([[ 0.5000003 ],\n", 927 | " [ 0.50001115],\n", 928 | " [ 0.49998885],\n", 929 | " [ 0.4999997 ]], dtype=float32))\n", 930 | "('Weights from input to hidden layer:', array([[-0.38033357, 0.28480721],\n", 931 | " [-0.64534467, 0.48302445]], dtype=float32))\n", 932 | "('Bias in the hidden layer:', array([ 0.02270905, -0.01355919], dtype=float32))\n", 933 | "('Weights from hidden layer to output layer:', array([[-0.39045283],\n", 934 | " [-0.5215745 ]], dtype=float32))\n", 935 | "('Bias in the output layer:', array([ 0.00179592], dtype=float32))\n", 936 | "('Cost:', 0.69314718)\n", 937 | "('Epoch ', 80000)\n", 938 | "('Prediction:', array([[ 0.5000003 ],\n", 939 | " [ 0.50001115],\n", 940 | " [ 0.49998885],\n", 941 | " [ 0.4999997 ]], dtype=float32))\n", 942 | "('Weights from input to hidden layer:', array([[-0.38033357, 0.28480721],\n", 943 | " [-0.64534467, 0.48302445]], dtype=float32))\n", 944 | "('Bias in the hidden layer:', array([ 0.02270905, -0.01355919], dtype=float32))\n", 945 | "('Weights from hidden layer to output layer:', array([[-0.39045283],\n", 946 | " [-0.5215745 ]], dtype=float32))\n", 947 | "('Bias in the output layer:', array([ 0.00179592], dtype=float32))\n", 948 | "('Cost:', 0.69314718)\n", 949 | "('Epoch ', 90000)\n", 950 | "('Prediction:', array([[ 0.5000003 ],\n", 951 | " [ 0.50001115],\n", 952 | " [ 0.49998885],\n", 953 | " [ 0.4999997 ]], dtype=float32))\n", 954 | "('Weights from input to hidden layer:', array([[-0.38033357, 0.28480721],\n", 955 | " [-0.64534467, 0.48302445]], dtype=float32))\n", 956 | "('Bias in the hidden layer:', array([ 0.02270905, -0.01355919], dtype=float32))\n", 957 | "('Weights from hidden layer to output layer:', array([[-0.39045283],\n", 958 | " [-0.5215745 ]], dtype=float32))\n", 959 | "('Bias in the output layer:', array([ 0.00179592], dtype=float32))\n", 960 | "('Cost:', 0.69314718)\n", 961 | "('Final Prediction', array([[ 0.5000003 ],\n", 962 | " [ 0.50001115],\n", 963 | " [ 0.49998885],\n", 964 | " [ 0.4999997 ]], dtype=float32))\n" 965 | ] 966 | } 967 | ], 968 | "source": [ 969 | "#----------------------------------------------------------------------------------------------\n", 970 | "#XOR implementation in Tensorflow with linear activation for hidden layers \n", 971 | "#----------------------------------------------------------------------------------------------\n", 972 | "import tensorflow as tf\n", 973 | "#----------------------------------------------------------------------------------------------\n", 974 | "# Create placeholders for training input and output labels\n", 975 | "#----------------------------------------------------------------------------------------------\n", 976 | "x_ = tf.placeholder(tf.float32, shape=[4,2], name=\"x-input\")\n", 977 | "y_ = tf.placeholder(tf.float32, shape=[4,1], name=\"y-input\")\n", 978 | "#----------------------------------------------------------------------------------------------\n", 979 | "#Define the weights to the hidden and output layer respectively. \n", 980 | "#----------------------------------------------------------------------------------------------\n", 981 | "w1 = tf.Variable(tf.random_uniform([2,2], -1, 1), name=\"Weights1\")\n", 982 | "w2 = tf.Variable(tf.random_uniform([2,1], -1, 1), name=\"Weights2\")\n", 983 | "#----------------------------------------------------------------------------------------------\n", 984 | "# Define the bias to the hideen and output layers respectively\n", 985 | "#----------------------------------------------------------------------------------------------\n", 986 | "b1 = tf.Variable(tf.zeros([2]), name=\"Bias1\")\n", 987 | "b2 = tf.Variable(tf.zeros([1]), name=\"Bias2\")\n", 988 | "#----------------------------------------------------------------------------------------------\n", 989 | "# Define the final output through forward pass\n", 990 | "#----------------------------------------------------------------------------------------------\n", 991 | "z2 = tf.matmul(x_, w1) + b1\n", 992 | "pred = tf.sigmoid(tf.matmul(z2,w2) + b2)\n", 993 | "#----------------------------------------------------------------------------------------------\n", 994 | "#Define the Cross-entropy/Log-loss Cost function based on the output label y and the predicted \n", 995 | "#probability by the forward pass\n", 996 | "#----------------------------------------------------------------------------------------------\n", 997 | "cost = tf.reduce_mean(( (y_ * tf.log(pred)) + \n", 998 | " ((1 - y_) * tf.log(1.0 - pred)) ) * -1)\n", 999 | "learning_rate = 0.01\n", 1000 | "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n", 1001 | "#----------------------------------------------------------------------------------------------\n", 1002 | "#Now that we have all that we need set up we will start the training\n", 1003 | "#----------------------------------------------------------------------------------------------\n", 1004 | "XOR_X = [[0,0],[0,1],[1,0],[1,1]]\n", 1005 | "XOR_Y = [[0],[1],[1],[0]]\n", 1006 | "\n", 1007 | "init = tf.initialize_all_variables()\n", 1008 | "sess = tf.Session()\n", 1009 | "sess.run(init)\n", 1010 | "for i in range(100000):\n", 1011 | " sess.run(train_step, feed_dict={x_: XOR_X, y_: XOR_Y})\n", 1012 | " if i % 10000 == 0:\n", 1013 | " print('Epoch ', i)\n", 1014 | " print('Prediction:', sess.run(pred,feed_dict={x_: XOR_X, y_: XOR_Y}))\n", 1015 | " print('Weights from input to hidden layer:', sess.run(w1))\n", 1016 | " print('Bias in the hidden layer:', sess.run(b1))\n", 1017 | " print('Weights from hidden layer to output layer:', sess.run(w2))\n", 1018 | " print('Bias in the output layer:', sess.run(b2))\n", 1019 | " print('Cost:', sess.run(cost, feed_dict={x_: XOR_X, y_: XOR_Y}))\n", 1020 | "#---------------------------------------------------------------------------------------------- \n", 1021 | "print('Final Prediction', sess.run(pred, feed_dict={x_: XOR_X, y_: XOR_Y}))\n", 1022 | "#----------------------------------------------------------------------------------------------" 1023 | ] 1024 | }, 1025 | { 1026 | "cell_type": "markdown", 1027 | "metadata": { 1028 | "collapsed": true, 1029 | "deletable": true, 1030 | "editable": true 1031 | }, 1032 | "source": [ 1033 | "# Listing 2-18. Linear Regression implementation in TensorFlow " 1034 | ] 1035 | }, 1036 | { 1037 | "cell_type": "code", 1038 | "execution_count": 46, 1039 | "metadata": { 1040 | "collapsed": false, 1041 | "deletable": true, 1042 | "editable": true 1043 | }, 1044 | "outputs": [ 1045 | { 1046 | "name": "stdout", 1047 | "output_type": "stream", 1048 | "text": [ 1049 | "MSE in training: 22.0334\n" 1050 | ] 1051 | } 1052 | ], 1053 | "source": [ 1054 | "#----------------------------------------------------------------------------------------------\n", 1055 | "\n", 1056 | "# Importing TensorFlow, Numpy and the Boston Housing price dataset\n", 1057 | "\n", 1058 | "#---------------------------------------------------------------------------------------------- \n", 1059 | "\n", 1060 | "import tensorflow as tf\n", 1061 | "import numpy as np\n", 1062 | "from sklearn.datasets import load_boston\n", 1063 | "\n", 1064 | "#----------------------------------------------------------------------------------------------\n", 1065 | "\n", 1066 | "# Function to load the Boston data set\n", 1067 | "\n", 1068 | "#---------------------------------------------------------------------------------------------- \n", 1069 | "\n", 1070 | "def read_infile():\n", 1071 | " data = load_boston()\n", 1072 | " features = np.array(data.data)\n", 1073 | " target = np.array(data.target)\n", 1074 | " return features,target\n", 1075 | "\n", 1076 | "#----------------------------------------------------------------------------------------------\n", 1077 | "\n", 1078 | "# Normalize the features by Z scaling i.e. subract form each feature value its mean and then divide by its \n", 1079 | "# standard deviation. Accelerates Gradient Descent.\n", 1080 | "\n", 1081 | "#---------------------------------------------------------------------------------------------- \n", 1082 | "\n", 1083 | "def feature_normalize(data):\n", 1084 | " mu = np.mean(data,axis=0)\n", 1085 | " std = np.std(data,axis=0)\n", 1086 | " return (data - mu)/std\n", 1087 | "\n", 1088 | "#----------------------------------------------------------------------------------------------\n", 1089 | "\n", 1090 | "# Append the feature for the bias term.\n", 1091 | "\n", 1092 | "#---------------------------------------------------------------------------------------------- \n", 1093 | "\n", 1094 | "def append_bias(features,target):\n", 1095 | " n_samples = features.shape[0]\n", 1096 | " n_features = features.shape[1]\n", 1097 | " intercept_feature = np.ones((n_samples,1))\n", 1098 | " X = np.concatenate((features,intercept_feature),axis=1)\n", 1099 | " X = np.reshape(X,[n_samples,n_features +1])\n", 1100 | " Y = np.reshape(target,[n_samples,1])\n", 1101 | " return X,Y\n", 1102 | "\n", 1103 | "#----------------------------------------------------------------------------------------------\n", 1104 | "\n", 1105 | "# Execute the functions to read, normalize and add append bias term to the data\n", 1106 | "\n", 1107 | "#---------------------------------------------------------------------------------------------- \n", 1108 | "\n", 1109 | "features,target = read_infile()\n", 1110 | "z_features = feature_normalize(features)\n", 1111 | "X_input,Y_input = append_bias(z_features,target)\n", 1112 | "num_features = X_input.shape[1]\n", 1113 | "\n", 1114 | "#----------------------------------------------------------------------------------------------\n", 1115 | "\n", 1116 | "# Create TensorFlow ops for placeholders, weights and weight initialization\n", 1117 | "\n", 1118 | "#---------------------------------------------------------------------------------------------- \n", 1119 | "\n", 1120 | "X = tf.placeholder(tf.float32,[None,num_features])\n", 1121 | "Y = tf.placeholder(tf.float32,[None,1])\n", 1122 | "w = tf.Variable(tf.random_normal((num_features,1)),name='weights')\n", 1123 | "init = tf.global_variables_initializer()\n", 1124 | "\n", 1125 | "#----------------------------------------------------------------------------------------------\n", 1126 | "\n", 1127 | "# Define the different TensforFlow ops and input parameters for Cost and Optimization.\n", 1128 | "\n", 1129 | "#---------------------------------------------------------------------------------------------- \n", 1130 | "\n", 1131 | "learning_rate = 0.01\n", 1132 | "num_epochs = 1000\n", 1133 | "cost_trace = []\n", 1134 | "pred = tf.matmul(X,w)\n", 1135 | "error = pred - Y\n", 1136 | "cost = tf.reduce_mean(tf.square(error))\n", 1137 | "train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n", 1138 | "\n", 1139 | "#----------------------------------------------------------------------------------------------\n", 1140 | "\n", 1141 | "# Execute the gradient descent learning\n", 1142 | "\n", 1143 | "#---------------------------------------------------------------------------------------------- \n", 1144 | "\n", 1145 | "with tf.Session() as sess:\n", 1146 | " sess.run(init)\n", 1147 | " for i in xrange(num_epochs):\n", 1148 | " sess.run(train_op,feed_dict={X:X_input,Y:Y_input})\n", 1149 | " cost_trace.append(sess.run(cost,feed_dict={X:X_input,Y:Y_input}))\n", 1150 | " error_ = sess.run(error,{X:X_input,Y:Y_input}) \n", 1151 | " pred_ = sess.run(pred,{X:X_input}) \n", 1152 | "\n", 1153 | "print 'MSE in training:',cost_trace[-1] \n", 1154 | " " 1155 | ] 1156 | }, 1157 | { 1158 | "cell_type": "markdown", 1159 | "metadata": { 1160 | "deletable": true, 1161 | "editable": true 1162 | }, 1163 | "source": [ 1164 | "# Listing 2-18a. Linear Regression Cost plot over Epochs or iterations " 1165 | ] 1166 | }, 1167 | { 1168 | "cell_type": "code", 1169 | "execution_count": 47, 1170 | "metadata": { 1171 | "collapsed": false, 1172 | "deletable": true, 1173 | "editable": true 1174 | }, 1175 | "outputs": [ 1176 | { 1177 | "data": { 1178 | "text/plain": [ 1179 | "[]" 1180 | ] 1181 | }, 1182 | "execution_count": 47, 1183 | "metadata": {}, 1184 | "output_type": "execute_result" 1185 | }, 1186 | { 1187 | "data": { 1188 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGONJREFUeJzt3X9wHOd93/H3937gDgABECAhkiYgg6opOXJs/SgqU3Ha\nOmIsW0ocKhPHY8etGIcznEyURm48k8rtH04y/cPuZCJHbUdjJXJNp7Zix3EqVtVYUUh5EiUxZdCW\naYkiQ5iURNIkAYo/wF8A8ePbP/Y58AAccAfggOPtfl4zx9t99rndZ7Gczz733N6euTsiIhJfqVo3\nQERElpaCXkQk5hT0IiIxp6AXEYk5Bb2ISMwp6EVEYk5BLyIScwp6EZGYU9CLiMRcptYNAFi9erX3\n9PTUuhkiInVl7969p929s1y96yLoe3p66Ovrq3UzRETqipm9UUk9Dd2IiMScgl5EJOYU9CIiMaeg\nFxGJOQW9iEjMKehFRGJOQS8iEnN1HfTfe/0Mf/TcQcbGJ2rdFBGR61ZdB/3Lb57jf7zQz/CYgl5E\nZDZ1HfS5bNT84dHxGrdEROT6Vd9Bn4maP6IevYjIrOo66PPZNKAevYjIXOo66Cd79KPq0YuIzKa+\ng77Qox9Tj15EZDb1HfTq0YuIlFVR0JvZ62b2IzN72cz6QlmHmT1vZofCc3soNzN7zMz6zWyfmd25\nVI0vjNGPqEcvIjKr+fTof87db3f33jD/CLDL3TcCu8I8wH3AxvDYDjxercZOV+jRD6tHLyIyq8UM\n3WwBdoTpHcADReVf8ch3gZVmtm4R25lVLqMevYhIOZUGvQN/Y2Z7zWx7KFvj7ifC9ElgTZheDxwt\neu2xUFZ1+azG6EVEyqn0N2N/1t2Pm9kNwPNmdqB4obu7mfl8NhxOGNsBbrzxxvm8dJJ69CIi5VXU\no3f34+F5APhr4C7gVGFIJjwPhOrHge6il3eFsunrfMLde929t7Oz7I+Yl5TPaoxeRKScskFvZs1m\n1lKYBu4FXgF2AltDta3A02F6J/BguPpmE3C+aIinqtSjFxEpr5KhmzXAX5tZof7X3P3bZvY94Btm\ntg14A/hoqP8scD/QD1wGPln1VgfZtJEy3etGRGQuZYPe3Q8Dt5UofwvYXKLcgYeq0royzIxcJq17\n3YiIzKGuvxkL0a2K1aMXEZld3Qd9Xj16EZE51X3Qq0cvIjK3ug969ehFROZW90GvHr2IyNzqPujz\nmbRugSAiMoe6D/pcNqUfHhERmUP9B30mpR69iMgc6j/os2n16EVE5lD/Qa8evYjInOo+6PPZtG5q\nJiIyh7oP+lwmpdsUi4jMoe6DvjGrL0yJiMwlFkE/NuFc1ZemRERKqv+gb4h+fOSKevUiIiXVfdA3\nNUS31L9yVUEvIlJK3Qd9Y0O0C5evjtW4JSIi16f6D/ps1KO/rB69iEhJdR/0TWGMXlfeiIiUFpug\nV49eRKS0ug/6fFZBLyIyl7oP+qbJyyv1YayISCkxCPrC5ZX6wpSISCl1H/SNk2P06tGLiJRS/0Ef\nxuj1hSkRkdLqPugbMikyKeOyLq8UESmp7oMeouEb9ehFREqLRdA3KehFRGYVi6BvzKY1dCMiMot4\nBH1Dhiu66kZEpKRYBH1TQ1r3oxcRmUXFQW9maTP7gZk9E+Y3mNkeM+s3s6+bWUMoz4X5/rC8Z2ma\nfk1TQ1q3QBARmcV8evQPA68VzX8eeNTd3wGcBbaF8m3A2VD+aKi3pPJZfRgrIjKbioLezLqAXwD+\nLMwbcA/wzVBlB/BAmN4S5gnLN4f6S0Y9ehGR2VXao/8C8HtA4YYyq4Bz7l74BPQYsD5MrweOAoTl\n50P9Kcxsu5n1mVnf4ODgApsfac5ldAsEEZFZlA16M/tFYMDd91Zzw+7+hLv3untvZ2fnota1Ipfh\nwrCCXkSklEwFdd4H/JKZ3Q/kgVbgT4CVZpYJvfYu4HiofxzoBo6ZWQZoA96qesuLNDdkGBmbYGx8\ngkw6FhcSiYhUTdlUdPfPuHuXu/cAHwN2u/sngBeAj4RqW4Gnw/TOME9YvtvdvaqtnmZFPjpfXRrR\nOL2IyHSL6f7+J+B3zayfaAz+yVD+JLAqlP8u8Mjimljeilx0B8sLI6NLvSkRkbpTydDNJHf/DvCd\nMH0YuKtEnWHgV6vQtoo159SjFxGZTSwGtFeEoL84og9kRUSmU9CLiMRcLIL+2tCNgl5EZLpYBL16\n9CIis4tX0OtLUyIiM8Qi6DV0IyIyu1gEfUMmRUMmxUXd70ZEZIZYBD1EwzcauhERmSk2Qd+cS2vo\nRkSkhNgE/YpcVlfdiIiUEKOgTyvoRURKiE3QN+cyuteNiEgJsQn6FbmMevQiIiUo6EVEYi42QR8N\n3SjoRUSmi03Qr8hluHx1nPGJJf0xKxGRuhOroAe4pG/HiohMEZugbwm/G3tB344VEZkiNkHf2pgF\nYOiKfjdWRKRYfII+r6AXESklNkHfFnr05xX0IiJTxCboWxujMfohjdGLiEwRm6Bv0xi9iEhJsQn6\nwuWVGroREZkqNkGfSadYkcswNKygFxEpFpugh2j4Rj16EZGpYhX0LfkMQ1f0YayISLFYBX1rY1ZD\nNyIi08Qq6Nsas7rqRkRkmlgFfWteQS8iMl3ZoDezvJm9ZGY/NLNXzewPQvkGM9tjZv1m9nUzawjl\nuTDfH5b3LO0uXKMPY0VEZqqkRz8C3OPutwG3Ax8ys03A54FH3f0dwFlgW6i/DTgbyh8N9ZZFa2OG\nS1fHGRufWK5Niohc98oGvUcuhtlseDhwD/DNUL4DeCBMbwnzhOWbzcyq1uI5FG5splsVi4hcU9EY\nvZmlzexlYAB4HvgxcM7dC4l6DFgfptcDRwHC8vPAqhLr3G5mfWbWNzg4uLi9CHRjMxGRmSoKencf\nd/fbgS7gLuCdi92wuz/h7r3u3tvZ2bnY1QFF96TXJZYiIpPmddWNu58DXgDuBlaaWSYs6gKOh+nj\nQDdAWN4GvFWV1pbRmtf9bkREpqvkqptOM1sZphuBDwCvEQX+R0K1rcDTYXpnmCcs3+3uy/KL3Sub\nGgA4d1lBLyJSkClfhXXADjNLE50YvuHuz5jZfuAvzOy/Aj8Angz1nwT+3Mz6gTPAx5ag3SW1N0VD\nN+cuX12uTYqIXPfKBr277wPuKFF+mGi8fnr5MPCrVWndPBV69GfVoxcRmRSrb8Y2ZFK05DKcuaQe\nvYhIQayCHqC9uYGzGroREZkUv6BvymroRkSkSPyCvrmBsxq6ERGZFLug72jS0I2ISLHYBf3KJvXo\nRUSKxS7oO5qzXLo6zsjYeK2bIiJyXYhd0Lc369uxIiLF4hf04UtTupZeRCQS26DXB7IiIpHYBX1H\nGLo5e0lDNyIiEMOgb2+Obmx2Rj16EREghkG/srHQo1fQi4hADIO+IZOiJa8bm4mIFMQu6AE6V+QY\nvDhS62aIiFwXYhn0q1tynL6goBcRgZgGvXr0IiLXxDLoV69oUI9eRCSIZdB3tuQYGh5jeFT3uxER\niWXQr16RA+AtXXkjIhLvoNfwjYhITIO+syUK+kEFvYhIPIN+dQj607ryRkQknkG/KtzYTEEvIhLT\noM9n07TmMxq6EREhpkEP4duxF3XVjYhIbIO+c0VOPXoREWIc9Gvb8pwcGq51M0REai6+Qd+a5+T5\nYdy91k0REamp+AZ9W56r4xO6L72IJF7ZoDezbjN7wcz2m9mrZvZwKO8ws+fN7FB4bg/lZmaPmVm/\nme0zszuXeidKWduaB9DwjYgkXiU9+jHg0+5+K7AJeMjMbgUeAXa5+0ZgV5gHuA/YGB7bgcer3uoK\nrG0LQX9eQS8iyVY26N39hLt/P0xfAF4D1gNbgB2h2g7ggTC9BfiKR74LrDSzdVVveRmTQa8evYgk\n3LzG6M2sB7gD2AOscfcTYdFJYE2YXg8cLXrZsVA2fV3bzazPzPoGBwfn2ezyOlfkSJl69CIiFQe9\nma0A/gr4lLsPFS/z6NKWeV3e4u5PuHuvu/d2dnbO56UVyaRT3NCS54SCXkQSrqKgN7MsUch/1d2/\nFYpPFYZkwvNAKD8OdBe9vCuULbs1bXlOaehGRBKukqtuDHgSeM3d/7ho0U5ga5jeCjxdVP5guPpm\nE3C+aIhnWa1rVY9eRCRTQZ33Af8e+JGZvRzK/jPwOeAbZrYNeAP4aFj2LHA/0A9cBj5Z1RbPw9q2\nPC/2n8bdic5XIiLJUzbo3f1FYLaU3FyivgMPLbJdVbF+ZSMXR8YYujJGW1O21s0REamJ2H4zFqC7\noxGAo2cv17glIiK1E+ug72pvAuCYgl5EEizWQd89GfRXatwSEZHaiXXQtzZmaMllOHpGPXoRSa5Y\nB72Z0dXRpB69iCRarIMeoKu9UR/GikiixT7ou9ujHr1+gEREkir2Qd/V3sjlq+P6ARIRSazYB/2N\nHdGVN2/qA1kRSajYB/2GzmYAjpy+VOOWiIjURuyDvru9iXTKFPQiklixD/qGTIru9kYOK+hFJKFi\nH/QAG1Y3c3hQQS8iyZSQoF/B66cvMTGhSyxFJHkSEfQ3dTZzZXScUxf0IyQikjzJCPrV4cobDd+I\nSAIlIugLl1j2D16scUtERJZfIoJ+bWuelnyGgycv1LopIiLLLhFBb2a8c22Lgl5EEikRQQ9wy9oW\nDp66oJubiUjiJCjoW7kwPMZPzuvKGxFJlsQE/U+tbQHg4MmhGrdERGR5JSbobw5Bf0Dj9CKSMIkJ\n+tZ8lvUrG9n/E/XoRSRZEhP0AO/pauNHx8/XuhkiIssqYUG/kjfeusy5y/q1KRFJjkQF/W1dbQD8\n8Jh69SKSHIkK+p8OQb/v6Lkat0REZPkkKuhb81lu6mxWj15EEiVRQQ9we/dKXj56Vt+QFZHEKBv0\nZvYlMxsws1eKyjrM7HkzOxSe20O5mdljZtZvZvvM7M6lbPxCvHdDB6cvXtVPC4pIYlTSo/8y8KFp\nZY8Au9x9I7ArzAPcB2wMj+3A49VpZvXctWEVAHsOn6lxS0RElkfZoHf3vwOmp+IWYEeY3gE8UFT+\nFY98F1hpZuuq1dhq6FnVRGdLjpeOvFXrpoiILIuFjtGvcfcTYfoksCZMrweOFtU7FspmMLPtZtZn\nZn2Dg4MLbMb8mRl3behgz5EzGqcXkURY9IexHqXlvBPT3Z9w91537+3s7FxsM+Zl002rOHF+mCMa\npxeRBFho0J8qDMmE54FQfhzoLqrXFcquK++/OTqxfOfg8r2TEBGplYUG/U5ga5jeCjxdVP5guPpm\nE3C+aIjnutHd0cS/6GzmhYMD5SuLiNS5Si6vfAr4J+AWMztmZtuAzwEfMLNDwM+HeYBngcNAP/Cn\nwG8tSaur4P233MCeI2e4fHWs1k0REVlSmXIV3P3jsyzaXKKuAw8ttlHL4eduuYEnXzzC3x86zQff\ntbbWzRERWTKJ+2ZswXtv6qC9Kcv/23fdjSyJiFRVYoM+m05x37vX8fz+Uxq+EZFYS2zQA3z4PW/j\nyug4uw/oQ1kRia9EB/1dGzq4oSXH0y//pNZNERFZMokO+nTK+OU71rP7wAAnzw/XujkiIksi0UEP\n8O82vZ0Jd762541aN0VEZEkkPui7O5rY/M4b+NpLbzIyNl7r5oiIVF3igx7gwbt7OH3xqsbqRSSW\nFPTAv964mnevb+O/7z7E1bGJWjdHRKSqFPREty7+9L03c/TMFf5y79HyLxARqSMK+uDf3txJ79vb\n+cLfHmJoeLTWzRERqRoFfWBmfPbD7+KtiyP8t28fqHVzRESqRkFf5N1dbfz6z2zgq3ve5KUj+k1Z\nEYkHBf00n773Zm7saOI/PPV9Tl8cqXVzREQWTUE/TXMuw+Of+JecuzzKb3/t+7q2XkTqnoK+hFvf\n1srnf+U9fPfwGX7nqR8wNq5LLkWkfinoZ/HAHev5/Q/fynOvnuI3//de3cpYROqWgn4Ov/6+Dfzh\nlnex+8AAH/3iP3F48GKtmyQiMm8K+jIevLuHP32wl6NnrvALj73Il//hiIZyRKSuKOgrsPmn1vDc\np/4NvT3t/P7/3c8Hv/B37PzhTxhV4ItIHbDo97xrq7e31/v6+mrdjLLcnef3n+Jz3z7A4cFLrG3N\n89F/1c39717LLWtaMLNaN1FEEsTM9rp7b9l6Cvr5G59wvnNwgC//4+u82H8ad3j7qiY2bVhFb087\nd9zYTs+qJjJpvWESkaWjoF8mgxdG+Jv9J9n92gB9b5zl/JXoPjnZtLFhdTPvuGEF69oaWdua54bW\nHGtb87Q3N9Caz9KSz9DUkNY7ARFZEAV9DUxMOP2DF9l37Dz9AxfpH7jA4cFLnDg/zJXR0l+8SqeM\nlnyGlnyGxmyaXCZNLpMil01dm85E09mMkTYjnUqRSRspMzIpI130yEybTk2WpUgZmEHKDDPDiKYL\n5WbROg1IpZhWx0IdiupEr4XCOqLnVDhxpcyi9RQtNwPj2roL5zjDKD7fFdpjxfNFdcJmKdQoXm9h\nefG6Maa8dq51F5fNtW6doKXWKg36zHI0JilSKePmNS3cvKZlSrm7MzQ8xsDQMCeHhjl3eZQLw2MM\nDY9yYXiUoStjXBgeZWRsIjzGGRmdYOjKWDQ9NsHI6ARXxycYn/Apj7GJCSZqf65OvHInEaYsn3oS\nmbKeGeu1OZfPLKhgHfPcZuk6c9cot43SdRbfzrLbmOc2K9nuYo/Jw5s38uHb3jbzRVWkoF8GZkZb\nY5a2xiwbp50EqsG9EPrOhEfP4+POeFH5+LjjOBMe1S88OzDhzsQEOI57NF94jk4i0fPExLX67hTV\nicp9cj3FdcK2irbtfm1bUfuZfH20teifaK3Fy0N9fLKsUGHKcvfJZdPXfa1s7nUX/22LtzWj7Vx7\nsVewbqasa+oZevqb6+nn75nLZ57hy71Bn7HNCl4/fTvzbefMGiXWUWbfym2zknWUmZ3xtyldZ/Hr\nmF7Q1pid8ZpqU9DHgJmRSRuZdK1bIiLXI10WIiIScwp6EZGYU9CLiMTckgS9mX3IzA6aWb+ZPbIU\n2xARkcpUPejNLA38T+A+4Fbg42Z2a7W3IyIilVmKHv1dQL+7H3b3q8BfAFuWYDsiIlKBpQj69cDR\novljoUxERGqgZh/Gmtl2M+szs77BwcFaNUNEJPaW4gtTx4HuovmuUDaFuz8BPAFgZoNm9sYCt7ca\nOL3A19Yr7XMyaJ+TYTH7/PZKKlX9pmZmlgH+GdhMFPDfA37N3V+t6oauba+vkpv6xIn2ORm0z8mw\nHPtc9R69u4+Z2W8DzwFp4EtLFfIiIlLektzrxt2fBZ5dinWLiMj8xOGbsU/UugE1oH1OBu1zMiz5\nPl8XPzwiIiJLJw49ehERmUNdB31c76ljZt1m9oKZ7TezV83s4VDeYWbPm9mh8Nweys3MHgt/h31m\ndmdt92BhzCxtZj8ws2fC/AYz2xP26+tm1hDKc2G+PyzvqWW7F8rMVprZN83sgJm9ZmZ3J+AY/8fw\nf/oVM3vKzPJxPM5m9iUzGzCzV4rK5n1szWxrqH/IzLYutD11G/Qxv6fOGPBpd78V2AQ8FPbtEWCX\nu28EdoV5iP4GG8NjO/D48je5Kh4GXiua/zzwqLu/AzgLbAvl24CzofzRUK8e/QnwbXd/J3Ab0b7H\n9hib2Xrgd4Bed/9poqvyPkY8j/OXgQ9NK5vXsTWzDuCzwHuJbi3z2cLJYd6in0qrvwdwN/Bc0fxn\ngM/Uul1LtK9PAx8ADgLrQtk64GCY/iLw8aL6k/Xq5UH0xbpdwD3AM0Q/s3kayEw/3kSX7t4dpjOh\nntV6H+a5v23AkentjvkxLtwepSMct2eAD8b1OAM9wCsLPbbAx4EvFpVPqTefR9326EnIPXXC29U7\ngD3AGnc/ERadBNaE6Tj8Lb4A/B4wEeZXAefcfSzMF+/T5P6G5edD/XqyARgE/lcYrvozM2smxsfY\n3Y8DfwS8CZwgOm57ifdxLjbfY1u1Y17PQR97ZrYC+CvgU+4+VLzMo1N8LC6ZMrNfBAbcfW+t27KM\nMsCdwOPufgdwiWtv5YF4HWOAMOywhegk9zagmZnDG4mw3Me2noO+onvq1CszyxKF/Ffd/Vuh+JSZ\nrQvL1wEDobze/xbvA37JzF4nuq31PUTj1yvDLTVg6j5N7m9Y3ga8tZwNroJjwDF33xPmv0kU/HE9\nxgA/Dxxx90F3HwW+RXTs43yci8332FbtmNdz0H8P2Bg+sW8g+lBnZ43bVBVmZsCTwGvu/sdFi3YC\nhU/etxKN3RfKHwyf3m8Czhe9Rbzuuftn3L3L3XuIjuNud/8E8ALwkVBt+v4W/g4fCfXrqufr7ieB\no2Z2SyjaDOwnpsc4eBPYZGZN4f94YZ9je5ynme+xfQ6418zaw7uhe0PZ/NX6A4tFfthxP9EN1H4M\n/Jdat6eK+/WzRG/r9gEvh8f9ROOTu4BDwN8CHaG+EV2B9GPgR0RXNdR8Pxa47+8HngnTNwEvAf3A\nXwK5UJ4P8/1h+U21bvcC9/V2oC8c5/8DtMf9GAN/ABwAXgH+HMjF8TgDTxF9DjFK9O5t20KOLfAb\nYf/7gU8utD36ZqyISMzV89CNiIhUQEEvIhJzCnoRkZhT0IuIxJyCXkQk5hT0IiIxp6AXEYk5Bb2I\nSMz9f3BssUJbPgCiAAAAAElFTkSuQmCC\n", 1189 | "text/plain": [ 1190 | "" 1191 | ] 1192 | }, 1193 | "metadata": {}, 1194 | "output_type": "display_data" 1195 | } 1196 | ], 1197 | "source": [ 1198 | "import matplotlib.pyplot as plt\n", 1199 | "%matplotlib inline\n", 1200 | "plt.plot(cost_trace)" 1201 | ] 1202 | }, 1203 | { 1204 | "cell_type": "markdown", 1205 | "metadata": { 1206 | "deletable": true, 1207 | "editable": true 1208 | }, 1209 | "source": [ 1210 | "# Listing 2-18b. Linear Regression Actual House Price vs Predicted House Price " 1211 | ] 1212 | }, 1213 | { 1214 | "cell_type": "code", 1215 | "execution_count": 48, 1216 | "metadata": { 1217 | "collapsed": false, 1218 | "deletable": true, 1219 | "editable": true 1220 | }, 1221 | "outputs": [ 1222 | { 1223 | "data": { 1224 | "text/plain": [ 1225 | "" 1226 | ] 1227 | }, 1228 | "execution_count": 48, 1229 | "metadata": {}, 1230 | "output_type": "execute_result" 1231 | }, 1232 | { 1233 | "data": { 1234 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnX2UXWV977+/OTmBSUQmaKQ4EBKBlSxyUxIzRbzRXonV\naHlxShDkxYu9rnJte6ugjobKksTay2hEsa2tcn0pXqKEN8dAqpGaxNooSOLkxZTkFoFQh1TSkkFI\nhuTMzO/+cfae2efMfvZ+9jn77Zz9/ayVNbP3OWfvZ3Zmnt/z/F6+P1FVEEIIKS4dWQ+AEEJIttAQ\nEEJIwaEhIISQgkNDQAghBYeGgBBCCg4NASGEFBwaAkIIKTg0BIQQUnBoCAghpOBMy3oANrz61a/W\nuXPnZj0MQghpKXbs2PEfqjo77H0tYQjmzp2L7du3Zz0MQghpKUTkgM376BoihJCCQ0NACCEFh4aA\nEEIKDg0BIYQUHBoCQggpOC2RNUQIIUViYHAIazftx7PDI3htVyf6VsxH75LuxO5HQ0AIITliYHAI\nNz2wByOVMQDA0PAIbnpgDwAkZgzoGiKEkByxdtP+CSPgMlIZw9pN+xO7Jw0BIYTkiGeHRyKdjwMa\nAkIIyRGv7eqMdD4OaAgIISRH9K2Yj3JJas6VS4K+FfMTuycNASGE5A0NOY4ZGgJCCMkRazftR2W8\nduavjCuDxYQQUhQYLCaEkILTNaMc6Xwc0BAQQkiOUEM8wHQ+DmgICCEkRwyPVCKdjwMaAkIIyREl\nkUjn44BaQ4QQkhI2YnJjBh+Q6XwccEdACCEp4IrJDQ2PQDEpJjcwOFTzvm5DBbHpfBzQEBBCSAqY\nxOQ+cs+uGmPQt2I+Ouq8QB0CVhYTQkirY6oDGFOt2RlsP/A86urJMK7V80lBQ0AIISkQJBrnlZle\n9+gzvu8xnY+DxA2BiJREZFBEHnKO54nIoyLyhIisF5HpSY+BEEKaZWBwCMv6N2Peqo1Y1r95im8/\njL4V89FZLhlfd3cM7VpH8CEAj3uOPwPgC6p6NoDDAN6fwhgIIaRhbAO9QfQu6catly0ypoEmKTMd\nRqKGQEROB3ARgK86xwJgOYD7nLfcCaA3yTEQQkizxNU1rHdJN2674rwpO4POcmkiGDyj7D8tm87H\nQdI7gtsBfAzAuHP8KgDDqjrqHP8KQHIdmQkhJAbiFIJzdwbdXZ0QVNNCb71s0UQ9wWVLT/f9nOl8\nHCRWUCYiFwN4TlV3iMhbGvj89QCuB4A5c+bEPDpCSJGwKeQK4rVdnRjymfQbdef0Luk23n/LvkOR\nzsdBkjuCZQAuFZGnAdyNqkvoiwC6RMQ1QKcD8HWyqeodqtqjqj2zZ89OcJiEkHYmDv++X6DX686J\nk7aSoVbVm1T1dFWdC+A9ADar6jUAtgC43HnbdQC+m9QYCCEkDv9+mDsnTrLoWZyF1tDHAdwtIp8G\nMAjgaxmMgRBSEBpdYfu5k7atWp7EEGvoWzEfNz2wp8Z4JbX7cEnFEKjqVgBbne+fBHB+GvclhBBb\n/7534j+5s4wjx0dRGasm77vuJACJ7AK8uNdvJqYRFaqPklzTbJCPtCdRfi9sVthuHMF9j5/2v+tO\nSuP3LyiYnAQ0BCS31P9xprkqI/kl6u+FzQrbL47gR5IB2yyhISC5JSjIR0OQP9LavQWpeN64fqfv\nvcNW2LYTfJbVv0lCQ0BySxZpdKQx0ty9Bal41t8bsPO1m+IIXpoN2EYxlGm7RGkISG6Ju4iHJEea\nuzebSXukMobVG/bi2Oi4r3Fyx+xOtBcumI37dwzV/AzlDsErTpyG4aOVpifjKIYyC5coDQHJLVmk\n0ZHGSHP35vd74Ycp4Lvmwb14uVJrIO7fMYSVS7uxZd+hRFbhUQxlFi5RGgKSW7JIo2sH4nQr2F4r\nzd1b/e9Fh0ikfr6Hj/obiC37DiVWJxDFUJp2O2G7oGagISC5Ju00ulYnTrdClGuF7d7i9nl7fy/q\nx+ne+8Ryh++kbyLJ2FMUQ1kyGDaTfHUcsEMZIS2KX6OUuOSSgWjSDEESDHFo/QRhuvctlyz01Qfq\n6iz7XifJ2FMUrSLT7ibKricq3BEQ0oKYVusmv3kjq13TZ4aGR7Csf7N1imYaPu+gnWN9UPihXQen\nvCfp2FMUN2e3YffQ3WZaQ4SQJjFNria3Qthq1891E5SdE8XllGUacJgLCQBmzSjjlksWNmSUori8\nbN2cFy6Yjbsemdqf+MIFyakw0zVESAsSlktfT9AkYnLdXLhgdmCPXVuXUxZqmn6YqodnTJ/WsBFI\nwuXVbv0ICCEJYZpETQHFeneIG1+Yu2ojbli/03d3sWXfoQnfuwmbVX2aWv5BxL0ziTMeYzOeJHdQ\ndA0RkiGNZtOYsnRMMYLhkQrmrtqIbkPxlB/PDo9MuDOW9W9uOD00L2nAcae4JjVhZ1FISUNASEY0\nk+rpN7leuGA21j3yDIJyS4aGR0Lf43KyJ7um2eK+IP94WnIKcRcoJjVhZxEjoCEgJCPCXAthk2P9\n5Lqsf7PVBG+bhOj1MiW1qk9TTiHunyGpyvcsYgQ0BIRkRFB6ZtDkaFpBx+1DPny0goHBoYmJ0m9V\n3+xqPm05hTgLFJMyjowREFIgTK6FkkjgTsFkJGzE2FwEdjuDoNW5aTW//cDz1po9ra4wm0TlexYx\nAmYNEZIRpmwaUwros8MjgStov+v50Vku4ZoL5lhJFrjXjlLFvO6RZ6xTKvOSWponssiyoiEgJAS/\nSTAOTNIIpnTNsAIvv+vdfuVi3H7l4in3+HTvIoxbSha4k3n95G4aS/1Vg1Iqo056Sf1f5IkguY6k\noGuIkAAaCWZGrTYFJv3Mazft903v7CyXjNkkwGT9gMlV0WgzFvfaUaqY/TC5eqL42YvUujSXPYtF\npBPAHFVtrlKCkBYjqC0i0HhTEddYDA2P1PjrTdr4rnEw4U7IA4NDWPPg3gnVza7OMlZf6i+fYKPr\nX+4QVMbNImjlkqAyFm4Mglw9tpMeW5cmR6ghEJFLAHwOwHQA80RkMYBPqeqlSQ+OkKwJknLwm+Bt\nJqt6Y+HnSvFq4w8MDuHD9+yEYT4GUHUfDAwOoe++XTUT8/BIBX33+hstUy2Ca4BO7izjyPFR4z1n\nzSjjpZdrX+8AUKozDnH5t1s9sByFPLaqXA3gfABbAUBVd4rIvMRGREiOCHKf+K1GbRQ7TZo3Xp4d\nHsHA4BBWb9jr22nLizvRrt2033d1XhlX46o5aDW+rH+z8d6d5RJUMWW3MA7gldOnYeYJ02KfxIrS\nujQLF5hNsLiiqi/UnUtOGJuQHBGWieNO8AODQxgYHEJHQCZOWJDVS9eMMm56YE+oESiJTAQSg1bG\nccpQA8Ctly3CC4axvTBSwbZVy/FU/0XYtmp5bJNXXjSLkiYpDaMgbHYEe0XkagAlETkHwAcB/CSx\nERGSI9xJ7CP37DIGRoeGR6ruFwlvHmITZHVX22G7BqB6P3eCCNq9NLJqNl2vu6sTvUu6J2IccdzL\nhrxoFiVNFi4wmx3BnwFYCOAYgG8BeAHADYmNiJCc0bukG7ddcV7gzqAyrlZBU2AyyOqHmypoWm37\n4ZWN9rtuuUMaWjWHrcCzWKH3LulOZLeRJ7KorQg1BKp6VFU/oaq/4/y7WVVfTmxEhOQQb253HIzV\nGY1yh+D2KxdPTG5R/+hHKmPYuPsgZk6v3eR3dZax9t3nNTRhhrWfdF0YbupqGvnuRSALAysaspUV\nkYcBvFtVh53jWQDuVtUViY2qjp6eHt2+fXtatyMtTFi2RRzZGCZJ5mbpdsbjTf9shs5yKZGJ2dQs\nnkYgPuLKGhKRHaraE/o+C0MwqKpLws4lCQ0BsSFsgoprAjO1PMwj3V2dE2mofjQy4ZgMYdi9SPrY\nGgKbYPG4iMxR1WecC58JZg2RHGLKtrhh/U6s3bQfR4+PxlKQ5A1aJrEzsMFWNC4owHjzwJ6a3gRD\nwyPou28XVm/YixdGKkbDUKR8/qJgYwg+AeCfReRHqP7+vRnA9YmOipAGCJqUg15rZAIL69wVFduJ\n3UUBK4kHU6xhYHDIt0FNZUwnUlZN+eutkM+fdkFWq2MTLP4+gNcDWA/gbgBLVXVT2OdE5EQR+ZmI\n7BKRvSKyxjk/T0QeFZEnRGS9iExv9ocgZGBwCOFamv50iDQsXmar+BlEd1dn5C12d1dnqGhcUIBx\n7ab9Vvf0y1/Pez5/Uk3l2xmjIRCRBc7X1wOYA+BZ598c51wYxwAsV9XzACwG8A4RuQDAZwB8QVXP\nBnAYwPub+xEIsZ/Y/HDlIhqZKOqziaIaI9evHiUbyZ10g1bg3kIzP6Lsgurfm4U6ZhSyKMhqdYJc\nQx9G1QV0m89rCiAwKqTVKPRLzmHZ+ed+7mrn/J2oSlj8nfWICfHBdmLr6izjxZdHp7hUmhEva9RN\n5CqKup/zcw91ljuwcunpxkYvfffumiLzUC4J1l4enDIapYmNn8FJWx0zCoxhRMdoCFT1ehHpAHCz\nqm5r5OIiUgKwA8DZAL4E4JcAhlXVVar6FQDf3yYRuR5OLGLOnDmN3J60IfW+X1ckzXY3cHx0LLDx\nSzNE/fzKpd01ctOKyVhBt4Vf233Nq0c0a0YZt1zirzbqxaQ82iGoEbfLk8vHllaIYeSNwGCxqo6L\nyN8AaChVVFXHACwWkS4A3wGwIMJn7wBwB1BNH23k/qS98BPjMunzmzhaGTe+1uxEEWWV3d3ViS37\nDk2ZiF0jsG3V8okmLFEa2NtikmvwO5fXlb+JpJrKtzM2WUM/FJGVAB7QsKIDA6o6LCJbALwRQJeI\nTHN2BacDYASHBOLV7k8Kv4kiauaJqXGMaZV94/qdvtdxlUebUaC0GXuUJjatRFE0ieLEpqDsRQAz\nAYwBGIGze1XVV4Z8bjaqyqXDTmObH6AaKL4OwP2qereIfBnAblX926BrsaCsuKRRvOXnhmmk+MwU\nI+jqLPvKMgcVZgH+Ka82RVus/CUutgVlNumjJ6lqh6qWVfWVznGgEXA4DcAWEdkN4DEAD6vqQwA+\nDuDDIvIEgFcB+JrFtUhBsdHub5YLF8yeMkFGzTwZGBwy7lhMssxBaZjNBDyZNUOiYtuq8jIAb0LV\nhfljVR0I+4yq7oZPbEFVn0S10Q0hoaSR6eG6cz7duyj0vn6TvbsCN+HWKdh0CHN3C81IPMedNcPi\nrPYndEcgIn8L4AMA9gD4BYAPiMiXkh4YIUB6mR7ffvTfrO9bX28QtmsZU0Xffbt86xRMssrNFG3F\nKWPM4qxiYNOPYDmAFar6DVX9BoDfR0gNASFx0bdiPkodjdYM21OfUho04a7esLfm2GalXRlTrHlw\nb+j7XJop2oqz8pdupmJg4xp6AtXK4gPO8RnOOUISZ/uB5zEW1LU9Jkp1LSZ7l3TjBkNWT337SNu0\n0ajS0nGnhjZyLdPPlZXYHkkGG0NwEoDHReRnqMYIzgewXUQ2AICqXprg+EjBqXfZJMUFr5sV6f3z\nVm2cmGBNxVlZElflr0nYrt5wktbGxhB8MvFREGIgTF3Ty/SS4Lhlu8h6Hnny8JSA7qwZZeMq3usv\nv/WyRbj1skUTK3D39Xq6OssNjS1LTM8/yv8LyT+hhkBVf5TGQAhplkaNADApPAdMulZuuWQh+u7b\nFdiL2PWXewO9A4NDUzSAyh2C1ZcubHh8WdEd0MCetA82wWJCEsOVUZi3aiOW9W/ONBulPgjau6Qb\nay8/byJga8JPnXPtu8+rCfQ22jc4a/IuOU3iwaqOgJAksJFR6OosTwnOJol3Uq/Pnz96fNTXVdRq\n6pxRoFxDMbAtKOsEMEdVmTNGYiMoNdGdaNKOSbqTup+RKncIyiWpcRUVYXXcLkatlUi7iC/UEIjI\nJQA+B2A6gHkishjAp5gtRFwa/aUNqoB1rxk15bIZvJO6n5GqjKtRN4iQuGhWcLARbHYEq1FNGd0K\nAKq6U0TmJTIakhhJrTCCfmmBYJeCKf/+5M5y6umY9cJzJiP1wkgFO295e+j1wp43ZRuICZudctzY\nGIKKqr4gtXt05o61EEmuMEy/tGse3IuXK+OB9zTJNo9UxnBs1Nw3IAmOHButOQ5rbmJqkPPs8Ai6\nZpTx0sujE1lD9cbR20im/nUaA5JFhzWbrKG9InI1gJKInCMifw3gJ4mNiMROkjIBpl/Ow0croffc\nsu+Q72fTNgJAtVrYq6ETlC3jp79z1yPPTBwfPlqZ0j5ypDKG1Rv24qYH9vgGvynbQFzi1IqyxcYQ\n/BmAhag2o/82gN8AuCGxEZHYSXKFEfWX03vPvPWQ9U7GQVo/jUpjD49MNY5ehpzYCCk2fSvmo1yq\nzZIolyTRpASbgrKjAD4B4BNOD+KZqvpyYiMisZNkD1dTW8ATpnX4rny994zS2jEtvMbJlC2TpAGj\ni4gAmOp8T9gZbyND/S0ReaWIzERVivpfRKQv2WGROEmyKMhv5bxyaTcqY1PdO/X39BtXWLbotRfM\naXrMQby2qzO0yK0RA9pZLsFGRJUuIrJ20/4prsXKuCb6e2HjGjpXVX8DoBfA9wDMA/DexEZEYqcZ\nSWPb67ua+n0r5mP9Y/+GI8enukBWLq1dYfuN6+zXzAy8V9Rm9VHoLJdw4YLZofr7fgasnnJJ0NVZ\nrnnetiKqeXOZkXTJIlhskzVUFpEyqobgb1S1IiLMGmoxbIuCmk1rXPPgXqM2z0O7Dk50ARsYHMKa\nB/dO1AkIspc2doXjwlL3/KptvVlDpudm6jpWT1rNeEg+SdKVa8LGEHwFwNMAdgH4JxE5E9WAMWkz\nmk0zHRgcCiwAGx6p4OaBPXho18Ep8YOsVxbdXZ3oXdKNGw09CPz0hKLuqGzkqptx2bE2oT3oWzF/\nithh0sFim+b1f6Wq3ar6+1rlAIALExsRyYxm00xt3nfXI8+kqh1kg3fyNa26FGhaFM/PFXbtBXNi\ncdmxpWSbkXKw2EZiwtSP4FMxj4VkTFDD9mX9m0NXmK3o266vKA5atcdR+JWUbk8W1agkGYKCxUn9\nX9oEi494/o0BeCeAuYmMhmRKkA/SZoXZir7to8drK4q9q3Y/8prVk0WAkSRDLoPFqnqb91hEPgdg\nU2IjIpn5esN82COVMdywfidWb9gLEWD4aKVmfHls2RjG4aOVKat8d9U+d9VG38/kcXLNIsBIkiGL\n/8tGGtPMAHB63AMhVbL09Yathl2GRyo4fLQyZXy2n88bfqv8gcEhY01DHidXNpBpH7L4v7QpKNsj\nIrudf3sB7Adwe2IjKjhJ6gLZ4NYERJnM66UZtq1annjhV9zUr8DWbtrvG58TIJeTa9K1IiQ9svi/\ntEkfvdjz/SiAX6vqqOnNpDny4uuN6uapH9/G3QeTGFZDdJZLWLm0G/fvGDL+PKW6Djim563Ir/wD\nG8i0D2n/X9qkjx4A0AXgEgB/AODcpAdVZLJQHvQjqpvHK8+8rH9zqg1lgiiJYKQyhi37DmHlUvMf\n1pjWrv9Nz7vV3F6E2GDjGvoQgHUAXuP8Wycif5b0wIpKFv5Bk7aO6+a5/crFgZIK7vje9vmtuGH9\nzswrhF3KJZmY4IeGR7AuQJ6ifoKnz50UCVENrlQQkd0A3qiqR5zjmQB+qqq/ncL4AAA9PT26ffv2\ntG6XOWlmDdVXEwOTrhSvZIJXQuHkzvJE1lDXjDJUkbsisSh0lku+PlhW6pJWR0R2qGpP6PssDMEe\nAL/jSk+LyIkAHlPVRbGM1IKiGYI0cCc50+pdUFvM6DdZ+hmRVqO+oIyQdsLWENgEi78B4FER+Y5z\n3Avga80MjmTHwODQlFaJftQvD/yqVBtt0JIntq1anvUQCMkcm4Kyz4vIVgBvck79oaoOhn1ORM4A\n8E0Ap6I6r9yhql8UkVMArEe1OvlpAFeo6uGGRl9Q6pU7uzrLuPi800LVL5tdwbsdtMIavKdBhyBQ\n1rncIYDAqIRKCJnE6BpyJmwjqvp84IVFTgNwmqr+XEROArAD1d3E+wA8r6r9IrIKwCxV/XjQtega\nmmRgcGiKMqEffq6cZf2bmw7kljsE06d1+PYbyBO3X7kYQLD086wZZQx+8u1pDouQVInDNbQD1ZW8\nADgNwLPutZ3zrwu6sKoeBHDQ+f5FEXkcQDeAdwF4i/O2OwFsBRBoCMgkazftt1rl+rly4ljBV8YV\nlRwYge6QNpfen7vv3l1TRLxKHYJbLlnY8P29gWQ3YP7CSIVBZdKSGA2Bqs5zvxeRQVVd0uhNRGQu\ngCUAHgVwqmMkAODfUXUdEUuiTOb17z25s9zS2T0u5Y6qNrvXPeZl1ozyxPd+So4AcNIJ0xqWe66P\nsXjHEIdCKSFpYxMsBppQwxaRVwC4H8ANqvob8VRwqqqaup2JyPUArgeAOXNaS64gbryrzw6RKcVP\nJrxFUQODQzhy3L8gfEa5o7rSbxF/erkkRiNQLtWu9E2G84UGDKJtjIXyz6TVsDUEDeG0uLwfwDpV\nfcA5/WsROU1VDzpxhOf8PquqdwC4A6jGCJIcZ56pn3xsjYDbf3dZ/2Y8OzwCMQRXXT+5bTZRFpRL\nAigmVvZHK+M4WhmfeN31Vfqlgsap5BglSyqPCqWEmDAaAhH5sOfwNXXHUNXPB11Yqkv/rwF4vO69\nGwBcB6Df+frdqIMuEqbJx5s145c1dOGC2TXaOib7MexZVR8bHfd/U8rMmlHGjOnTJn6WI8dGAw2U\nawT8UkH9NJMarRCOMrnnUaGUEBNBO4KTPN//n7pjG5YBeC+APSLiNoL9c1QNwD0i8n4ABwBcEfG6\nbYFt1apR/EyBp/svMl73rgA5BS+v7erEzQN7rN+fBsNHKzXZPPMMfQG8mJ6TX6P5RoO5pt1FPZSi\nIK1GULB4TTMXVtV/BoyS7m9t5tqtjk2TeHdCNzmC/FacjdQJzH1VZ6gRqK8yTpouT7AXsJuAg1bg\ncSk5mhRZZ04voVzqYNYQaVkSjREQf8L6y4ZN6G7WjM11gxAAP/llYDkISiK47YrzcOP6nakZgxdG\nKjWFa2GS2GmtwOPcXRCSJ2gIfEhabCys50DYhF4ZV2w/8PwU3Z+oxWI2E/tJJ1Z/RbpmlFOTlh5X\nYM2De2taRwKTE7BX9C7tyZia/6QdaaRVZVuTRqvIsJ4DNkHJdY88MzEmd8xJMDxSQd+9uxpKt2yG\neqPjSmJ/4crFmHnCtEyMACHtim3W0BTCsoZalTC3TRyEZbLY+MQVwEfu2WUcc5z4FWRlgV9spe/e\nXfjEd/ZMSF50dZax+tKFNA6ERCBoR3CS868HwB+jKg/RDeADAF6f/NCyIY1WkWE9Sf2aovgxpjqx\ne2k3ujrLU875GbzKuNboHrk7mDh3cIS0O6FZQyLyTwBer6ovOserAYTn87UocRYguZhiDqZVq9cn\nHjbJj1TGUIpQbRxG2hlCfnQIsPrSqTpAtsa4Mq6s7CUkAjYxglMBHPccH0cb6wPF3aKw0ZiD6xN/\nuv8iXHvBHGMeLlDdGQS9bqIkgmsvmFOzM7nmgjlWu5EkeeWJZd9JPIoxZmUvIfbYZA19E8DP6hrT\n3JnckLLFNkXQNrMojpjDp3sXoefMU/CRe3YZV/6NrOLHVfHp3qmN5nrOPMVqN5IUpsB0WBqpF1b2\nEmKPTWOavxSR7wF4s3PKqjFNKxOWImhTEOZimkz9zgcZF/drnK0hRapVu373cusZbli/M+Qq8WOa\nxP3SSF88NoqxumC2qc6CEOKPbR3BDAC/UdVviMhsEZmnqk8lObA8E2WVb/Lfl6TWmWNjXNyvJuVN\nE6ZuXu65oeER9N23q+Ye7vdpC9GFueHqjbRftzZmDRESjVBDICK3oJo5NB/V/sVlAHehqiVUSKJk\nFplcOfXnoxiXl172l5P2QwBc/YY5eGjXwcAJvTKm+PA9O3Hj+p01jVZO9sneaZRlZ52CbQGVzI00\nkmeBFyHNY7Mj+ANUm8r8HABU9Vmn9WRhiZJZZOqk1V33XlvjYmq0YkIBbNwdbARc3Mt6dxtx7AY6\nyx14/C/eCQBYvOYHvtc0qYcmXeVNCLHLGjqu1cbGCgAiMjPZIeWfKJlFppqAI8dGazKHwqqNXRrJ\nhklKGqJcEt98fy/V3sm/PXG8+tKF1s8ujSpvQoidIbhHRL4CoEtE/gjAPwL4arLDyj8nlicfXVdn\neUqjeBe3eGxWnaLm8EilZlKzNS55yoZZe/l5vhO7G/2oL5QDwovpaq4f4C4jhMSHTdbQ50TkbQB+\ng2qc4JOq+nDiI8spfsqg9Q1d/NwZM6ZPm7Iy98YAbNNW+1bM923GnjZusNudrN2guI2f39avn0aV\nNyHEYkcgIp9R1YdVtU9VP6qqD4vIZ9IYXB4JW6Wa3BlBaaTL+jdbuzt6l3TjyvPPCHxPRwPVZeVS\ntA/Vy1uMqU7sYPxqLpb1b8a8VRsj/ay27jJCSHPYBIvfBuDjdefe6XOuEDQiIR0mAzE0PDIlX99U\nmzAwOIT7dwRPpCURjEeQnHBX8e5uxJs11BGQ/hpkEL25/keOj6IypoE/lx9xtpkkhJgJUh/9YwB/\nAuAsEdnteekkAD9JemB5JSxjyGQo3BVzlGIwv/RRG6XRKG4j7yreb2L2c4UF/RzuRO++7pchZFtZ\nnedGMMxmIu1E0I7gWwC+B+BWAKs8519U1eC2Vm1MoxLS3lV3FOmGesPSrOxDuUPwihOr8Qp3Ve+u\n4k3BbmDqZGz6Ofx2Cn7Y+vnzWCcQpbKckFbAGCNQ1RdU9WkAXwTwvKoeUNUDAEZF5A1pDTBL/Hzb\njUhIe1fd21Ytn1JDEES9P7y+IjkqrhEAJova3MriKGmZprRYWxXUVvbzM5uJtBs26aN/B+Alz/FL\nzrm2xhT0vXlgT6BLoHdJN1Yu7Z6YsEsiWLm0dlXbt2K+tVrohQtm14ypGblpgbmmoDKmWPPg3inn\nTc8BgG9arA2t7udnNhNpN2yCxeIUlAEAVHVcRNq+17Fp1bfukWcmlD79XAJuMNedsMdUcf+OIfSc\necrEdZ8dHsGM6aWahiom1j3yDO565Bl0OUHXRrHpM+BnJIJWv9tWLcfaTftDC9Zcd1S7tJdMomcF\nIVliM6EeBl5gAAARCklEQVQ/KSIfxOQu4E8APJnckPKBaXVXP5nWBz5NE+fqDXtxbHR84rUjx8dQ\nLslENo0J99VmpB5MMhc2hK1+g1bBArTFxF8Ps5lIu2FjCD4A4K8A3IzqvPRDANcnOag8YNM32MU7\nGZomRr+JvDKmsXYXq+faC+ZM9BtY1r859Ofxk4sIW/0GBcf9tIPagTxnMxHSCKExAlV9TlXfo6qv\nUdVTVfVqVX0ujcFliV8w1OTX97oEoroH3LTSuCmXZMIdBdj1Qb74vNOmnAuTvoi7o1ur4Ab+n+q/\nCNtWLacRIC2N0RCIyMecr38tIn9V/y+9IcZD1OpWv+wgvzaO9ZOeaWI0BVXdrCM3k6i5nKBJKmNa\nk8Xi/XlMbNl3aMq5sCypKNpBhJB8ImpwS4jIJar6oIhc5/e6qqbWrrKnp0e3b9/e8OdNRVGNTFg2\nhUR+7wH8u4vNmlHGLZcsrAk2uzn6zTaSFwBP9V805fy8VRuN1+3u6qS7g5A2QUR2qGpP6PtMhiBP\nNGsITP7xtP3YA4NDvh2/6o2S1xgExRCWnXUKfvbUYWMlsennMz2PesMjAK7xxBkIIa2FrSEIcg09\nKCIbTP/iHW6y5CXvu3dJN2aeMDU+bxKtA6oxBJO76On/HMHad5/nG+QN8tOb4h/15kRRTV9tNf3/\nRkXuCCkqQVlDn3O+Xgbgt1BtTwkAVwH4dZKDihtTZkvXjDKW9W9O1RXSiGidac/27PBITaN52ywW\nv6wXU0aROu9rFRcR5R8IiY7REKjqjwBARG6r21o8KCKN+2kywC/vu1wSvPTy6EQxVFoTRqOidaZr\nuUTV5Kl/f1B6aStVzEbp/UwIqWIjMTFTRF7nHojIPAAt1a7SL7Nl5vRpU3zraejFhKVbmtJP691D\ncadoBsle5Llitt4N1A7GjJC0sSkouxHAVhF5EtX56EwA/zPsQyLydQAXA3hOVf+Lc+4UAOsBzAXw\nNIArVPVwQyOPSP0KeN6qjb7vS3rCCCtGMlWtrlzajS37DiXmxupd0o3tB56vkdBw753XmgA/N5Ap\n0yrPxoyQrLHKGhKREwAscA73qeoxi8/8LqoCdd/0GILPoqpk2i8iqwDMUtXQBjfNZg35YVo9dnWW\nMfOEacYJNw0d+iy17htNj83C7WKb/dRoqjAhrU5s6aMiMgPAhwGcqap/JCLnAJivqg9ZDGIugIc8\nhmA/gLeo6kEROQ3AVlUNXW4mYQj8agvKHQIIavR/vCmUUeoR8jJZxk2cNRnNwnoIQoKxNQQ2rqFv\nANgB4I3O8RCAewGEGgIfTlXVg873/w7gVNMbReR6OJpGc+bMaeBWwfi5aI4eH52ipOmmUPaceYp1\nILKdM1fyFIwtos4RIUlgEyw+S1U/C6ACAKp6FDEoITjS1sbtiKreoao9qtoze/Zs09uaol4vZtgg\np+ymUNrWI6TRuCSrXPm81GQAxdU5IiRubAzBcRHphDNpi8hZAEJjBAZ+7biE4HzNlXhdUEDR3TX4\noUDNZNzMZGkzwZuaxXjfm5ShMD2DLIKx1DkiJB5sDMEtAL4P4AwRWYeqDPXHGrzfBgCudtF1AL7b\n4HVCaWQiDEuhDFLw9E7GjU6WNhM8EL7j8LvOjet34uaBPYH3tyFvq3CqgBLSPIGGQEQEwD5Uq4vf\nB+DbAHpUdWvYhUXk2wB+CmC+iPxKRN4PoB/A20TkXwH8nnMcO7YTaj29S7pxzQVzjDn7YQqe7mTc\n6GRp61JqtDo5DrkIrsIJaT8Cg8WqqiLyD6q6CIB/4r35s1cZXnprlOs0QjMBzU/3LpoIDPtlnbj1\nCKaMFVf2wR1HlMwVW5dSo9XJcclFRK1iJoTkG5usoZ+LyO+o6mOJjyYm0ghohk3GjUyWpmueXCcq\nF9YqMUg7iBW2hJB6bGIEbwDwiIj8UkR2i8geEdmd9MCaoZmApq1b6cIFs2OXfehbMb9ay1DHkeOj\nNfcPc8+0qlwEISQbbHYEKxIfRcw001zcxq00MDiE+3cMTdHuX7m0OZdJ75JurHlw75RaBrfbmPfa\nQTuOVpSLIIRkh9EQiMiJqDauPxvAHgBfU9XRtAbWDM00F7dxK5mCsX6tHqNiqmWI6tIJi3UQQohL\n0I7gTlSLyH4M4J0AzgXwoTQGFQeNBjTDfP9A4zEIG9kJm/vbwqAuIcSGoBjBuap6rap+BcDlAN6c\n0phiodGCKpvUT9Ok3CFivI9t7CFvefqEkPYnyBBM+ChaxSXk0mgdAWCXJ28qLBtT9b3PwOAQPnLP\nLqsaAebpE0LSxqg+KiJjAI64hwA6Abg6Q6qqr0xlhIiuPppGs3p3cvdrLO+9j59apxcB8FT/RbGM\niRBCvDStPqqq/loKLUAadQS9S7px4/qdoffxCyx76ZoxtfF8I7Sr7DUhJHls0kdbjjgCrnEFdsOM\nj0VfIKuxtqvsNSEkeWwKylqOZgOucQZ2w4zPCyP+6aJRSEP2mhDSvrSlIWg24GqaWFdv2FuTiQSg\n4cCySxyVvnnqEUAIaT3a0jUENJdDb5pAh0cqGHZW8EPDI7hh/U50dZax+tKFgVW+AHwrhuNKC42z\n9oAQUjzackfQLFEm0OGRSmhqau+Sbgx+8u24/crFiaSFsvaAENIMoc3r80ASzeuDCEv59CPrPrnM\nGiKE1BNn8/rCYdvY3kvW/vh6V5hbWU3DQAgJg4bAgN/EGrRLyJM/numkhJAoMEZgiZuJNMunACxv\n/nimkxJCokBDEIGkg75xwXRSQkgU6BpqAK/byA3S3rh+Z2588UwnJYREgTuCJmhG5TRJmE5KCIkC\nDUET5NUXTylrQkgU6Bpqgjz74tmdjBBiC3cETWDyudMXTwhpJWgImoC+eEJIO0DXUBP4VSDnIWuI\nEEKiQEPQJEX0xVPXiJD2goaARILyFYS0H4wRkEjkNWWWENI4NAQkEnlOmSWENAZdQ21CWn57ylcQ\n0n5ksiMQkXeIyH4ReUJEVmUxhnYiTakLpswS0n6kbghEpATgSwDeCeBcAFeJyLlpj6OdSNNvT/kK\nQtqPLFxD5wN4QlWfBAARuRvAuwD8SwZjaQvS9tsXMWWWkHYmC9dQN4B/8xz/yjlHGoRSF4SQZsht\n1pCIXC8i20Vk+6FDh7IeTq6h354Q0gxZGIIhAGd4jk93ztWgqneoao+q9syePTu1wbUi9NsTQpoh\nixjBYwDOEZF5qBqA9wC4OoNxtBX02xNCGiV1Q6CqoyLyvwBsAlAC8HVV3Zv2OAghhFTJpKBMVf8B\nwD9kcW9CCCG15DZYTAghJB1oCAghpODQEBBCSMGhISCEkIJDQ0AIIQWHhoAQQgoODQEhhBQcGgJC\nCCk4NASEEFJw2KqyjrRaPhJCSF6gIfDgtnx0u325LR8B0BgQQtoWuoY8pNnykRBC8gINgYe0Wz4S\nQkgeoCHwwJaPhJAiQkPggS0fCSFFhMFiD25AmFlDhJAiQUNQB1s+EkKKBl1DhBBScGgICCGk4NAQ\nEEJIwaEhIISQgkNDQAghBUdUNesxhCIihwAcyHocTfJqAP+R9SByBJ/HJHwWtfB5TNLsszhTVWeH\nvaklDEE7ICLbVbUn63HkBT6PSfgsauHzmCStZ0HXECGEFBwaAkIIKTg0BOlxR9YDyBl8HpPwWdTC\n5zFJKs+CMQJCCCk43BEQQkjBoSFIABH5uog8JyK/8Jw7RUQeFpF/db7OynKMaSEiZ4jIFhH5FxHZ\nKyIfcs4X9XmcKCI/E5FdzvNY45yfJyKPisgTIrJeRKZnPda0EJGSiAyKyEPOcZGfxdMiskdEdorI\ndudc4n8rNATJ8PcA3lF3bhWAH6rqOQB+6BwXgVEAH1HVcwFcAOBPReRcFPd5HAOwXFXPA7AYwDtE\n5AIAnwHwBVU9G8BhAO/PcIxp8yEAj3uOi/wsAOBCVV3sSRtN/G+FhiABVPWfADxfd/pdAO50vr8T\nQG+qg8oIVT2oqj93vn8R1T/4bhT3eaiqvuQclp1/CmA5gPuc84V5HiJyOoCLAHzVORYU9FkEkPjf\nCg1Bepyqqged7/8dwKlZDiYLRGQugCUAHkWBn4fjCtkJ4DkADwP4JYBhVR113vIrVI1lEbgdwMcA\njDvHr0JxnwVQXRT8QER2iMj1zrnE/1bYmCYDVFVFpFDpWiLyCgD3A7hBVX9TXfhVKdrzUNUxAItF\npAvAdwAsyHhImSAiFwN4TlV3iMhbsh5PTniTqg6JyGsAPCwi+7wvJvW3wh1BevxaRE4DAOfrcxmP\nJzVEpIyqEVinqg84pwv7PFxUdRjAFgBvBNAlIu7C7HQAQ5kNLD2WAbhURJ4GcDeqLqEvopjPAgCg\nqkPO1+dQXSScjxT+VmgI0mMDgOuc768D8N0Mx5Iajs/3awAeV9XPe14q6vOY7ewEICKdAN6Gatxk\nC4DLnbcV4nmo6k2qerqqzgXwHgCbVfUaFPBZAICIzBSRk9zvAbwdwC+Qwt8KC8oSQES+DeAtqCoH\n/hrALQAGANwDYA6qSqpXqGp9QLntEJE3AfgxgD2Y9AP/OapxgiI+j99GNeBXQnUhdo+qfkpEXofq\nqvgUAIMArlXVY9mNNF0c19BHVfXioj4L5+f+jnM4DcC3VPUvReRVSPhvhYaAEEIKDl1DhBBScGgI\nCCGk4NAQEEJIwaEhIISQgkNDQAghBYeGgOQGEekVERWR0EpbEXmfiLy2iXu9xVW7DDsvIn8vIpfX\nvzcviMgHROS/Zz0O0rrQEJA8cRWAf3a+hvE+AA0bgnZBRKap6pdV9ZtZj4W0LjQEJBc4WkRvQlVy\n+D11r33c0WjfJSL9zuq8B8A6R7e909Fxf7Xz/h4R2ep8f76I/NTRu/+JiMxvcpxvda61x+k7cYJz\n3nT//+aMcafzObdytE9EHhOR3W5PAp97vSQiX3D6FvxQRGY757eKyO2OXv2HRGS1iHzUee1sEflH\n51n9XETOsr0fKS40BCQvvAvA91X1/wH4TxFZCgAi8k7ntTc4Gv6fVdX7AGwHcI2j2z4ScN19AN6s\nqksAfBLA/7YYy5s9k/dOAJc6YzkR1V4TV6rqIlSrP/845FofBfCnqroYwJsBjIjI2wGcg6qOzGIA\nS0Xkd30+OxPAdlVdCOBHqFaou0xX1R5Vva3uM+sAfMl5Vv8VwMEI9yMFhYaA5IWrUJUVgPPVdQ/9\nHoBvqOpRAGigtP5kAPdKtVvcFwAstPjMjx0Ds9iZwDc45+cDeMoxVkBVKiJsQt0G4PMi8kEAXY68\n8tudf4MAfo6q+ug5Pp8dB7De+f4uVHdMLuvr3+zsNrpV9TsAoKovO8/N9n6koFCGmmSOiJyCqvLk\nIkditwRARaQvwmVGMbmwOdFz/i8AbFHVP3D6IWxtesAR7q+q/SKyEcDvA9gmIisACIBbVfUrEe/h\n1YM5EuFzjd6PFATuCEgeuBzA/1XVM1V1rqqeAeApVF0pDwP4QxGZAUwYDQB4EcBJnms8DWCp8/1K\nz/mTMSlj/L4mx7kfwFwROds5fi+qLhvj/UXkLFXdo6qfAfAYqqvxTQD+hxMXgYh0O/rz9XRgUoXz\nalQD6UacDnC/EpFe57onOM/N9n6koNAQkDxwFSZVF13uB3CVqn4fVdfMdsdf/1Hn9b8H8GU3WAxg\nDYAvOgHUMc91PgvgVhEZRJM7YFV9GcAfoupqctVUv+y8bLr/DSLyCxHZDaAC4Huq+gMA3wLwU+c6\n96HWqLkcAXC+49ZaDuBTFsN8L4APOvf7CYDfinA/UlCoPkpIThGRl1T1FVmPg7Q/3BEQQkjB4Y6A\nEEIKDncEhBBScGgICCGk4NAQEEJIwaEhIISQgkNDQAghBYeGgBBCCs7/B5DXqX8PnL6vAAAAAElF\nTkSuQmCC\n", 1235 | "text/plain": [ 1236 | "" 1237 | ] 1238 | }, 1239 | "metadata": {}, 1240 | "output_type": "display_data" 1241 | } 1242 | ], 1243 | "source": [ 1244 | "#----------------------------------------------------------------------------------------------\n", 1245 | "\n", 1246 | "# Plot the Predicted house Prices vs the Actual House Prices\n", 1247 | "\n", 1248 | "#----------------------------------------------------------------------------------------------\n", 1249 | "\n", 1250 | "fig, ax = plt.subplots()\n", 1251 | "plt.scatter(Y_input,pred_)\n", 1252 | "ax.set_xlabel('Actual House price')\n", 1253 | "ax.set_ylabel('Predicted House price') " 1254 | ] 1255 | }, 1256 | { 1257 | "cell_type": "markdown", 1258 | "metadata": { 1259 | "deletable": true, 1260 | "editable": true 1261 | }, 1262 | "source": [ 1263 | "# Listing 2-19. Multiclass Classification with Softmax function using Full Batch Gradient Descent " 1264 | ] 1265 | }, 1266 | { 1267 | "cell_type": "code", 1268 | "execution_count": 49, 1269 | "metadata": { 1270 | "collapsed": false, 1271 | "deletable": true, 1272 | "editable": true 1273 | }, 1274 | "outputs": [ 1275 | { 1276 | "name": "stdout", 1277 | "output_type": "stream", 1278 | "text": [ 1279 | "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", 1280 | "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", 1281 | "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", 1282 | "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n", 1283 | "Epoch: 100 loss: 1.61233 accuracy: 0.658818181818\n", 1284 | "Epoch: 200 loss: 1.24327 accuracy: 0.753472727273\n", 1285 | "Epoch: 300 loss: 1.04145 accuracy: 0.788018181818\n", 1286 | "Epoch: 400 loss: 0.917034 accuracy: 0.806836363636\n", 1287 | "Epoch: 500 loss: 0.832832 accuracy: 0.818527272727\n", 1288 | "Epoch: 600 loss: 0.771862 accuracy: 0.827054545455\n", 1289 | "Epoch: 700 loss: 0.725474 accuracy: 0.833545454545\n", 1290 | "Epoch: 800 loss: 0.688842 accuracy: 0.839054545455\n", 1291 | "Epoch: 900 loss: 0.659069 accuracy: 0.843727272727\n", 1292 | "Epoch: 1000 loss: 0.634311 accuracy: 0.847763636364\n", 1293 | "Final training result: loss: 0.634311 accuracy: 0.847763636364\n", 1294 | "Results on test dataset: loss: 0.608456 accuracy: 0.8567\n" 1295 | ] 1296 | } 1297 | ], 1298 | "source": [ 1299 | "#----------------------------------------------------------------------------------------------\n", 1300 | "\n", 1301 | "# Import the required libraries\n", 1302 | "\n", 1303 | "#---------------------------------------------------------------------------------------------- \n", 1304 | "\n", 1305 | "import tensorflow as tf\n", 1306 | "import numpy as np\n", 1307 | "from sklearn import datasets\n", 1308 | "from tensorflow.examples.tutorials.mnist import input_data\n", 1309 | "\n", 1310 | "\n", 1311 | "\n", 1312 | "#----------------------------------------------------------------------------------------------\n", 1313 | "\n", 1314 | "# Function to Read the MNIST dataset along with the labels\n", 1315 | "\n", 1316 | "#---------------------------------------------------------------------------------------------- \n", 1317 | "\n", 1318 | "def read_infile():\n", 1319 | " mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n", 1320 | " train_X, train_Y,test_X, test_Y = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels\n", 1321 | " return train_X, train_Y,test_X, test_Y\n", 1322 | "\n", 1323 | "#----------------------------------------------------------------------------------------------\n", 1324 | "\n", 1325 | "# Define the weights and biases for the neural network\n", 1326 | "\n", 1327 | "#---------------------------------------------------------------------------------------------- \n", 1328 | "\n", 1329 | "def weights_biases_placeholder(n_dim,n_classes):\n", 1330 | " X = tf.placeholder(tf.float32,[None,n_dim])\n", 1331 | " Y = tf.placeholder(tf.float32,[None,n_classes])\n", 1332 | " w = tf.Variable(tf.random_normal([n_dim,n_classes],stddev=0.01),name='weights')\n", 1333 | " b = tf.Variable(tf.random_normal([n_classes]),name='weights')\n", 1334 | " return X,Y,w,b\n", 1335 | "\n", 1336 | "#----------------------------------------------------------------------------------------------\n", 1337 | "\n", 1338 | "# Define the forward pass\n", 1339 | "\n", 1340 | "#---------------------------------------------------------------------------------------------- \n", 1341 | "\n", 1342 | "def forward_pass(w,b,X):\n", 1343 | " out = tf.matmul(X,w) + b\n", 1344 | " return out \n", 1345 | "\n", 1346 | "#----------------------------------------------------------------------------------------------\n", 1347 | "\n", 1348 | "# Define the cost function for the softmax unit\n", 1349 | "\n", 1350 | "#---------------------------------------------------------------------------------------------- \n", 1351 | "\n", 1352 | "def multiclass_cost(out,Y):\n", 1353 | " cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=out,labels=Y))\n", 1354 | " return cost\n", 1355 | "\n", 1356 | "#----------------------------------------------------------------------------------------------\n", 1357 | "\n", 1358 | "# Define the initialiazation op\n", 1359 | "\n", 1360 | "#---------------------------------------------------------------------------------------------- \n", 1361 | "\n", 1362 | "def init():\n", 1363 | " return tf.global_variables_initializer()\n", 1364 | "\n", 1365 | "#----------------------------------------------------------------------------------------------\n", 1366 | "\n", 1367 | "# Define the training op\n", 1368 | "\n", 1369 | "#---------------------------------------------------------------------------------------------- \n", 1370 | " \n", 1371 | "def train_op(learning_rate,cost):\n", 1372 | " op_train = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n", 1373 | " return op_train\n", 1374 | " \n", 1375 | "\n", 1376 | "\n", 1377 | "train_X, train_Y,test_X, test_Y = read_infile()\n", 1378 | "X,Y,w,b = weights_biases_placeholder(train_X.shape[1],train_Y.shape[1])\n", 1379 | "out = forward_pass(w,b,X)\n", 1380 | "cost = multiclass_cost(out,Y)\n", 1381 | "learning_rate,epochs = 0.01,1000\n", 1382 | "op_train = train_op(learning_rate,cost)\n", 1383 | "init = init()\n", 1384 | "loss_trace = []\n", 1385 | "accuracy_trace = []\n", 1386 | "\n", 1387 | "#----------------------------------------------------------------------------------------------\n", 1388 | "\n", 1389 | "# Activate the Tensorflow session and execute the Stochastic gradient descent\n", 1390 | "\n", 1391 | "#---------------------------------------------------------------------------------------------- \n", 1392 | "\n", 1393 | "\n", 1394 | "with tf.Session() as sess:\n", 1395 | " sess.run(init)\n", 1396 | " \n", 1397 | " for i in xrange(epochs):\n", 1398 | " sess.run(op_train,feed_dict={X:train_X,Y:train_Y})\n", 1399 | " loss_ = sess.run(cost,feed_dict={X:train_X,Y:train_Y})\n", 1400 | " accuracy_ = np.mean(np.argmax(sess.run(out,feed_dict={X:train_X,Y:train_Y}),axis=1) == np.argmax(train_Y,axis=1))\n", 1401 | " loss_trace.append(loss_)\n", 1402 | " accuracy_trace.append(accuracy_)\n", 1403 | " if (((i+1) >= 100) and ((i+1) % 100 == 0 )) :\n", 1404 | " print 'Epoch:',(i+1),'loss:',loss_,'accuracy:',accuracy_\n", 1405 | " \n", 1406 | " print 'Final training result:','loss:',loss_,'accuracy:',accuracy_ \n", 1407 | " loss_test = sess.run(cost,feed_dict={X:test_X,Y:test_Y})\n", 1408 | " test_pred = np.argmax(sess.run(out,feed_dict={X:test_X,Y:test_Y}),axis=1)\n", 1409 | " accuracy_test = np.mean(test_pred == np.argmax(test_Y,axis=1))\n", 1410 | " print 'Results on test dataset:','loss:',loss_test,'accuracy:',accuracy_test \n", 1411 | " \n", 1412 | "\n", 1413 | " \n", 1414 | " " 1415 | ] 1416 | }, 1417 | { 1418 | "cell_type": "markdown", 1419 | "metadata": { 1420 | "deletable": true, 1421 | "editable": true 1422 | }, 1423 | "source": [ 1424 | "# Listing 2-19a. Display the Actual digits vs the Predicted digits along with the images of the actual digits " 1425 | ] 1426 | }, 1427 | { 1428 | "cell_type": "code", 1429 | "execution_count": 50, 1430 | "metadata": { 1431 | "collapsed": false, 1432 | "deletable": true, 1433 | "editable": true 1434 | }, 1435 | "outputs": [ 1436 | { 1437 | "name": "stdout", 1438 | "output_type": "stream", 1439 | "text": [ 1440 | "Actual digits: [7 2 1 0 4 1 4 9 5 9]\n", 1441 | "Predicted digits: [7 2 1 0 4 1 4 9 6 9]\n", 1442 | "Actual images of the digits follow:\n" 1443 | ] 1444 | }, 1445 | { 1446 | "data": { 1447 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAABSCAYAAABwglFkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXd4VFX6xz8nlSQESKghlFASqgKiiLL+RF1xRVREEVkL\nuihrYVHXhiz27qprw4KCrogVFHFRV0VRVuldCSBNikAAAwRIz/n98c6kkAQmM5OZO5P38zx5mDn3\n3Dvny33vvee+5z3vMdZaFEVRFEVRFO+ICHYDFEVRFEVRQhntTCmKoiiKoviAdqYURVEURVF8QDtT\niqIoiqIoPqCdKUVRFEVRFB/QzpSiKIqiKIoPaGdKURRFURTFB3zqTBlj/mSMWWuMWW+MGeuvRjkJ\n1Rj6hLs+UI3hQrhrDHd9oBrrLNZar/6ASGAD0B6IAVYAXb09nhP/VGPo/4W7PtUY/LapRtWnGsNL\nozd/vnim+gDrrbUbrbUFwHvAhT4cz4moxtAn3PWBagwXwl1juOsD1VhnifJh31Rga7nv24CTj6xk\njBkFjAKIJLJ3PA18+MnAUo8EiimkgUm2eRwCuIo6qDFc9LmKcoC3jqwXLhrrsp1C+GsMF32uIr0W\nUY1OJ49DFNh8c6x6vnSmPMJaOxGYCNDAJNuTzVm1/ZN+Y5fdxl520tWcyAI7m0IKqqwX7hrDRR/A\n13banqrqhYvGumynEP4aw0Uf6LWIagwJFtjZHtXzZZhvO9C63PdWrrKwIZY48sgtX6QaQ4wq9MUQ\nRvog/M8hqMZwQK/F8KAuaPQGXzpTi4B0Y0w7Y0wMcBkw0z/NcgYNSCKXg+TaQ1gsqMaQo7y+ElsC\nkEwY6YPwP4egGsMBvRbDg7qg0Ru8Huaz1hYZY0YD/0Wi+ydba3/2W8scQISJoJPtyTLmksthgA8C\noXHzw6cAUFxPQguadtvNvB7TK9Tp8M01JC6MA6D58z96/VvB0hgoyutzXfi/h5M+CP9zCKoxHNBr\nMTyoCxq9wac8U9baz6y1GdbaDtbaR/zVKCfRxKRwqvkT9WmIagxN3Pr6mXMBdga7PbVBuJ9DUI3h\ngF6L4UFd0FhTaj0AXfGc7FnpAPzU88VK2wptxe9rznidqSemAPDBV6cDUJz5S+02MMCY3t0AmDVz\nCgDHvTIagNYPee+JCxaRjRqy9sX2gJw7gPFZvVl1eQYAxavXBa1tiqLULaJaNAegIL1lpW3R6yT8\nae3d7Wm0WiaxJWfmARAxd1mAWhh66HIyiqIoiqIoPqCeKYeQPSudH3q+V+W2V/a155l5ZwOQ1nY3\nAF92/YjLE3cA8MjVTQBof1d4eaayTpK8JEUUAxD/mz1adUdT0q4Vq/q/CpR5GR9utoQeF50KQOsQ\n9EwVn3ECAKMnfgDAy+kda7R/zrC+NFous+OL1673b+MCzL6rJM5xweMvA9B1wo0AtHliIbaoKGjt\nqo6otjIRu9n7+wD4bklXADq/tI/in9d6dczIpk0B2HtuR5LeXwqAzc/3tamKn9h/RV/2DhQP09he\nXwBwVYPPKtWbtL8NAEMSPyZpaL0K2wal9q7lVoYu2pkKMkVniXF+02MCEA3As9ky9PPtMMnHwm9Z\nZGQvBiCinhj3owuOY1yTVXKMJOfdrP1B9vHSidpWJDfkxpPmBbM5XhHVuhUA7SaGdmehKn49JxaA\n5MiDXu2/87wCCq8U53jyIL81K+BEpbbkoXtfr1C2+qaXADj3+dOwOTnBaFa1RLVozoNzZEJLp+gS\nAM7c2wKA4p9r/kLm7kRd/j/pQPWt9zE3rfqrbFzmnLjkyCaNAVj7rzb0Txed208vBMKv0xfRowtr\n/pYAwNwBzwLQNHIRER4MRo1suMX1qd5R6ykV0WE+RVEURVEUH3CkZ2rvdeIyb3OlvM2vyWpOQb54\nbVLflX/jt8nbcMny1UFoof84mBoDQAQRpR6pORccB0Dxxsru9vUP9ALgneSnAfEMtPoi/PrEtl9P\n5g56BoDTv/8bAB0JneDHLffK8F3vP4l9Ppkyt8p69U+VYdut90j9JivFyxj3ycLabqLXmGix2TPP\nXO7TcRKX1ePSkd8B8G0j8eAV79vvW+OCQNY5bRkQX1ih7ITFwwBoetA5w7dRrVIBaPj+YY6PiQSg\n09fXA5A+YqnXx818OA2AS+vL0NEJz95Jy2XOmSSSNVqurftulpVrzov/snTb4CbnA1C0/bfAN6wW\nOdQukXXnvuz6FufRPq/skwkyU389qdo6DXGWhz2ipwxP57UQL9zmwYZL+iwCoNCKjX87pQ8AKd/t\nx9aipzT8nsKKoiiKoigBxJGeqTvveAeAixOypaBDuY395Z/NRYcBeG73GTU+/sKstgAkPN0QgKjZ\nS7xqpz9o9JbEAV2y+ApM9gEAinZsrrb+tQO/BqB+RGxtNy2o/N41jpTIeABSp0UHuTU1Z+VfXwCg\n0BYftd6cHlPlQw/55+NDku5ics5gor4Jnl0ejZyLJPD8+VTR2GWGpKxIZ0GNjpOfZBmTtAaAOYld\npDCEPFMR8WKf54z5X6Vtse8lyQfrnEkT2f0k6HxG2oTSsi7jswDwNurSntKD9YNkYsXpq4YC0Hry\nGo5u9YEhMkMeHK/fJjFDPWPkcVdSrs6OlxMBSPlrC4p2hFbaq6hWqWTeJR7d5j9KCoMG784HICLf\nsq5Q1szbWtQIgNZR+7j6pxEAZGdK/FjzRWKfjX7cij0ooz0N9znL+3Qktl9PADbeBO+c8hoAvV2e\n1iq5Q7z8ubcXMHGfeLJeWiHphNJHZlKSl+eXdjmyM/X8uMsAuPd4cZwlZVqyu4ixxBwvs0+e7P4R\nAP9KWcCsw/UBOC++ciBsrhWDWpAvbsD+9QohRW76HYdJkGSGZ+sY1irHyjO0+REZ+hzZ6ClXST1u\n29EXgMSvM+UYtda6wHPWjfOYcUhuAvXnyHBnKOiLniOdoWhzlIvbxbKCEjYXSvDuRQm/A3BpfXm4\nXTploiNnzth+PZnwxHMAvH1AXko6jxfbren5OWXAT/5sWsDJP1U6gA83m1RadrhE7jcN3pkflDZV\nhXvm3u4Lyx4aJz4lQ+cttno3HGdPkd7/+Kn/Li07OEuC2BP2bvTqmP4mc6x0aI8/yoN2QW95cV83\nr4AhU/4OQPtHJJzAXw9ZfxPZSJwAfWZtYkYTWcWl3+LRFerEfr6IO867GqB0dmZkl3SS124AILmk\n4vPGyVOYSv4gnafNMkGWWf3kZaBDVByy+Ap8lStDmeNWD2bfFnlu/DRYXvbu2SXPySdbLKZH3K8A\nPNPnfQDuvvVqWj3mnyFpHeZTFEVRFEXxAUd6phKmLXD9W1bW4Ig6L7ToD8DD/dJo8J24JZ/sXznP\nTVSuOHUTVkpOpsbfT+e4GFcQ++bQGD7ad+Up/HCVeKQaRsh01Xn5kSx/WILR4w44N1i5pkR26wTA\no83eZdKB0ApKzh3ch2tSPgTKhveqGubrPluCfpvOjiV2v2y/u7+816wa+nxpvW13S+Csv96c/EH2\n3YdpFSXvsX//23kARGfXbDgyKkU8GG+0+YJCG7rvc5uGVPZ4XPLLYNcn5wQ0b31OPPe/9HkTgPFZ\nPUl9QwJxvfX2bu8vnv5+sSV0/1GGjtq84Bw7jeyawddnPev6Jl6LJ/aKJ3Hxvja83+GLCvUzomN4\n7XIJ2H5i8oUAlGz6NTCN9RB3Wpz8aeKZGtfkGzp9JO6azh9XPp9H5gsLxRUyNr7Tk6mVhvLkfA7f\ndDaL1rQDoPPNMjrT9NBamrpqXd/7jwBkjREP+q0vRzK++RwA5ubKCMLy0S8w+G0530Vbt/nU1tC9\nkymKoiiKojgAR3qmPKFo5y4AEqbvKu2NJ0zbW239XddKzFG3mCie+l28H2lvyNi+k8eLAfacYEs9\nUm5GzLmWjBnh45Fys/3sxqWfl+S0dX3KDU5jPMTtTXv4mYmcGFPgLq1Q5+NDKYz/9mIAutwpQdfF\nBw6Ubu/0i6TFWHiBnOc+sXl8fsOTAAyodycAaY8uCVpyQXe6kg+P+ydv7T8egOivvQuQX/2gxPAU\n2mJGbJa3x+Ks3X5oZWA576QVpZ/3l4iNFt4va55FOMgzZa3Em7q9pAv2phGZm1WjY0QkSqD22kck\ngHfGBZK2pIRo2gxd5a+m+o09fRqTFiUTBEZt/T8AtvWVmNqIhMP0vl5ixm6/TrL3X56Yxf+5brGf\nTpeklavPEw+qEwLTI5OSWPOQ3CPWdpGEsEvyofOD8gwrfy8JZSISxOP5y4OSHijz9AlEuO6li/Il\nWP7yT24CoNMDmWTsk2TWJUceCDguUdYY/CpKvFeL/9mbxs/IqNfghH2uWsZ/bffbkRRFURRFUeog\nIeuZ8hT3TJYXx70IyCyrD5+Tt+HGO5y9PEnBV+KZmdf5adyp/XvMk/iELrdtCInZbTXlQNey5IfL\nX5RZHI1w9nkqcU25LvNKlfGXX/8EQM6wODK2iSexqvPmns1545sST7X4r8+SEimxAUtHSuzHxR+N\nwK7I9GvbPSVisKyh1zIqlknviKZW1CxGxu3Be/ssmUqfbwvZ8oy8bSfk1yytQjDJHyhJDV9Mfa20\nbJvLvR3xnfMTy37WeQYj50hKmS05EjtSMKlFtfV3nmYZeLIkaJ3Z8iVXqcSb9lt+GUk4LxanOBZK\nEE/GylfFy5Hsuo+UHDpEytNiux+cL+dyeOJ/wIp/Y1e+eOFsnnOWmPntii6svUhmp808JLMUJw06\nm+LdG4LZLL+zz5Ww+puhEiMcQTyzcyUN0OM3yrOv45cyU7aq+6iJiiKikyslxoxkAP75lsw4PS4m\nCxBvZaQRP9JxC/5MapZ//g/DvjO15lbJ+ntSrLjzfi7IJXn14WA26ZhEtU8D4KGOEsycFFGPJa7r\nuu1DYkLF2dnBaFqtkX+u3NQ+GSA3jAf39CZ5+kqgaheu0xm3S9ZVPHCtDFsWb/PsgZM2XTot9wzu\ny+MtFtVO42qAe9218RmzSstaPepdoPGaG2XK8omxYsMTsruSMD10OlFudp1UeeLK+f+5Bah5rq1A\n0OwF6ZR/O1FeyM6Iy2NSm28BiHANc5Q8U30+rAhMacfEzbs5MpzZeFyUI6/PxIt3lH7ef84hAJLf\nqFzv3rYzXZ/KBmnmLusMQEa2c8Iock4uC3V4btNZAMStC6+OFIAraTl5tmz4LadE7HfnybLyQu4Q\nyWjeMb3cOc4T2x7adik3NZoCwOICqd8v1m2h8aX1f8iTstSHjd9CJ3SYT1EURVEUxQfC1jOVf554\nOpZe8i9XibgKb7j5ZuJ+dM4bR1V0+EAC53rFlPV1h7um02esCL63ojbYdqaY4vEx8oYxYvNxNDu0\nJphNqjHlE3WuPMH9Jl/DIRAjb2RRESWVEn/+9gC0GFzVTrWHiZfzcU68pKfos+gqWuDdUGOTtN8r\nfJ+66USa4Jy16zwlpldFr3BmwWE6Py8eRScOvbsz6T/3hzMBeOjUNLYNEPtcf/4rACzMF7u74svr\nK+2f/lY+sz6cXKHsydXnAJC6ovbWOvOFnOkp0E0+X91VvIXfnyQejd296mMHiS12j5ZnQWZhId1c\na05+fK54x+/qe50cYP7KQDW7Wt7tNxG372Na17cBOOWZ22g3U0ILIud4v7aik0j6ROxp1FWXA/B2\n57e5IEFs9eIbZIi52Jb5QvOtjK/HmvJdGflc5pESiiim/0pJCJ58k1ypdqP/7Fc9U4qiKIqiKD4Q\ntp6pLedKP7G+EY/U8E1nAxD/xQqcs1pWRbJHyPTzB5o/7SqRto/Y/Ee63CmJSZ345usPmnaXqdru\nt46oT5KC2ZwasfYGGYs/1jp8nrB5iMRYTWu6sHTVc/dxW94X+Pixkt9lCvFDu2U9vj93WMz3KRLg\n6emUcfckkB96vucqkWszd34TCDHPVN6gPiw+6WXXNzk/awubURwC8SvudDLxH+0iQ1bjYuD1J1So\nk0Flr33E8Z1LY6se3tMdgLY3i6fSqWllWszcxLq7xWtzR+PVANw1Qzyq5eO/hm2QxLO5Y5py0btz\nALimwVYANowRO+3ggJWB+sRGl94HklxpctYMm0DhpVLmTgTccJFsO9jK0sC1qk+TlYdKj7PneEk9\n0HyO637rMLstyckBIHaA/Duq+RAy708DYEBvScGxbn8zAH7d3oTIGNF/QSfxHj7ZYnG1x+767Sg6\n3SajPkW7apYaxBPCsjMVkZjIlafJ4qMHSmR9paxH2wMQm+/MYbKo1JacNkbc0UcuYjxvdUcysp3Z\nbn8Q1a4tT3WSYPvX9suDN3mys2fwlWf8aZ96vW9Ua8nyntO7JQCvXPNSpToL8+UGaQoC/+hy39y+\n3C5BuXN7vsOO/0gG5rmvnlLtfvu6ygOrftp++rbcLMc6oitonPpWcxRym0RWGn69c8kQ2hH8oaDa\nYst9kaUdkC8fkZxN9bc6oIdxFIp27GTUHTIp4I2nJCdWRrR0JLAldPxShvA6j5ZQgpJDq3n8m/MB\nGDnYlQn9ROlxvt7jPEqCNIvWTbtPr2PdoFcqlbttce0fXTNL/+jZ8RaOlc7xLatdw16DnPlSU7wr\ni4wbpOOz2VUWg2SmT6csQ/2XH0v+s/Kdqc1FMtFs8AuSpy/92YUUF9XePVSH+RRFURRFUXwgLD1T\nv9zfjf80kTf8C3+RrNOxnznbs5M5rjUzWlT0cJyxaigAXe5cH7bDewC//LUlfV3OuOuWSv6b1vwU\nxBYFjtUPSH6fnwe8WGnb9INNAHj5drGDepnBmziR9IB4x06/fzgfd38TgCfuq957uDhf3piLiSiX\nf6tituE2L6xy5LT6o5E/eF/p58wCefNt9XporPFZU/aMEs/jyr4T2FwkU/PjdlfOpeZU6n8onv5r\n+DsAv18q5ytvfyxd7pDhreJDZUNgncbKcOBZ6UMA+KrbdADuuy+C1CGBaXN1dLppGed8OAqAq16U\n50R8RD6D4mXlgCO9pceiT6x4Gv/XayoA3f45hg53hM5ogJtNj4qNLj3JPdEspnTbJU+KR6rlBEnl\nUtuOcPVMKYqiKIqi+MAxPVPGmNbAW0BzpHM30Vr7nDEmGXgfSEOGMy+11gY1k+T+K/oCsHLY82wo\nkkzaB5+QmJRYdlS7X549zM8sooA8wJBKO9qYdAptAauYTy6HKSAPY0xSbWlccsG/cAecu2l4o7y3\nF/mYoNMTfXHEY4PkJyhpnVf6OXdfvaPUrB5PNXLkonlBJHpOCo+lTK92+5vbTwWg3qfikQqqnS6U\n4M+GA+HK/mMA2JceW231xq+VveVu/0jmqC85+c0KddzxWOVxwrVYFZEZEnS/+KS3cZvQ5wclGLsm\naxQ6/Vosz+GzD5Z+vmT5tQA0+/bYU/Cddi26PVT1Pywrq8rT77bHAx/LeXWnVnji+Om8lNIfKJt0\nEWg7tUVFpXb2bueWpeXPXyIxT8XR4vU99Xa5V3ia8DfC5U9p1aPy89Gp16Kb3+44lf9eLuuXxpmy\nhJzPZXcEoMUbkrU/UFeSJ8N8RcBt1tqlxphEYIkx5ivgamC2tfZxY8xYYCxwV+01tfYwGNI5ngYm\niSJbyEJmk2ybs4PNJNOMNNOZ7+0siikKSY2e6Nts17DFgctCeIqnGn8nq/p1MxxOuNsphL9GvRb1\nWgwV6oJGf3LMzpS1dgeIW8dam2OMyQRSgQuB/q5q/wbmEKT/0KhU6anfcs/7gCTwumzFlQA0/fzY\nPfRYE0cskrI+ykQTbxPJJ5fd/EZvTgcgmhgKyB1MADUWNpdZU9EFqVVuL94tiQLd6fBNrHgKIps2\nKavTtBH1gK23xeCeDGqfXsumAansfXMpzcddS+Q9WaTsb8sGVteOkGPw0slvl35O/dy7l1VPzmEK\nbVnPT37PuRBp5N2nfNzCgT/3rVDngQcncUZcXoWyaBNZLp1CZd32zO0VvjvFTt0JAhvP8ax+7mZZ\n64yTK5bbfj0xPyyvUOYUjUey6wyZjl3+HL/4raRbqckSMp7aabCuxfK82luW5dhRfJjGz8Yfo3YZ\nwbwW/UHTV8W7c/K5fwZgQe93uPn2NAA63CaeKafYacK0irb3aQ+JIXr8ykUcthLf1vv7GwBo+3ok\ne8ZI3Jh4WI+OUzQeSeEAWaprxugnaRNV0S63FB1m5l2y3E7s4cDGSdcoAN0Ykwb0AhYAzV0dLYCd\nyDBgwDFRUfT4zzYAhtbfC8DUnGY0v0fclzV18eXaQ+Swj4YkU0A+sUaMyUjwbEA1zpo2+ajbT102\nHIA9uxoAkNRU3NQLer9T7T6btxbyx507WHpNLukv/c6GS96na/5o2t2xHxvgDFx550tG4j/UW4g/\n50JUdw5jZLFov0+6ePz9SwC41LUgMcD3/5wAVMw9VVjFf291uam6z76edKofUnGSnR4TV9x5xBEh\nmkd2pI7ESRrzksuC55fky0OqyxNy3/F2svXR7DTQ12J5tt0tw8v9YsX+5ufHE+nB8F5VBPpa9Asl\nck02floe1Hum5JJ5mVzP579zFQB2SVnmbCfZaZv/utaZuxLijQRjZ54+SYrans1naf911ax4LW7Z\nmUx6afKByjhJ4+ZB8kKTVq4jtaNYOolX3XIb8bOCsz6mx8ZsjKkPTAdusdYeMKbs5mKttcZUnTXG\nGDMKGAVQD8/fboJBkS1iJfPoRE+iTHSF8H+X0YS0xoOHShh+bRZPPdiYBolHPNiMqXa6Q6jog2Oc\nw7qgMQzsFMJfY52307qgMQzsFOqGRn/gUWfKGBONdKSmWmtduXPZZYxJsdbuMMakAFWmFLXWTgQm\nAjQwyf5/3erRiYeaTalQNOHRoTRaUbNpniW2hJXMowVtaGZkWC2GWPJtLrEmzp1wsNY0Xrj6cmZ3\nn1ajfX7s9W6129wu3kJXRvHCQkvXYXE0OPk0xmcNZPybcDjmcTo/fxWtVxaRb3PdF0YlauscbrlA\nDhVronhwz3EA1P9Egiy9+ZFjncN8mwvVOBJ80dj+fRluXXhFPfrE5h2jdkXcCTkn7hS3efaNEkbS\neVPV6TCCbade4fqlI5N2VocTNTYrN+Q680AvoGyYvaZ4YqeBvhbLc/nw2dJO14kbufhq2iITECIb\nJ0ulZpKpvziz6tiuYF2L/iTiu2UA9P/3Haz+i3imch6RFBENhiZSdGC/4+w0erGcj75LhzP/hIrP\nhylpX+H2SOVbmaA1yJW0s/OYDY6/37htb9kQ9whA2QSY/v8bDUCHj4PjlQIPUiMYcUFNAjKttc+U\n2zQTGOH6PAL4xP/NCwzWWlazmAQSaWsySsub0pIdriyrhRRAiGq01nLr7fup17oxzYf0KS1P6NKN\nnGUyrryDX4kidPPleHIOXf/uq/oIzifc7RTCX6OndqrXorMJdzuFuqHRn3jimeoHXAmsMsa4gxzG\nAY8DHxhjRgK/ApfWThOrJrKrnNxR75Wdx66TbwIgbUrNljrYz152soX6NGS+/QqAjnSnLZ1YxXy2\n280UUwiiuVaIO2cT3R6V3rWt4qwkdpZVzquKh+o29xrZb0tCaVn7aa5pzQtXsc/uYTFzqM8vrP1E\n3lw60p2WJLOK+Szme+KIJxbv0hLUlMgGEuN1V7/PSsve+VyWqWhf5F3iOE/OoWs6dvU5MrykeLUs\nxXDv369l6/nifVl37qse7XvjZFlTq/UjP7pKqp9h7AQ79YaSehU9UruL86ut6zSN7kkdF7ZcUVq2\nt6A+UDbxoyZ4aqeBuhY9oaQ4gqzREkd13rVzAZixMQWgymSWwbwWa4OOE7cyZah4jL8/TkYP+rY7\nk50rnWOnbtzpHVr8LYnzJ18AwLi0WQCcEltcmgj4H58NkzbfKs/KqrxSTrkWI5NknsItC8T23Ovt\nAjyxtwsA6dfJcy2YCUWMtYHzojYwyfZkc5ZfjvXLizI16JeLXi4tG3CVrLcUNdvzvC+essDO5oD9\nvWrfezn8qTHQeKLRH/rcD6iMH8T0V2W3JO5i6UQUHzjg07GPxdd22hJr7YlHq+MPjQeGy2y+6Ktl\ncdkvur3PgJ/EpV7ypswMswaSlsukieqGS2qKE+303o0SvJxgZGhh+Ju3AtDmgR+r3edoBFKjiZI3\nmy3vydqEq099m+7zLwcgdcjP1e7nK4G6Fqvi9JUylOVeIDgCUzrk1+37vwDQ8X7JHF68dr3XvxOo\na9EfRHZJB+DTr2XG+ImL/0yzC9ccdR+nXIu7xkhHOOekXDqPl6Hpol+3+uXYgdCYPUJmKC58TJ71\nxbasy3T6GJmpmDC99ob3PNWoGdAVRVEURVF8wJlTU4+Cezr97POfdpWE/yyBcMM9PLLW9U4aw69h\nt/Zgg3ddQ82uGNCL6EMCG11bN5bWCzfdVfHgJhluOPSSBLC2me6dRyoYWNcq82ljxRPT5bErMcsT\ng9mkWue//5DJEKvvlqG8eQs60/m53wDosHMtAMV5NZtoEeq4PcfDNg4A4NNerzOy742ycf7KYDXL\nI5o/L9dbc7xP4xFMLr79a6CiRwqg46fXk1GLHqmaop4pRVEURVEUHwg5z9Rv/SRhV/nMp1NzXNmJ\nD0g6gOClu1MUpRJnSXLLBLYFuSHeU7x+EwBthga5IQHAvRbk7k/le0fmh6RHozY4fJE8XRb82JLs\nTjLhJ6lm852UGtIjbgsAkUZ8P/PzxJ/f9cksR9mleqYURVEURVF8IOQ8U0fy2N6uzDsnDQC7Y1Vw\nG6MoiqKELcV7ZPbtxIz2JOFdGhelZtwydSQAa657CYC/TP4bAK03Oiv2MuQ6U+3HigEPHHtCudKd\nwWmMoiiKoii1Rtv7pNN0zn09AWiNszpRbnSYT1EURVEUxQcCmrTTGLMbOAR4t6hVYGlCxXa2tdY2\nPdZOxpgcYG2ttcq/1FhjiJ9DCH+NntppXdCo16Jz0GuxGuqIxrC+FiHAnSkAY8ziY2W9dQLetjNU\n9EH4a/SlnarROYS7nUL4a1Q7rb19A0m42yl431Yd5lMURVEURfEB7UwpiqIoiqL4QDA6UxOD8Jve\n4G07Q0UfhL9GX9qpGp1DuNsphL9GtdPa2zeQhLudgpdtDXjMlKIoiqIoSjihw3yKoiiKoig+ELDO\nlDHmT8aTRstqAAACSklEQVSYtcaY9caYsYH63WNhjGltjPnWGLPaGPOzMeZmV/n9xpjtxpjlrr+B\nHhxLNQYJf2l0qj4If41qp6rxiOOEtT7XPqoxSPhTIwDW2lr/AyKBDUB7IAZYAXQNxG970LYU4ATX\n50RgHdAVuB+4XTXWHY1O1lcXNKqdqsa6ok81ho9G91+gPFN9gPXW2o3W2gLgPeDCAP32UbHW7rDW\nLnV9zgEygVQvDqUag4ifNDpWH4S/RrXTGhHuGsNdH6jGoOJHjUDghvlSga3lvm/Dh0bXFsaYNKAX\nsMBVNNoYs9IYM9kYk3SM3VWjQ/BBY0jog/DXqHZa5zWGuz5QjY7BR42ABqCXYoypD0wHbrHWHgBe\nBjoAPYEdwNNBbJ5fUI2qMRQId32gGgkDjeGuD1QjNdAYqM7UdqB1ue+tXGWOwBgTjfxnTrXWfgRg\nrd1lrS221pYAryHuyqOhGoOMHzQ6Wh+Ev0a1U9XoItz1gWoMOn7SCASuM7UISDfGtDPGxACXATMD\n9NtHxRhjgElAprX2mXLlKeWqXQT8dIxDqcYg4ieNjtUH4a9R7bQU1Rj++kA1BhU/ahRqGrHu7R8w\nEImW3wD8I1C/60G7/gBYYCWw3PU3EJgCrHKVzwRSVGP4a3SqvrqgUe1UNdYlfaoxfDRaazUDuqIo\niqIoii9oALqiKIqiKIoPaGdKURRFURTFB7QzpSiKoiiK4gPamVIURVEURfEB7UwpiqIoiqL4gHam\nFEVRFEVRfEA7U4qiKIqiKD6gnSlFURRFURQf+H/0CahrypDOjgAAAABJRU5ErkJggg==\n", 1448 | "text/plain": [ 1449 | "" 1450 | ] 1451 | }, 1452 | "metadata": {}, 1453 | "output_type": "display_data" 1454 | } 1455 | ], 1456 | "source": [ 1457 | "import matplotlib.pyplot as plt\n", 1458 | "%matplotlib inline\n", 1459 | "f, a = plt.subplots(1, 10, figsize=(10, 2))\n", 1460 | "print 'Actual digits: ', np.argmax(test_Y[0:10],axis=1)\n", 1461 | "print 'Predicted digits:',test_pred[0:10]\n", 1462 | "print 'Actual images of the digits follow:'\n", 1463 | "for i in range(10):\n", 1464 | " a[i].imshow(np.reshape(test_X[i],(28, 28)))" 1465 | ] 1466 | }, 1467 | { 1468 | "cell_type": "markdown", 1469 | "metadata": { 1470 | "deletable": true, 1471 | "editable": true 1472 | }, 1473 | "source": [ 1474 | "# Listing 2-20. Multiclass Classification with Softmax function using Stochastic Gradient Descent " 1475 | ] 1476 | }, 1477 | { 1478 | "cell_type": "code", 1479 | "execution_count": 42, 1480 | "metadata": { 1481 | "collapsed": false, 1482 | "deletable": true, 1483 | "editable": true 1484 | }, 1485 | "outputs": [ 1486 | { 1487 | "name": "stdout", 1488 | "output_type": "stream", 1489 | "text": [ 1490 | "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", 1491 | "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", 1492 | "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", 1493 | "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n", 1494 | "Epoch: 100 Average loss: 0.217315986888 accuracy: 0.938909090909\n", 1495 | "Epoch: 200 Average loss: 0.212252810056 accuracy: 0.939781818182\n", 1496 | "Epoch: 300 Average loss: 0.21044576737 accuracy: 0.940036363636\n", 1497 | "Epoch: 400 Average loss: 0.209570910985 accuracy: 0.940236363636\n", 1498 | "Epoch: 500 Average loss: 0.209083208171 accuracy: 0.940509090909\n", 1499 | "Epoch: 600 Average loss: 0.208780150793 accuracy: 0.940636363636\n", 1500 | "Epoch: 700 Average loss: 0.208576056226 accuracy: 0.940618181818\n", 1501 | "Epoch: 800 Average loss: 0.208429297805 accuracy: 0.940672727273\n", 1502 | "Epoch: 900 Average loss: 0.208318462426 accuracy: 0.9408\n", 1503 | "Epoch: 1000 Average loss: 0.208231372183 accuracy: 0.940872727273\n", 1504 | "Final epoch training results: Average loss: 0.208231372183 accuracy: 0.940872727273\n", 1505 | "Results on test dataset: Average loss: 0.459081 accuracy: 0.9155\n" 1506 | ] 1507 | } 1508 | ], 1509 | "source": [ 1510 | "def read_infile():\n", 1511 | " mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n", 1512 | " train_X, train_Y,test_X, test_Y = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels\n", 1513 | " return train_X, train_Y,test_X, test_Y\n", 1514 | "\n", 1515 | "\n", 1516 | "def weights_biases_placeholder(n_dim,n_classes):\n", 1517 | " X = tf.placeholder(tf.float32,[None,n_dim])\n", 1518 | " Y = tf.placeholder(tf.float32,[None,n_classes])\n", 1519 | " w = tf.Variable(tf.random_normal([n_dim,n_classes],stddev=0.01),name='weights')\n", 1520 | " b = tf.Variable(tf.random_normal([n_classes]),name='weights')\n", 1521 | " return X,Y,w,b\n", 1522 | "\n", 1523 | "def forward_pass(w,b,X):\n", 1524 | " out = tf.matmul(X,w) + b\n", 1525 | " return out \n", 1526 | "\n", 1527 | "def multiclass_cost(out,Y):\n", 1528 | " cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=out,labels=Y))\n", 1529 | " return cost\n", 1530 | "\n", 1531 | "def init():\n", 1532 | " return tf.global_variables_initializer()\n", 1533 | " \n", 1534 | " \n", 1535 | "def train_op(learning_rate,cost):\n", 1536 | " op_train = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n", 1537 | " return op_train\n", 1538 | " \n", 1539 | "\n", 1540 | "\n", 1541 | "train_X, train_Y,test_X, test_Y = read_infile()\n", 1542 | "X,Y,w,b = weights_biases_placeholder(train_X.shape[1],train_Y.shape[1])\n", 1543 | "out = forward_pass(w,b,X)\n", 1544 | "cost = multiclass_cost(out,Y)\n", 1545 | "learning_rate,epochs,batch_size = 0.01,1000,1000\n", 1546 | "num_batches = train_X.shape[0]/batch_size\n", 1547 | "op_train = train_op(learning_rate,cost)\n", 1548 | "init = init()\n", 1549 | "epoch_cost_trace = []\n", 1550 | "epoch_accuracy_trace = []\n", 1551 | "\n", 1552 | "with tf.Session() as sess:\n", 1553 | " sess.run(init)\n", 1554 | " \n", 1555 | " for i in xrange(epochs):\n", 1556 | " epoch_cost,epoch_accuracy = 0,0\n", 1557 | " \n", 1558 | " for j in xrange(num_batches):\n", 1559 | " sess.run(op_train,feed_dict={X:train_X[j*batch_size:(j+1)*batch_size],Y:train_Y[j*batch_size:(j+1)*batch_size]})\n", 1560 | " actual_batch_size = train_X[j*batch_size:(j+1)*batch_size].shape[0]\n", 1561 | " epoch_cost += actual_batch_size*sess.run(cost,feed_dict={X:train_X[j*batch_size:(j+1)*batch_size],Y:train_Y[j*batch_size:(j+1)*batch_size]})\n", 1562 | " \n", 1563 | " \n", 1564 | " epoch_cost = epoch_cost/float(train_X.shape[0])\n", 1565 | " epoch_accuracy = np.mean(np.argmax(sess.run(out,feed_dict={X:train_X,Y:train_Y}),axis=1) == np.argmax(train_Y,axis=1))\n", 1566 | " epoch_cost_trace.append(epoch_cost)\n", 1567 | " epoch_accuracy_trace.append(epoch_accuracy)\n", 1568 | " \n", 1569 | " if (((i +1) >= 100) and ((i+1) % 100 == 0 )) :\n", 1570 | " print 'Epoch:',(i+1),'Average loss:',epoch_cost,'accuracy:',epoch_accuracy\n", 1571 | " \n", 1572 | " print 'Final epoch training results:','Average loss:',epoch_cost,'accuracy:',epoch_accuracy\n", 1573 | " loss_test = sess.run(cost,feed_dict={X:test_X,Y:test_Y})\n", 1574 | " test_pred = np.argmax(sess.run(out,feed_dict={X:test_X,Y:test_Y}),axis=1)\n", 1575 | " accuracy_test = np.mean(test_pred == np.argmax(test_Y,axis=1))\n", 1576 | " print 'Results on test dataset:','Average loss:',loss_test,'accuracy:',accuracy_test \n", 1577 | " " 1578 | ] 1579 | }, 1580 | { 1581 | "cell_type": "markdown", 1582 | "metadata": { 1583 | "deletable": true, 1584 | "editable": true 1585 | }, 1586 | "source": [ 1587 | "# Listing 2-20a. Actual Digit vs Predicted digits for Softmax classification through Stochastic Gradient descent " 1588 | ] 1589 | }, 1590 | { 1591 | "cell_type": "code", 1592 | "execution_count": 43, 1593 | "metadata": { 1594 | "collapsed": false, 1595 | "deletable": true, 1596 | "editable": true 1597 | }, 1598 | "outputs": [ 1599 | { 1600 | "name": "stdout", 1601 | "output_type": "stream", 1602 | "text": [ 1603 | "Actual digits: [7 2 1 0 4 1 4 9 5 9]\n", 1604 | "Predicted digits: [7 2 1 0 4 1 4 9 6 9]\n", 1605 | "Actual images of the digits follow:\n" 1606 | ] 1607 | }, 1608 | { 1609 | "data": { 1610 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAABSCAYAAABwglFkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXd4VFX6xz8nlSQESKghlFASqgKiiLL+RF1xRVREEVkL\nuihrYVHXhiz27qprw4KCrogVFHFRV0VRVuldCSBNikAAAwRIz/n98c6kkAQmM5OZO5P38zx5mDn3\n3Dvny33vvee+5z3vMdZaFEVRFEVRFO+ICHYDFEVRFEVRQhntTCmKoiiKoviAdqYURVEURVF8QDtT\niqIoiqIoPqCdKUVRFEVRFB/QzpSiKIqiKIoPaGdKURRFURTFB3zqTBlj/mSMWWuMWW+MGeuvRjkJ\n1Rj6hLs+UI3hQrhrDHd9oBrrLNZar/6ASGAD0B6IAVYAXb09nhP/VGPo/4W7PtUY/LapRtWnGsNL\nozd/vnim+gDrrbUbrbUFwHvAhT4cz4moxtAn3PWBagwXwl1juOsD1VhnifJh31Rga7nv24CTj6xk\njBkFjAKIJLJ3PA18+MnAUo8EiimkgUm2eRwCuIo6qDFc9LmKcoC3jqwXLhrrsp1C+GsMF32uIr0W\nUY1OJ49DFNh8c6x6vnSmPMJaOxGYCNDAJNuTzVm1/ZN+Y5fdxl520tWcyAI7m0IKqqwX7hrDRR/A\n13banqrqhYvGumynEP4aw0Uf6LWIagwJFtjZHtXzZZhvO9C63PdWrrKwIZY48sgtX6QaQ4wq9MUQ\nRvog/M8hqMZwQK/F8KAuaPQGXzpTi4B0Y0w7Y0wMcBkw0z/NcgYNSCKXg+TaQ1gsqMaQo7y+ElsC\nkEwY6YPwP4egGsMBvRbDg7qg0Ru8Huaz1hYZY0YD/0Wi+ydba3/2W8scQISJoJPtyTLmksthgA8C\noXHzw6cAUFxPQguadtvNvB7TK9Tp8M01JC6MA6D58z96/VvB0hgoyutzXfi/h5M+CP9zCKoxHNBr\nMTyoCxq9wac8U9baz6y1GdbaDtbaR/zVKCfRxKRwqvkT9WmIagxN3Pr6mXMBdga7PbVBuJ9DUI3h\ngF6L4UFd0FhTaj0AXfGc7FnpAPzU88VK2wptxe9rznidqSemAPDBV6cDUJz5S+02MMCY3t0AmDVz\nCgDHvTIagNYPee+JCxaRjRqy9sX2gJw7gPFZvVl1eQYAxavXBa1tiqLULaJaNAegIL1lpW3R6yT8\nae3d7Wm0WiaxJWfmARAxd1mAWhh66HIyiqIoiqIoPqCeKYeQPSudH3q+V+W2V/a155l5ZwOQ1nY3\nAF92/YjLE3cA8MjVTQBof1d4eaayTpK8JEUUAxD/mz1adUdT0q4Vq/q/CpR5GR9utoQeF50KQOsQ\n9EwVn3ECAKMnfgDAy+kda7R/zrC+NFous+OL1673b+MCzL6rJM5xweMvA9B1wo0AtHliIbaoKGjt\nqo6otjIRu9n7+wD4bklXADq/tI/in9d6dczIpk0B2HtuR5LeXwqAzc/3tamKn9h/RV/2DhQP09he\nXwBwVYPPKtWbtL8NAEMSPyZpaL0K2wal9q7lVoYu2pkKMkVniXF+02MCEA3As9ky9PPtMMnHwm9Z\nZGQvBiCinhj3owuOY1yTVXKMJOfdrP1B9vHSidpWJDfkxpPmBbM5XhHVuhUA7SaGdmehKn49JxaA\n5MiDXu2/87wCCq8U53jyIL81K+BEpbbkoXtfr1C2+qaXADj3+dOwOTnBaFa1RLVozoNzZEJLp+gS\nAM7c2wKA4p9r/kLm7kRd/j/pQPWt9zE3rfqrbFzmnLjkyCaNAVj7rzb0Txed208vBMKv0xfRowtr\n/pYAwNwBzwLQNHIRER4MRo1suMX1qd5R6ykV0WE+RVEURVEUH3CkZ2rvdeIyb3OlvM2vyWpOQb54\nbVLflX/jt8nbcMny1UFoof84mBoDQAQRpR6pORccB0Dxxsru9vUP9ALgneSnAfEMtPoi/PrEtl9P\n5g56BoDTv/8bAB0JneDHLffK8F3vP4l9Ppkyt8p69U+VYdut90j9JivFyxj3ycLabqLXmGix2TPP\nXO7TcRKX1ePSkd8B8G0j8eAV79vvW+OCQNY5bRkQX1ih7ITFwwBoetA5w7dRrVIBaPj+YY6PiQSg\n09fXA5A+YqnXx818OA2AS+vL0NEJz95Jy2XOmSSSNVqurftulpVrzov/snTb4CbnA1C0/bfAN6wW\nOdQukXXnvuz6FufRPq/skwkyU389qdo6DXGWhz2ipwxP57UQL9zmwYZL+iwCoNCKjX87pQ8AKd/t\nx9aipzT8nsKKoiiKoigBxJGeqTvveAeAixOypaBDuY395Z/NRYcBeG73GTU+/sKstgAkPN0QgKjZ\nS7xqpz9o9JbEAV2y+ApM9gEAinZsrrb+tQO/BqB+RGxtNy2o/N41jpTIeABSp0UHuTU1Z+VfXwCg\n0BYftd6cHlPlQw/55+NDku5ics5gor4Jnl0ejZyLJPD8+VTR2GWGpKxIZ0GNjpOfZBmTtAaAOYld\npDCEPFMR8WKf54z5X6Vtse8lyQfrnEkT2f0k6HxG2oTSsi7jswDwNurSntKD9YNkYsXpq4YC0Hry\nGo5u9YEhMkMeHK/fJjFDPWPkcVdSrs6OlxMBSPlrC4p2hFbaq6hWqWTeJR7d5j9KCoMG784HICLf\nsq5Q1szbWtQIgNZR+7j6pxEAZGdK/FjzRWKfjX7cij0ooz0N9znL+3Qktl9PADbeBO+c8hoAvV2e\n1iq5Q7z8ubcXMHGfeLJeWiHphNJHZlKSl+eXdjmyM/X8uMsAuPd4cZwlZVqyu4ixxBwvs0+e7P4R\nAP9KWcCsw/UBOC++ciBsrhWDWpAvbsD+9QohRW76HYdJkGSGZ+sY1irHyjO0+REZ+hzZ6ClXST1u\n29EXgMSvM+UYtda6wHPWjfOYcUhuAvXnyHBnKOiLniOdoWhzlIvbxbKCEjYXSvDuRQm/A3BpfXm4\nXTploiNnzth+PZnwxHMAvH1AXko6jxfbren5OWXAT/5sWsDJP1U6gA83m1RadrhE7jcN3pkflDZV\nhXvm3u4Lyx4aJz4lQ+cttno3HGdPkd7/+Kn/Li07OEuC2BP2bvTqmP4mc6x0aI8/yoN2QW95cV83\nr4AhU/4OQPtHJJzAXw9ZfxPZSJwAfWZtYkYTWcWl3+LRFerEfr6IO867GqB0dmZkl3SS124AILmk\n4vPGyVOYSv4gnafNMkGWWf3kZaBDVByy+Ap8lStDmeNWD2bfFnlu/DRYXvbu2SXPySdbLKZH3K8A\nPNPnfQDuvvVqWj3mnyFpHeZTFEVRFEXxAUd6phKmLXD9W1bW4Ig6L7ToD8DD/dJo8J24JZ/sXznP\nTVSuOHUTVkpOpsbfT+e4GFcQ++bQGD7ad+Up/HCVeKQaRsh01Xn5kSx/WILR4w44N1i5pkR26wTA\no83eZdKB0ApKzh3ch2tSPgTKhveqGubrPluCfpvOjiV2v2y/u7+816wa+nxpvW13S+Csv96c/EH2\n3YdpFSXvsX//23kARGfXbDgyKkU8GG+0+YJCG7rvc5uGVPZ4XPLLYNcn5wQ0b31OPPe/9HkTgPFZ\nPUl9QwJxvfX2bu8vnv5+sSV0/1GGjtq84Bw7jeyawddnPev6Jl6LJ/aKJ3Hxvja83+GLCvUzomN4\n7XIJ2H5i8oUAlGz6NTCN9RB3Wpz8aeKZGtfkGzp9JO6azh9XPp9H5gsLxRUyNr7Tk6mVhvLkfA7f\ndDaL1rQDoPPNMjrT9NBamrpqXd/7jwBkjREP+q0vRzK++RwA5ubKCMLy0S8w+G0530Vbt/nU1tC9\nkymKoiiKojgAR3qmPKFo5y4AEqbvKu2NJ0zbW239XddKzFG3mCie+l28H2lvyNi+k8eLAfacYEs9\nUm5GzLmWjBnh45Fys/3sxqWfl+S0dX3KDU5jPMTtTXv4mYmcGFPgLq1Q5+NDKYz/9mIAutwpQdfF\nBw6Ubu/0i6TFWHiBnOc+sXl8fsOTAAyodycAaY8uCVpyQXe6kg+P+ydv7T8egOivvQuQX/2gxPAU\n2mJGbJa3x+Ks3X5oZWA576QVpZ/3l4iNFt4va55FOMgzZa3Em7q9pAv2phGZm1WjY0QkSqD22kck\ngHfGBZK2pIRo2gxd5a+m+o09fRqTFiUTBEZt/T8AtvWVmNqIhMP0vl5ixm6/TrL3X56Yxf+5brGf\nTpeklavPEw+qEwLTI5OSWPOQ3CPWdpGEsEvyofOD8gwrfy8JZSISxOP5y4OSHijz9AlEuO6li/Il\nWP7yT24CoNMDmWTsk2TWJUceCDguUdYY/CpKvFeL/9mbxs/IqNfghH2uWsZ/bffbkRRFURRFUeog\nIeuZ8hT3TJYXx70IyCyrD5+Tt+HGO5y9PEnBV+KZmdf5adyp/XvMk/iELrdtCInZbTXlQNey5IfL\nX5RZHI1w9nkqcU25LvNKlfGXX/8EQM6wODK2iSexqvPmns1545sST7X4r8+SEimxAUtHSuzHxR+N\nwK7I9GvbPSVisKyh1zIqlknviKZW1CxGxu3Be/ssmUqfbwvZ8oy8bSfk1yytQjDJHyhJDV9Mfa20\nbJvLvR3xnfMTy37WeQYj50hKmS05EjtSMKlFtfV3nmYZeLIkaJ3Z8iVXqcSb9lt+GUk4LxanOBZK\nEE/GylfFy5Hsuo+UHDpEytNiux+cL+dyeOJ/wIp/Y1e+eOFsnnOWmPntii6svUhmp808JLMUJw06\nm+LdG4LZLL+zz5Ww+puhEiMcQTyzcyUN0OM3yrOv45cyU7aq+6iJiiKikyslxoxkAP75lsw4PS4m\nCxBvZaQRP9JxC/5MapZ//g/DvjO15lbJ+ntSrLjzfi7IJXn14WA26ZhEtU8D4KGOEsycFFGPJa7r\nuu1DYkLF2dnBaFqtkX+u3NQ+GSA3jAf39CZ5+kqgaheu0xm3S9ZVPHCtDFsWb/PsgZM2XTot9wzu\ny+MtFtVO42qAe9218RmzSstaPepdoPGaG2XK8omxYsMTsruSMD10OlFudp1UeeLK+f+5Bah5rq1A\n0OwF6ZR/O1FeyM6Iy2NSm28BiHANc5Q8U30+rAhMacfEzbs5MpzZeFyUI6/PxIt3lH7ef84hAJLf\nqFzv3rYzXZ/KBmnmLusMQEa2c8Iock4uC3V4btNZAMStC6+OFIAraTl5tmz4LadE7HfnybLyQu4Q\nyWjeMb3cOc4T2x7adik3NZoCwOICqd8v1m2h8aX1f8iTstSHjd9CJ3SYT1EURVEUxQfC1jOVf554\nOpZe8i9XibgKb7j5ZuJ+dM4bR1V0+EAC53rFlPV1h7um02esCL63ojbYdqaY4vEx8oYxYvNxNDu0\nJphNqjHlE3WuPMH9Jl/DIRAjb2RRESWVEn/+9gC0GFzVTrWHiZfzcU68pKfos+gqWuDdUGOTtN8r\nfJ+66USa4Jy16zwlpldFr3BmwWE6Py8eRScOvbsz6T/3hzMBeOjUNLYNEPtcf/4rACzMF7u74svr\nK+2f/lY+sz6cXKHsydXnAJC6ovbWOvOFnOkp0E0+X91VvIXfnyQejd296mMHiS12j5ZnQWZhId1c\na05+fK54x+/qe50cYP7KQDW7Wt7tNxG372Na17cBOOWZ22g3U0ILIud4v7aik0j6ROxp1FWXA/B2\n57e5IEFs9eIbZIi52Jb5QvOtjK/HmvJdGflc5pESiiim/0pJCJ58k1ypdqP/7Fc9U4qiKIqiKD4Q\ntp6pLedKP7G+EY/U8E1nAxD/xQqcs1pWRbJHyPTzB5o/7SqRto/Y/Ee63CmJSZ345usPmnaXqdru\nt46oT5KC2ZwasfYGGYs/1jp8nrB5iMRYTWu6sHTVc/dxW94X+Pixkt9lCvFDu2U9vj93WMz3KRLg\n6emUcfckkB96vucqkWszd34TCDHPVN6gPiw+6WXXNzk/awubURwC8SvudDLxH+0iQ1bjYuD1J1So\nk0Flr33E8Z1LY6se3tMdgLY3i6fSqWllWszcxLq7xWtzR+PVANw1Qzyq5eO/hm2QxLO5Y5py0btz\nALimwVYANowRO+3ggJWB+sRGl94HklxpctYMm0DhpVLmTgTccJFsO9jK0sC1qk+TlYdKj7PneEk9\n0HyO637rMLstyckBIHaA/Duq+RAy708DYEBvScGxbn8zAH7d3oTIGNF/QSfxHj7ZYnG1x+767Sg6\n3SajPkW7apYaxBPCsjMVkZjIlafJ4qMHSmR9paxH2wMQm+/MYbKo1JacNkbc0UcuYjxvdUcysp3Z\nbn8Q1a4tT3WSYPvX9suDN3mys2fwlWf8aZ96vW9Ua8nyntO7JQCvXPNSpToL8+UGaQoC/+hy39y+\n3C5BuXN7vsOO/0gG5rmvnlLtfvu6ygOrftp++rbcLMc6oitonPpWcxRym0RWGn69c8kQ2hH8oaDa\nYst9kaUdkC8fkZxN9bc6oIdxFIp27GTUHTIp4I2nJCdWRrR0JLAldPxShvA6j5ZQgpJDq3n8m/MB\nGDnYlQn9ROlxvt7jPEqCNIvWTbtPr2PdoFcqlbttce0fXTNL/+jZ8RaOlc7xLatdw16DnPlSU7wr\ni4wbpOOz2VUWg2SmT6csQ/2XH0v+s/Kdqc1FMtFs8AuSpy/92YUUF9XePVSH+RRFURRFUXwgLD1T\nv9zfjf80kTf8C3+RrNOxnznbs5M5rjUzWlT0cJyxaigAXe5cH7bDewC//LUlfV3OuOuWSv6b1vwU\nxBYFjtUPSH6fnwe8WGnb9INNAHj5drGDepnBmziR9IB4x06/fzgfd38TgCfuq957uDhf3piLiSiX\nf6tituE2L6xy5LT6o5E/eF/p58wCefNt9XporPFZU/aMEs/jyr4T2FwkU/PjdlfOpeZU6n8onv5r\n+DsAv18q5ytvfyxd7pDhreJDZUNgncbKcOBZ6UMA+KrbdADuuy+C1CGBaXN1dLppGed8OAqAq16U\n50R8RD6D4mXlgCO9pceiT6x4Gv/XayoA3f45hg53hM5ogJtNj4qNLj3JPdEspnTbJU+KR6rlBEnl\nUtuOcPVMKYqiKIqi+MAxPVPGmNbAW0BzpHM30Vr7nDEmGXgfSEOGMy+11gY1k+T+K/oCsHLY82wo\nkkzaB5+QmJRYdlS7X549zM8sooA8wJBKO9qYdAptAauYTy6HKSAPY0xSbWlccsG/cAecu2l4o7y3\nF/mYoNMTfXHEY4PkJyhpnVf6OXdfvaPUrB5PNXLkonlBJHpOCo+lTK92+5vbTwWg3qfikQqqnS6U\n4M+GA+HK/mMA2JceW231xq+VveVu/0jmqC85+c0KddzxWOVxwrVYFZEZEnS/+KS3cZvQ5wclGLsm\naxQ6/Vosz+GzD5Z+vmT5tQA0+/bYU/Cddi26PVT1Pywrq8rT77bHAx/LeXWnVnji+Om8lNIfKJt0\nEWg7tUVFpXb2bueWpeXPXyIxT8XR4vU99Xa5V3ia8DfC5U9p1aPy89Gp16Kb3+44lf9eLuuXxpmy\nhJzPZXcEoMUbkrU/UFeSJ8N8RcBt1tqlxphEYIkx5ivgamC2tfZxY8xYYCxwV+01tfYwGNI5ngYm\niSJbyEJmk2ybs4PNJNOMNNOZ7+0siikKSY2e6Nts17DFgctCeIqnGn8nq/p1MxxOuNsphL9GvRb1\nWgwV6oJGf3LMzpS1dgeIW8dam2OMyQRSgQuB/q5q/wbmEKT/0KhU6anfcs/7gCTwumzFlQA0/fzY\nPfRYE0cskrI+ykQTbxPJJ5fd/EZvTgcgmhgKyB1MADUWNpdZU9EFqVVuL94tiQLd6fBNrHgKIps2\nKavTtBH1gK23xeCeDGqfXsumAansfXMpzcddS+Q9WaTsb8sGVteOkGPw0slvl35O/dy7l1VPzmEK\nbVnPT37PuRBp5N2nfNzCgT/3rVDngQcncUZcXoWyaBNZLp1CZd32zO0VvjvFTt0JAhvP8ax+7mZZ\n64yTK5bbfj0xPyyvUOYUjUey6wyZjl3+HL/4raRbqckSMp7aabCuxfK82luW5dhRfJjGz8Yfo3YZ\nwbwW/UHTV8W7c/K5fwZgQe93uPn2NAA63CaeKafYacK0irb3aQ+JIXr8ykUcthLf1vv7GwBo+3ok\ne8ZI3Jh4WI+OUzQeSeEAWaprxugnaRNV0S63FB1m5l2y3E7s4cDGSdcoAN0Ykwb0AhYAzV0dLYCd\nyDBgwDFRUfT4zzYAhtbfC8DUnGY0v0fclzV18eXaQ+Swj4YkU0A+sUaMyUjwbEA1zpo2+ajbT102\nHIA9uxoAkNRU3NQLer9T7T6btxbyx507WHpNLukv/c6GS96na/5o2t2xHxvgDFx550tG4j/UW4g/\n50JUdw5jZLFov0+6ePz9SwC41LUgMcD3/5wAVMw9VVjFf291uam6z76edKofUnGSnR4TV9x5xBEh\nmkd2pI7ESRrzksuC55fky0OqyxNy3/F2svXR7DTQ12J5tt0tw8v9YsX+5ufHE+nB8F5VBPpa9Asl\nck02floe1Hum5JJ5mVzP579zFQB2SVnmbCfZaZv/utaZuxLijQRjZ54+SYrans1naf911ax4LW7Z\nmUx6afKByjhJ4+ZB8kKTVq4jtaNYOolX3XIb8bOCsz6mx8ZsjKkPTAdusdYeMKbs5mKttcZUnTXG\nGDMKGAVQD8/fboJBkS1iJfPoRE+iTHSF8H+X0YS0xoOHShh+bRZPPdiYBolHPNiMqXa6Q6jog2Oc\nw7qgMQzsFMJfY52307qgMQzsFOqGRn/gUWfKGBONdKSmWmtduXPZZYxJsdbuMMakAFWmFLXWTgQm\nAjQwyf5/3erRiYeaTalQNOHRoTRaUbNpniW2hJXMowVtaGZkWC2GWPJtLrEmzp1wsNY0Xrj6cmZ3\nn1ajfX7s9W6129wu3kJXRvHCQkvXYXE0OPk0xmcNZPybcDjmcTo/fxWtVxaRb3PdF0YlauscbrlA\nDhVronhwz3EA1P9Egiy9+ZFjncN8mwvVOBJ80dj+fRluXXhFPfrE5h2jdkXcCTkn7hS3efaNEkbS\neVPV6TCCbade4fqlI5N2VocTNTYrN+Q680AvoGyYvaZ4YqeBvhbLc/nw2dJO14kbufhq2iITECIb\nJ0ulZpKpvziz6tiuYF2L/iTiu2UA9P/3Haz+i3imch6RFBENhiZSdGC/4+w0erGcj75LhzP/hIrP\nhylpX+H2SOVbmaA1yJW0s/OYDY6/37htb9kQ9whA2QSY/v8bDUCHj4PjlQIPUiMYcUFNAjKttc+U\n2zQTGOH6PAL4xP/NCwzWWlazmAQSaWsySsub0pIdriyrhRRAiGq01nLr7fup17oxzYf0KS1P6NKN\nnGUyrryDX4kidPPleHIOXf/uq/oIzifc7RTCX6OndqrXorMJdzuFuqHRn3jimeoHXAmsMsa4gxzG\nAY8DHxhjRgK/ApfWThOrJrKrnNxR75Wdx66TbwIgbUrNljrYz152soX6NGS+/QqAjnSnLZ1YxXy2\n280UUwiiuVaIO2cT3R6V3rWt4qwkdpZVzquKh+o29xrZb0tCaVn7aa5pzQtXsc/uYTFzqM8vrP1E\n3lw60p2WJLOK+Szme+KIJxbv0hLUlMgGEuN1V7/PSsve+VyWqWhf5F3iOE/OoWs6dvU5MrykeLUs\nxXDv369l6/nifVl37qse7XvjZFlTq/UjP7pKqp9h7AQ79YaSehU9UruL86ut6zSN7kkdF7ZcUVq2\nt6A+UDbxoyZ4aqeBuhY9oaQ4gqzREkd13rVzAZixMQWgymSWwbwWa4OOE7cyZah4jL8/TkYP+rY7\nk50rnWOnbtzpHVr8LYnzJ18AwLi0WQCcEltcmgj4H58NkzbfKs/KqrxSTrkWI5NknsItC8T23Ovt\nAjyxtwsA6dfJcy2YCUWMtYHzojYwyfZkc5ZfjvXLizI16JeLXi4tG3CVrLcUNdvzvC+essDO5oD9\nvWrfezn8qTHQeKLRH/rcD6iMH8T0V2W3JO5i6UQUHzjg07GPxdd22hJr7YlHq+MPjQeGy2y+6Ktl\ncdkvur3PgJ/EpV7ypswMswaSlsukieqGS2qKE+303o0SvJxgZGhh+Ju3AtDmgR+r3edoBFKjiZI3\nmy3vydqEq099m+7zLwcgdcjP1e7nK4G6Fqvi9JUylOVeIDgCUzrk1+37vwDQ8X7JHF68dr3XvxOo\na9EfRHZJB+DTr2XG+ImL/0yzC9ccdR+nXIu7xkhHOOekXDqPl6Hpol+3+uXYgdCYPUJmKC58TJ71\nxbasy3T6GJmpmDC99ob3PNWoGdAVRVEURVF8wJlTU4+Cezr97POfdpWE/yyBcMM9PLLW9U4aw69h\nt/Zgg3ddQ82uGNCL6EMCG11bN5bWCzfdVfHgJhluOPSSBLC2me6dRyoYWNcq82ljxRPT5bErMcsT\ng9mkWue//5DJEKvvlqG8eQs60/m53wDosHMtAMV5NZtoEeq4PcfDNg4A4NNerzOy742ycf7KYDXL\nI5o/L9dbc7xP4xFMLr79a6CiRwqg46fXk1GLHqmaop4pRVEURVEUHwg5z9Rv/SRhV/nMp1NzXNmJ\nD0g6gOClu1MUpRJnSXLLBLYFuSHeU7x+EwBthga5IQHAvRbk7k/le0fmh6RHozY4fJE8XRb82JLs\nTjLhJ6lm852UGtIjbgsAkUZ8P/PzxJ/f9cksR9mleqYURVEURVF8IOQ8U0fy2N6uzDsnDQC7Y1Vw\nG6MoiqKELcV7ZPbtxIz2JOFdGhelZtwydSQAa657CYC/TP4bAK03Oiv2MuQ6U+3HigEPHHtCudKd\nwWmMoiiKoii1Rtv7pNN0zn09AWiNszpRbnSYT1EURVEUxQcCmrTTGLMbOAR4t6hVYGlCxXa2tdY2\nPdZOxpgcYG2ttcq/1FhjiJ9DCH+NntppXdCo16Jz0GuxGuqIxrC+FiHAnSkAY8ziY2W9dQLetjNU\n9EH4a/SlnarROYS7nUL4a1Q7rb19A0m42yl431Yd5lMURVEURfEB7UwpiqIoiqL4QDA6UxOD8Jve\n4G07Q0UfhL9GX9qpGp1DuNsphL9GtdPa2zeQhLudgpdtDXjMlKIoiqIoSjihw3yKoiiKoig+ELDO\nlDHmT8aTRstqAAACSklEQVSYtcaY9caYsYH63WNhjGltjPnWGLPaGPOzMeZmV/n9xpjtxpjlrr+B\nHhxLNQYJf2l0qj4If41qp6rxiOOEtT7XPqoxSPhTIwDW2lr/AyKBDUB7IAZYAXQNxG970LYU4ATX\n50RgHdAVuB+4XTXWHY1O1lcXNKqdqsa6ok81ho9G91+gPFN9gPXW2o3W2gLgPeDCAP32UbHW7rDW\nLnV9zgEygVQvDqUag4ifNDpWH4S/RrXTGhHuGsNdH6jGoOJHjUDghvlSga3lvm/Dh0bXFsaYNKAX\nsMBVNNoYs9IYM9kYk3SM3VWjQ/BBY0jog/DXqHZa5zWGuz5QjY7BR42ABqCXYoypD0wHbrHWHgBe\nBjoAPYEdwNNBbJ5fUI2qMRQId32gGgkDjeGuD1QjNdAYqM7UdqB1ue+tXGWOwBgTjfxnTrXWfgRg\nrd1lrS221pYAryHuyqOhGoOMHzQ6Wh+Ev0a1U9XoItz1gWoMOn7SCASuM7UISDfGtDPGxACXATMD\n9NtHxRhjgElAprX2mXLlKeWqXQT8dIxDqcYg4ieNjtUH4a9R7bQU1Rj++kA1BhU/ahRqGrHu7R8w\nEImW3wD8I1C/60G7/gBYYCWw3PU3EJgCrHKVzwRSVGP4a3SqvrqgUe1UNdYlfaoxfDRaazUDuqIo\niqIoii9oALqiKIqiKIoPaGdKURRFURTFB7QzpSiKoiiK4gPamVIURVEURfEB7UwpiqIoiqL4gHam\nFEVRFEVRfEA7U4qiKIqiKD6gnSlFURRFURQf+H/0CahrypDOjgAAAABJRU5ErkJggg==\n", 1611 | "text/plain": [ 1612 | "" 1613 | ] 1614 | }, 1615 | "metadata": {}, 1616 | "output_type": "display_data" 1617 | } 1618 | ], 1619 | "source": [ 1620 | "import matplotlib.pyplot as plt\n", 1621 | "%matplotlib inline\n", 1622 | "f, a = plt.subplots(1, 10, figsize=(10, 2))\n", 1623 | "print 'Actual digits: ', np.argmax(test_Y[0:10],axis=1)\n", 1624 | "print 'Predicted digits:',test_pred[0:10]\n", 1625 | "print 'Actual images of the digits follow:'\n", 1626 | "for i in range(10):\n", 1627 | " a[i].imshow(np.reshape(test_X[i],(28, 28)))" 1628 | ] 1629 | } 1630 | ], 1631 | "metadata": { 1632 | "kernelspec": { 1633 | "display_name": "Python 2", 1634 | "language": "python", 1635 | "name": "python2" 1636 | }, 1637 | "language_info": { 1638 | "codemirror_mode": { 1639 | "name": "ipython", 1640 | "version": 2 1641 | }, 1642 | "file_extension": ".py", 1643 | "mimetype": "text/x-python", 1644 | "name": "python", 1645 | "nbconvert_exporter": "python", 1646 | "pygments_lexer": "ipython2", 1647 | "version": "2.7.12" 1648 | } 1649 | }, 1650 | "nbformat": 4, 1651 | "nbformat_minor": 2 1652 | } 1653 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-deep-learning-w-tensorflow/59ff2ad451216b6891d014f1d3a6df8fedcf8dbe/LICENSE.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apress Source Code 2 | 3 | This repository accompanies [*Pro Deep Learning with TensorFlow*](http://www.apress.com/9781484230954) by Santanu Pattanayak (Apress, 2018). 4 | 5 | [comment]: #cover 6 | 7 | 8 | Download the files as a zip using the green button, or clone the repository to your machine using Git. 9 | 10 | ## Releases 11 | 12 | Release v1.0 corresponds to the code in the published book, without corrections or updates. 13 | 14 | ## Contributions 15 | 16 | See the file Contributing.md for more information on how you can contribute to this repository. 17 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to Apress Source Code 2 | 3 | Copyright for Apress source code belongs to the author(s). However, under fair use you are encouraged to fork and contribute minor corrections and updates for the benefit of the author(s) and other readers. 4 | 5 | ## How to Contribute 6 | 7 | 1. Make sure you have a GitHub account. 8 | 2. Fork the repository for the relevant book. 9 | 3. Create a new branch on which to make your change, e.g. 10 | `git checkout -b my_code_contribution` 11 | 4. Commit your change. Include a commit message describing the correction. Please note that if your commit message is not clear, the correction will not be accepted. 12 | 5. Submit a pull request. 13 | 14 | Thank you for your contribution! --------------------------------------------------------------------------------