├── 1_overview_of_tensorflow.ipynb ├── 2_tensorflow_ops.ipynb ├── 3_1_linear_regression_in_tensorflow.ipynb ├── 3_2_logistic_regression_in_tensorflow.ipynb ├── 4_word2vec_visualize.ipynb ├── 5_randomization.ipynb ├── README.md ├── china_embed.png ├── process_data.py └── wrap_functions.py /1_overview_of_tensorflow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lecture 1: overview of tensorflow\n", 8 | "\n", 9 | "**Useful links:** \n", 10 | "1. [Lecture note](http://web.stanford.edu/class/cs20si/lectures/notes_01.pdf);\n", 11 | "2. [Leture slides](http://web.stanford.edu/class/cs20si/lectures/slides_01.pdf);\n", 12 | "3. [A blog: What is a TensorFlow Session?](http://danijar.com/what-is-a-tensorflow-session/);\n", 13 | "4. [Quora: How does graph creation work in TensorFlow](https://www.quora.com/How-does-graph-creation-work-in-TensorFlow);" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 1, 19 | "metadata": { 20 | "collapsed": true 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "import tensorflow as tf\n", 25 | "import numpy as np" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "metadata": {}, 31 | "source": [ 32 | "# Part 1" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "## Graphs and Session\n", 40 | "TensorFlow separates definition of computations from their execution\n", 41 | "1. definite your computations by assembling a graph\n", 42 | "2. execute your computations by running a (sub)graph in a session" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "### Tensor\n", 50 | "Tensor is an n-dimensional matrix\n", 51 | "1. 0-d tensor: scalar(number)\n", 52 | "2. 1-d tensor: vector\n", 53 | "3. 2-d tensor: matrix\n", 54 | "4. and so no (an rgb image can be 3-d tensor and an rgbd image can be 4-d tensor)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "metadata": { 61 | "collapsed": false 62 | }, 63 | "outputs": [ 64 | { 65 | "name": "stdout", 66 | "output_type": "stream", 67 | "text": [ 68 | "Tensor(\"Add:0\", shape=(), dtype=int32)\n", 69 | "Add\n" 70 | ] 71 | } 72 | ], 73 | "source": [ 74 | "a = tf.add(3, 5) # tf.int32 implicitly\n", 75 | "print(a)\n", 76 | "print(a.op.name)\n", 77 | "# Not 8: We just defince a graph. To get the value of a, we nend to create a session and\n", 78 | "# run the graph in the sesssion" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 28, 84 | "metadata": { 85 | "collapsed": false 86 | }, 87 | "outputs": [ 88 | { 89 | "name": "stdout", 90 | "output_type": "stream", 91 | "text": [ 92 | "Tensor(\"Const_12:0\", shape=(), dtype=int32)\n", 93 | "Const_12\n", 94 | "Tensor(\"Const_13:0\", shape=(3,), dtype=int32)\n", 95 | "Const_13\n", 96 | "Tensor(\"Const_14:0\", shape=(2, 2), dtype=int32)\n", 97 | "Const_14\n" 98 | ] 99 | } 100 | ], 101 | "source": [ 102 | "# understand print tensor\n", 103 | "a_1 = tf.constant(3)\n", 104 | "a_2 = tf.constant([1, 2, 3])\n", 105 | "a_3 = tf.constant([[1, 2],\n", 106 | " [3, 4]])\n", 107 | "print(a_1)\n", 108 | "print(a_1.op.name)\n", 109 | "print(a_2)\n", 110 | "print(a_2.op.name)\n", 111 | "print(a_3)\n", 112 | "print(a_3.op.name)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 3, 118 | "metadata": { 119 | "collapsed": false 120 | }, 121 | "outputs": [ 122 | { 123 | "name": "stdout", 124 | "output_type": "stream", 125 | "text": [ 126 | "Tensor(\"Add_1:0\", shape=(), dtype=float32)\n" 127 | ] 128 | } 129 | ], 130 | "source": [ 131 | "b = tf.add(3.0, 5.0) # tf.float32 implicitly\n", 132 | "print(b)" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 4, 138 | "metadata": { 139 | "collapsed": false 140 | }, 141 | "outputs": [ 142 | { 143 | "name": "stdout", 144 | "output_type": "stream", 145 | "text": [ 146 | "8\n" 147 | ] 148 | }, 149 | { 150 | "data": { 151 | "text/plain": [ 152 | ">" 153 | ] 154 | }, 155 | "execution_count": 4, 156 | "metadata": {}, 157 | "output_type": "execute_result" 158 | } 159 | ], 160 | "source": [ 161 | "sess = tf.Session() # Without arguments the session constructor launches the default graph\n", 162 | "print(sess.run(a)) # run the graph, now we get the value of a\n", 163 | "#print(sess.run(b))\n", 164 | "sess.close()" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 5, 170 | "metadata": { 171 | "collapsed": false 172 | }, 173 | "outputs": [ 174 | { 175 | "name": "stdout", 176 | "output_type": "stream", 177 | "text": [ 178 | "8\n" 179 | ] 180 | } 181 | ], 182 | "source": [ 183 | "# use 'with as' statement to make your code neat\n", 184 | "with tf.Session() as sess:\n", 185 | " print(sess.run(a))" 186 | ] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "metadata": {}, 191 | "source": [ 192 | "### running a graph or a subgraph" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": { 199 | "collapsed": false 200 | }, 201 | "outputs": [], 202 | "source": [ 203 | "# defining a graph\n", 204 | "x = 2\n", 205 | "y = 3\n", 206 | "op1 = tf.add(x, y)\n", 207 | "op2 = tf.mul(x, y)\n", 208 | "op3 = tf.pow(op2, op1)\n", 209 | "# tf.mul, tf.sub and tf.neg are replaced by tf.multiply, tf.substract and tf.negative\n", 210 | "# since my tensorflow version is 1.0.0, it raise a error\n", 211 | "# you might replace tf.mul by tf.multiply if you are using 1.0.0\n", 212 | "# to get your version by:\n", 213 | "# python -c 'import tensorflow as tf; print(tf.__version__)' # for Python 2\n", 214 | "# python3 -c 'import tensorflow as tf; print(tf.__version__)' # for Python 3" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": 6, 220 | "metadata": { 221 | "collapsed": false 222 | }, 223 | "outputs": [ 224 | { 225 | "name": "stdout", 226 | "output_type": "stream", 227 | "text": [ 228 | "[, , , , ]\n", 229 | "5\n" 230 | ] 231 | } 232 | ], 233 | "source": [ 234 | "# replace tf.mul by tf.multiply if you are using 1.0.0\n", 235 | "graph = tf.Graph()\n", 236 | "with graph.as_default():\n", 237 | " x = tf.constant(2) # tf.constant() will build a node\n", 238 | " y = tf.constant(3)\n", 239 | " op1 = tf.add(x, y)\n", 240 | " op2 = tf.multiply(x, y)\n", 241 | " op3 = tf.pow(op2, op1)\n", 242 | "print(graph.get_operations()) # Return the list of operations in the graph.\n", 243 | "print(graph.version) #Returns a version number that increases as ops are added to the graph." 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": 7, 249 | "metadata": { 250 | "collapsed": false 251 | }, 252 | "outputs": [ 253 | { 254 | "name": "stdout", 255 | "output_type": "stream", 256 | "text": [ 257 | "7776\n" 258 | ] 259 | } 260 | ], 261 | "source": [ 262 | "# run the graph\n", 263 | "with tf.Session(graph = graph) as sess:\n", 264 | " op3 = sess.run(op3)\n", 265 | " print(op3) # 6^5 = 7776" 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": {}, 271 | "source": [ 272 | "#### running a subgraph example" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": 8, 278 | "metadata": { 279 | "collapsed": false 280 | }, 281 | "outputs": [ 282 | { 283 | "name": "stdout", 284 | "output_type": "stream", 285 | "text": [ 286 | "[, , , , , ]\n", 287 | "6\n" 288 | ] 289 | } 290 | ], 291 | "source": [ 292 | "graph = tf.Graph()\n", 293 | "with graph.as_default():\n", 294 | " x = tf.constant(2)\n", 295 | " y = tf.constant(3)\n", 296 | " op1 = tf.add(x, y)\n", 297 | " op2 = tf.multiply(x, y)\n", 298 | " useless = tf.multiply(x, op1) # this multipy operation wolud not be run in case 1\n", 299 | " op3 = tf.pow(op2, op1)\n", 300 | "print(graph.get_operations())\n", 301 | "print(graph.version)" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 9, 307 | "metadata": { 308 | "collapsed": false 309 | }, 310 | "outputs": [ 311 | { 312 | "name": "stdout", 313 | "output_type": "stream", 314 | "text": [ 315 | "7776\n" 316 | ] 317 | } 318 | ], 319 | "source": [ 320 | "# case 1\n", 321 | "with tf.Session(graph = graph) as sess:\n", 322 | " op3 = sess.run(op3)\n", 323 | " print(op3)\n", 324 | " #print(useless)" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": 10, 330 | "metadata": { 331 | "collapsed": false 332 | }, 333 | "outputs": [ 334 | { 335 | "name": "stdout", 336 | "output_type": "stream", 337 | "text": [ 338 | "[, , , , , ]\n", 339 | "6\n" 340 | ] 341 | } 342 | ], 343 | "source": [ 344 | "graph = tf.Graph()\n", 345 | "with graph.as_default():\n", 346 | " x = tf.constant(2)\n", 347 | " y = tf.constant(3)\n", 348 | " op1 = tf.add(x, y)\n", 349 | " op2 = tf.multiply(x, y)\n", 350 | " useless = tf.multiply(x, op1) # run this operation\n", 351 | " op3 = tf.pow(op2, op1)\n", 352 | "print(graph.get_operations())\n", 353 | "print(graph.version)" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": 11, 359 | "metadata": { 360 | "collapsed": false 361 | }, 362 | "outputs": [ 363 | { 364 | "name": "stdout", 365 | "output_type": "stream", 366 | "text": [ 367 | "7776\n", 368 | "10\n" 369 | ] 370 | } 371 | ], 372 | "source": [ 373 | "# case 2\n", 374 | "with tf.Session(graph = graph) as sess:\n", 375 | " op3, not_useless = sess.run([op3, useless])\n", 376 | " print(op3)\n", 377 | " print(not_useless)" 378 | ] 379 | }, 380 | { 381 | "cell_type": "markdown", 382 | "metadata": {}, 383 | "source": [ 384 | "## Distributed comptation" 385 | ] 386 | }, 387 | { 388 | "cell_type": "code", 389 | "execution_count": 12, 390 | "metadata": { 391 | "collapsed": false 392 | }, 393 | "outputs": [ 394 | { 395 | "name": "stdout", 396 | "output_type": "stream", 397 | "text": [ 398 | "[, , ]\n", 399 | "3\n" 400 | ] 401 | } 402 | ], 403 | "source": [ 404 | "with tf.device('/gpu:2'):\n", 405 | " graph = tf.Graph()\n", 406 | " with graph.as_default():\n", 407 | " a = tf.constant(np.array([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]]), name='a')\n", 408 | " b = tf.constant(np.array([[1.0], \n", 409 | " [2.0], \n", 410 | " [3.0], \n", 411 | " [4.0], \n", 412 | " [5.0], \n", 413 | " [6.0]]), name='b')\n", 414 | " c = tf.matmul(a, b)\n", 415 | " print(graph.get_operations())\n", 416 | " print(graph.version)" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": 13, 422 | "metadata": { 423 | "collapsed": false 424 | }, 425 | "outputs": [ 426 | { 427 | "name": "stdout", 428 | "output_type": "stream", 429 | "text": [ 430 | "[[ 91.]]\n" 431 | ] 432 | } 433 | ], 434 | "source": [ 435 | "with tf.Session(graph=graph, config=tf.ConfigProto(log_device_placement=True)) as sess:\n", 436 | " c = sess.run(c)\n", 437 | " print(c)" 438 | ] 439 | }, 440 | { 441 | "cell_type": "code", 442 | "execution_count": 22, 443 | "metadata": { 444 | "collapsed": false 445 | }, 446 | "outputs": [ 447 | { 448 | "name": "stdout", 449 | "output_type": "stream", 450 | "text": [ 451 | "[, , ]\n", 452 | "3\n" 453 | ] 454 | } 455 | ], 456 | "source": [ 457 | "with tf.device('/gpu:2'):\n", 458 | " graph = tf.Graph()\n", 459 | " with graph.as_default():\n", 460 | " a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name='a', shape=[2, 3])\n", 461 | " b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name='b', shape=[3, 2])\n", 462 | " c = tf.matmul(a, b)\n", 463 | " print(graph.get_operations())\n", 464 | " print(graph.version)" 465 | ] 466 | }, 467 | { 468 | "cell_type": "code", 469 | "execution_count": 23, 470 | "metadata": { 471 | "collapsed": false 472 | }, 473 | "outputs": [ 474 | { 475 | "name": "stdout", 476 | "output_type": "stream", 477 | "text": [ 478 | "[[ 22. 28.]\n", 479 | " [ 49. 64.]]\n" 480 | ] 481 | } 482 | ], 483 | "source": [ 484 | "with tf.Session(graph=graph, config=tf.ConfigProto(log_device_placement=True)) as sess:\n", 485 | " c = sess.run(c)\n", 486 | " print(c)" 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 24, 492 | "metadata": { 493 | "collapsed": false 494 | }, 495 | "outputs": [ 496 | { 497 | "name": "stdout", 498 | "output_type": "stream", 499 | "text": [ 500 | "[[ 44. 56.]\n", 501 | " [ 98. 128.]]\n" 502 | ] 503 | } 504 | ], 505 | "source": [ 506 | "c = []\n", 507 | "for d in ['/gpu:2', '/gpu:3']:\n", 508 | " with tf.device(d):\n", 509 | " a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3])\n", 510 | " b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2])\n", 511 | " c.append(tf.matmul(a, b))\n", 512 | "with tf.device('/cpu:0'):\n", 513 | " sum = tf.add_n(c)\n", 514 | "# Creates a session with log_device_placement set to True.\n", 515 | "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n", 516 | "# Runs the op.\n", 517 | "print sess.run(sum)" 518 | ] 519 | }, 520 | { 521 | "cell_type": "markdown", 522 | "metadata": {}, 523 | "source": [ 524 | "## More about graph" 525 | ] 526 | }, 527 | { 528 | "cell_type": "code", 529 | "execution_count": 14, 530 | "metadata": { 531 | "collapsed": false 532 | }, 533 | "outputs": [ 534 | { 535 | "name": "stdout", 536 | "output_type": "stream", 537 | "text": [ 538 | "[, , ]\n", 539 | "3\n", 540 | "8\n" 541 | ] 542 | } 543 | ], 544 | "source": [ 545 | "g = tf.Graph()\n", 546 | "with g.as_default():\n", 547 | " a = tf.constant(3)\n", 548 | " b = tf.constant(5)\n", 549 | " x = tf.add(a, b)\n", 550 | "print(g.get_operations())\n", 551 | "print(g.version)\n", 552 | "\n", 553 | "with tf.Session(graph = g) as sess:\n", 554 | " x = sess.run(x)\n", 555 | " print(x)" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": 15, 561 | "metadata": { 562 | "collapsed": false 563 | }, 564 | "outputs": [ 565 | { 566 | "name": "stdout", 567 | "output_type": "stream", 568 | "text": [ 569 | "Tensor(\"Const:0\", shape=(), dtype=int32)\n", 570 | "[]\n", 571 | "1\n" 572 | ] 573 | } 574 | ], 575 | "source": [ 576 | "# Do not mix default graph and user created graphs\n", 577 | "g = tf.Graph()\n", 578 | "a = tf.constant(3) # this op will be placed into default graph\n", 579 | "print(a)\n", 580 | "with g.as_default():\n", 581 | " b = tf.constant(5)\n", 582 | "print(g.get_operations())\n", 583 | "print(g.version)\n", 584 | "# we only get one operation in g" 585 | ] 586 | }, 587 | { 588 | "cell_type": "code", 589 | "execution_count": 16, 590 | "metadata": { 591 | "collapsed": false 592 | }, 593 | "outputs": [ 594 | { 595 | "name": "stdout", 596 | "output_type": "stream", 597 | "text": [ 598 | "[]\n", 599 | "0\n", 600 | "[]\n", 601 | "0\n" 602 | ] 603 | } 604 | ], 605 | "source": [ 606 | "g1 = tf.Graph()\n", 607 | "print(g1.get_operations())\n", 608 | "print(g1.version)\n", 609 | "g2 = tf.Graph()\n", 610 | "print(g2.get_operations())\n", 611 | "print(g2.version)" 612 | ] 613 | }, 614 | { 615 | "cell_type": "code", 616 | "execution_count": 17, 617 | "metadata": { 618 | "collapsed": false 619 | }, 620 | "outputs": [ 621 | { 622 | "name": "stdout", 623 | "output_type": "stream", 624 | "text": [ 625 | "[]\n", 626 | "1\n", 627 | "[]\n", 628 | "1\n" 629 | ] 630 | } 631 | ], 632 | "source": [ 633 | "with g1.as_default():\n", 634 | " a = tf.constant(3)\n", 635 | "with g2.as_default():\n", 636 | " b = tf.constant(5)\n", 637 | "print(g1.get_operations())\n", 638 | "print(g1.version)\n", 639 | "print(g2.get_operations())\n", 640 | "print(g2.version)" 641 | ] 642 | }, 643 | { 644 | "cell_type": "markdown", 645 | "metadata": {}, 646 | "source": [ 647 | "# Part 2" 648 | ] 649 | }, 650 | { 651 | "cell_type": "markdown", 652 | "metadata": {}, 653 | "source": [ 654 | "## Defining a computation graph" 655 | ] 656 | }, 657 | { 658 | "cell_type": "code", 659 | "execution_count": 18, 660 | "metadata": { 661 | "collapsed": false 662 | }, 663 | "outputs": [ 664 | { 665 | "name": "stdout", 666 | "output_type": "stream", 667 | "text": [ 668 | "[, , , , , , ]\n", 669 | "7\n" 670 | ] 671 | } 672 | ], 673 | "source": [ 674 | "graph = tf.Graph()\n", 675 | "with graph.as_default():\n", 676 | " variable = tf.Variable(42, name='foo')\n", 677 | " initialize = tf.global_variables_initializer()\n", 678 | " assign = variable.assign(13)\n", 679 | "print(graph.get_operations())\n", 680 | "print(graph.version)" 681 | ] 682 | }, 683 | { 684 | "cell_type": "markdown", 685 | "metadata": {}, 686 | "source": [ 687 | "### print them out to get some intution" 688 | ] 689 | }, 690 | { 691 | "cell_type": "code", 692 | "execution_count": 19, 693 | "metadata": { 694 | "collapsed": false 695 | }, 696 | "outputs": [ 697 | { 698 | "name": "stdout", 699 | "output_type": "stream", 700 | "text": [ 701 | "\n", 702 | "\n", 703 | "Tensor(\"foo/read:0\", shape=(), dtype=int32)\n", 704 | "\n", 705 | "name: \"init\"\n", 706 | "op: \"NoOp\"\n", 707 | "input: \"^foo/Assign\"\n", 708 | "\n", 709 | "\n", 710 | "Tensor(\"Assign:0\", shape=(), dtype=int32_ref)\n" 711 | ] 712 | } 713 | ], 714 | "source": [ 715 | "print(graph)\n", 716 | "print \n", 717 | "print(variable)\n", 718 | "print\n", 719 | "print(initialize)\n", 720 | "print\n", 721 | "print(assign)" 722 | ] 723 | }, 724 | { 725 | "cell_type": "markdown", 726 | "metadata": {}, 727 | "source": [ 728 | "## Running defined graph in a Session" 729 | ] 730 | }, 731 | { 732 | "cell_type": "code", 733 | "execution_count": 20, 734 | "metadata": { 735 | "collapsed": false 736 | }, 737 | "outputs": [ 738 | { 739 | "name": "stdout", 740 | "output_type": "stream", 741 | "text": [ 742 | "13\n" 743 | ] 744 | } 745 | ], 746 | "source": [ 747 | "with tf.Session(graph = graph) as sess:\n", 748 | " sess.run(initialize)\n", 749 | " sess.run(assign)\n", 750 | " print(sess.run(variable))" 751 | ] 752 | }, 753 | { 754 | "cell_type": "code", 755 | "execution_count": null, 756 | "metadata": { 757 | "collapsed": false 758 | }, 759 | "outputs": [], 760 | "source": [ 761 | "with tf.Session(graph = graph) as sess:\n", 762 | " print(sess.run(variable))\n", 763 | " \n", 764 | "# FailedPreconditionError: Attempting to use uninitialized value foo" 765 | ] 766 | }, 767 | { 768 | "cell_type": "code", 769 | "execution_count": 21, 770 | "metadata": { 771 | "collapsed": false 772 | }, 773 | "outputs": [ 774 | { 775 | "name": "stdout", 776 | "output_type": "stream", 777 | "text": [ 778 | "42\n" 779 | ] 780 | } 781 | ], 782 | "source": [ 783 | "with tf.Session(graph = graph) as sess:\n", 784 | " sess.run(initialize)\n", 785 | " print(sess.run(variable))" 786 | ] 787 | } 788 | ], 789 | "metadata": { 790 | "kernelspec": { 791 | "display_name": "Python 2", 792 | "language": "python", 793 | "name": "python2" 794 | }, 795 | "language_info": { 796 | "codemirror_mode": { 797 | "name": "ipython", 798 | "version": 2 799 | }, 800 | "file_extension": ".py", 801 | "mimetype": "text/x-python", 802 | "name": "python", 803 | "nbconvert_exporter": "python", 804 | "pygments_lexer": "ipython2", 805 | "version": "2.7.12" 806 | } 807 | }, 808 | "nbformat": 4, 809 | "nbformat_minor": 2 810 | } 811 | -------------------------------------------------------------------------------- /2_tensorflow_ops.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lecture 2: tensorflow ops\n", 8 | "\n", 9 | "**Useful links:** \n", 10 | "1. [Lecture note](http://web.stanford.edu/class/cs20si/lectures/notes_02.pdf);\n", 11 | "2. [Leture slides](http://web.stanford.edu/class/cs20si/lectures/slides_02.pdf);\n", 12 | "3. [Structuring Your TensorFlow Models](http://danijar.com/structuring-your-tensorflow-models/)\n", 13 | "4. [blog_tensorflow_scope_decorator.py](https://gist.github.com/danijar/8663d3bbfd586bffecf6a0094cd116f2#file-blog_tensorflow_scope_decorator-py-L1)" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "## Fun with TensorBoard\n", 21 | "TensorFlow = TensorFlow + TensorBoard + TensorServing\n", 22 | "\n", 23 | "TensorBoard is graph visualzation software included with any standard TensorFlow installation.\n", 24 | "\n", 25 | "When a user perform certain operations in a TensorBoard-activated TensorFlow program, these operations are exported to an event file. TensorBoard is able to convert these event files to graphs that give insight into a model's behavior." 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 1, 31 | "metadata": { 32 | "collapsed": true 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "import tensorflow as tf\n", 37 | "import numpy as np" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": { 44 | "collapsed": false 45 | }, 46 | "outputs": [], 47 | "source": [ 48 | "g = tf.Graph()\n", 49 | "with g.as_default():\n", 50 | " a = tf.constant(2)\n", 51 | " b = tf.constant(3)\n", 52 | " x = tf.add(a, b)" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": { 59 | "collapsed": false 60 | }, 61 | "outputs": [ 62 | { 63 | "name": "stdout", 64 | "output_type": "stream", 65 | "text": [ 66 | "5\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "with tf.Session(graph=g) as sess:\n", 72 | " x = sess.run(x)\n", 73 | " print(x)" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 4, 79 | "metadata": { 80 | "collapsed": false 81 | }, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "\n" 88 | ] 89 | } 90 | ], 91 | "source": [ 92 | "print(sess)\n", 93 | "writer = tf.summary.FileWriter('./graphs_test1', sess.graph)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 5, 99 | "metadata": { 100 | "collapsed": true 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "g = tf.Graph()\n", 105 | "with g.as_default():\n", 106 | " a = tf.constant(2)\n", 107 | " b = tf.constant(3)\n", 108 | " x = tf.add(a, b)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 6, 114 | "metadata": { 115 | "collapsed": false 116 | }, 117 | "outputs": [ 118 | { 119 | "name": "stdout", 120 | "output_type": "stream", 121 | "text": [ 122 | "5\n" 123 | ] 124 | } 125 | ], 126 | "source": [ 127 | "with tf.Session(graph=g) as sess:\n", 128 | " writer = tf.summary.FileWriter('./graphs_test2', sess.graph)\n", 129 | " x = sess.run(x)\n", 130 | " print(x)\n", 131 | "writer.close()" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "metadata": {}, 137 | "source": [ 138 | "### Display logs by tensorboard" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 7, 144 | "metadata": { 145 | "collapsed": true 146 | }, 147 | "outputs": [], 148 | "source": [ 149 | "# $ tensorboard --logdir=\"./graphs_test1\"\n", 150 | "# $ tensorboard --logdir=\"./graphs_test2\" -port 6007 # to avoid port is in use" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "Standford note suggest we to add the summary line right before the running loop, however,\n", 158 | "I try to summary afer session being closed. Both of them get the same log......I didn't get the purpose of this suggestion yet." 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "### Name the ops" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 8, 171 | "metadata": { 172 | "collapsed": true 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "g = tf.Graph()\n", 177 | "with g.as_default():\n", 178 | " a = tf.constant([2, 2], name='a')\n", 179 | " b = tf.constant([3, 6], name='b')\n", 180 | " x = tf.add(a, b, name='add')" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 9, 186 | "metadata": { 187 | "collapsed": false 188 | }, 189 | "outputs": [ 190 | { 191 | "name": "stdout", 192 | "output_type": "stream", 193 | "text": [ 194 | "[5 8]\n" 195 | ] 196 | } 197 | ], 198 | "source": [ 199 | "# tf.Session.run(fetches, feed_dict=None, options=None, run_metadata=None)\n", 200 | "with tf.Session(graph=g) as sess:\n", 201 | " writer = tf.summary.FileWriter('./graphs', sess.graph)\n", 202 | " x = sess.run(x)\n", 203 | " print(x)\n", 204 | "writer.close()\n", 205 | "# display your log again to see the name you had explicitly assigned." 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "### Note:\n", 213 | "If you've run your code several times, there will be muliple event files, TF will show only the lastest graph and display the warning of multiple event files. To get rid of the warning, delete all the event files you no longer need." 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "## Constant types" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": {}, 226 | "source": [ 227 | "tf.convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None)\n", 228 | "\n", 229 | "All standard Python op constructors apply this function to each of their Tensor-valued inputs, which allows those ops to accept numpy arrays, Python lists, and scalars in addition to Tensor objects.\n", 230 | "https://www.tensorflow.org/api_docs/python/tf/\n", 231 | "https://www.tensorflow.org/api_docs/python/tf/convert_to_tensor" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 10, 237 | "metadata": { 238 | "collapsed": false 239 | }, 240 | "outputs": [ 241 | { 242 | "name": "stdout", 243 | "output_type": "stream", 244 | "text": [ 245 | "Tensor(\"vector:0\", shape=(2,), dtype=int32)\n", 246 | "Tensor(\"b:0\", shape=(2, 2), dtype=int32)\n" 247 | ] 248 | } 249 | ], 250 | "source": [ 251 | "# tf.constant(value, dtype=None, shape=None, name='Const', verify_shape=False)\n", 252 | "# constant of 1d tensor (vector)\n", 253 | "a = tf.constant([2, 2], name='vector')\n", 254 | "print(a)\n", 255 | "\n", 256 | "# constant of 2*2 tensor (matrix)\n", 257 | "b = tf.constant([[0, 1], [2, 3]], name='b')\n", 258 | "print(b)" 259 | ] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": {}, 264 | "source": [ 265 | "### You can creat tensors whose elements are of a specific value\n", 266 | "Note the similarity to numpy.zeros, numpy.zeros_like, numpy.ones, numpy.ones_like" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": 11, 272 | "metadata": { 273 | "collapsed": false 274 | }, 275 | "outputs": [ 276 | { 277 | "name": "stdout", 278 | "output_type": "stream", 279 | "text": [ 280 | "Tensor(\"zeros:0\", shape=(2, 3), dtype=int32)\n", 281 | "[[0 0 0]\n", 282 | " [0 0 0]]\n" 283 | ] 284 | } 285 | ], 286 | "source": [ 287 | "# tf.zeros(shape, dtype=tf.float32, name=Nome)\n", 288 | "# shape: Either a list of integers, or a 1-D Tensor of type int32\n", 289 | "# create a tensor of shape and all elements are zeros\n", 290 | "a = tf.zeros([2, 3], tf.int32)\n", 291 | "print(a)\n", 292 | "with tf.Session() as sess:\n", 293 | " a = sess.run(a)\n", 294 | " print(a)" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 12, 300 | "metadata": { 301 | "collapsed": false 302 | }, 303 | "outputs": [ 304 | { 305 | "name": "stdout", 306 | "output_type": "stream", 307 | "text": [ 308 | "Tensor(\"zeros_like:0\", shape=(3, 2), dtype=int32)\n", 309 | "[[0 0]\n", 310 | " [0 0]\n", 311 | " [0 0]]\n" 312 | ] 313 | } 314 | ], 315 | "source": [ 316 | "# tf.zeros_like(tensor, dtype=None, name=None, optimize=True)\n", 317 | "# Creates a tensor of shape and type(unless type is specified) as the input_tensor but\n", 318 | "# all elements are zeros\n", 319 | "a = tf.constant([[0, 1],\n", 320 | " [2, 3],\n", 321 | " [4, 5]])\n", 322 | "b = tf.zeros_like(a)\n", 323 | "print(b)\n", 324 | "with tf.Session() as sess:\n", 325 | " b = sess.run(b)\n", 326 | " print(b)" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 13, 332 | "metadata": { 333 | "collapsed": false 334 | }, 335 | "outputs": [ 336 | { 337 | "name": "stdout", 338 | "output_type": "stream", 339 | "text": [ 340 | "Tensor(\"ones:0\", shape=(2, 3), dtype=int32)\n", 341 | "[[1 1 1]\n", 342 | " [1 1 1]]\n" 343 | ] 344 | } 345 | ], 346 | "source": [ 347 | "# tf.ones(shape, dtype=tf.float32, name=None)\n", 348 | "# create a tensor of shape and all elements are ones\n", 349 | "a = tf.ones([2, 3], tf.int32)\n", 350 | "print(a)\n", 351 | "with tf.Session() as sess:\n", 352 | " a = sess.run(a)\n", 353 | " print(a)" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": 14, 359 | "metadata": { 360 | "collapsed": false 361 | }, 362 | "outputs": [ 363 | { 364 | "name": "stdout", 365 | "output_type": "stream", 366 | "text": [ 367 | "Tensor(\"ones_like:0\", shape=(3, 2), dtype=int32)\n", 368 | "[[1 1]\n", 369 | " [1 1]\n", 370 | " [1 1]]\n" 371 | ] 372 | } 373 | ], 374 | "source": [ 375 | "# tf.ones_like(tensor, dtype=None, name=None, optimize=True)\n", 376 | "# create a tensor of shape and type (unless type is specified) as the input_tensor but all\n", 377 | "# elements are ones\n", 378 | "a = tf.constant([[0, 1],\n", 379 | " [2, 3],\n", 380 | " [4, 5]])\n", 381 | "b = tf.ones_like(a)\n", 382 | "print(b)\n", 383 | "with tf.Session() as sess:\n", 384 | " b = sess.run(b)\n", 385 | " print(b)" 386 | ] 387 | }, 388 | { 389 | "cell_type": "code", 390 | "execution_count": 15, 391 | "metadata": { 392 | "collapsed": false 393 | }, 394 | "outputs": [ 395 | { 396 | "name": "stdout", 397 | "output_type": "stream", 398 | "text": [ 399 | "Tensor(\"Fill:0\", shape=(2, 3), dtype=int32)\n", 400 | "[[8 8 8]\n", 401 | " [8 8 8]]\n" 402 | ] 403 | } 404 | ], 405 | "source": [ 406 | "# tf.fill(dims, value, name=None)\n", 407 | "# create a tensor filled with a scalar value\n", 408 | "a = tf.fill([2, 3], 8)\n", 409 | "print(a)\n", 410 | "with tf.Session() as sess:\n", 411 | " a = sess.run(a)\n", 412 | " print(a)" 413 | ] 414 | }, 415 | { 416 | "cell_type": "markdown", 417 | "metadata": {}, 418 | "source": [ 419 | "### You can create constants that are sequences" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": 16, 425 | "metadata": { 426 | "collapsed": false 427 | }, 428 | "outputs": [ 429 | { 430 | "name": "stdout", 431 | "output_type": "stream", 432 | "text": [ 433 | "Tensor(\"linspace:0\", shape=(4,), dtype=float32)\n", 434 | "[ 10. 11. 12. 13.]\n" 435 | ] 436 | } 437 | ], 438 | "source": [ 439 | "# tf.linspace(start, stop, num, name=None)\n", 440 | "# create a sequence of num evenly-spaced values are generated beginning at start. If num>1\n", 441 | "# the values in the sequence increase by stop - start / num -1, so that the last one is\n", 442 | "# exactly stop\n", 443 | "# start, stop, num must be scalars\n", 444 | "# start, stop Must be one of the following types: float32, float64, num: int32, int64\n", 445 | "# comparable to but sligtly different from numpy.linspace\n", 446 | "# numpy.linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None)\n", 447 | "\n", 448 | "a = tf.linspace(10.0, 13.0, 4, name='linspace')\n", 449 | "print(a)\n", 450 | "with tf.Session() as sess:\n", 451 | " a = sess.run(a)\n", 452 | " print(a)" 453 | ] 454 | }, 455 | { 456 | "cell_type": "code", 457 | "execution_count": 17, 458 | "metadata": { 459 | "collapsed": false 460 | }, 461 | "outputs": [ 462 | { 463 | "name": "stdout", 464 | "output_type": "stream", 465 | "text": [ 466 | "Tensor(\"range:0\", shape=(5,), dtype=int32)\n", 467 | "[ 3 6 9 12 15]\n" 468 | ] 469 | } 470 | ], 471 | "source": [ 472 | "# tf.range(start, limit=None, delta=1, dtype=None, name='range')\n", 473 | "# Creates a sequence of numbers that begins at start and extends by increments of delta\n", 474 | "# up to but not including limit\n", 475 | "# Like the Python builtin range, start defaults to 0, so that range(n) = range(0, n)\n", 476 | "a = tf.range(3, 18, 3)\n", 477 | "print(a)\n", 478 | "with tf.Session() as sess:\n", 479 | " a = sess.run(a)\n", 480 | " print(a)" 481 | ] 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": 18, 486 | "metadata": { 487 | "collapsed": false 488 | }, 489 | "outputs": [ 490 | { 491 | "name": "stdout", 492 | "output_type": "stream", 493 | "text": [ 494 | "Tensor(\"range_1:0\", shape=(4,), dtype=float32)\n", 495 | "[ 3. 2.5 2. 1.5]\n" 496 | ] 497 | } 498 | ], 499 | "source": [ 500 | "# a = tf.range(3, 1, -0.5) maybe for dtype reason this line may raise error\n", 501 | "a = tf.range(3.0, 1.0, -0.5)\n", 502 | "print(a)\n", 503 | "with tf.Session() as sess:\n", 504 | " a = sess.run(a)\n", 505 | " print(a)" 506 | ] 507 | }, 508 | { 509 | "cell_type": "code", 510 | "execution_count": 19, 511 | "metadata": { 512 | "collapsed": false 513 | }, 514 | "outputs": [ 515 | { 516 | "name": "stdout", 517 | "output_type": "stream", 518 | "text": [ 519 | "Tensor(\"range_2:0\", shape=(5,), dtype=int32)\n", 520 | "[0 1 2 3 4]\n" 521 | ] 522 | } 523 | ], 524 | "source": [ 525 | "a = tf.range(5)\n", 526 | "print(a)\n", 527 | "with tf.Session() as sess:\n", 528 | " a = sess.run(a)\n", 529 | " print(a)" 530 | ] 531 | }, 532 | { 533 | "cell_type": "markdown", 534 | "metadata": {}, 535 | "source": [ 536 | "#### Note that unlike NumPy or Python sequences, TensorFlow sequences are not iterable" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": 20, 542 | "metadata": { 543 | "collapsed": false 544 | }, 545 | "outputs": [ 546 | { 547 | "name": "stdout", 548 | "output_type": "stream", 549 | "text": [ 550 | "0.0\n", 551 | "3.33333333333\n", 552 | "6.66666666667\n", 553 | "10.0\n" 554 | ] 555 | } 556 | ], 557 | "source": [ 558 | "for i in np.linspace(0, 10, 4):\n", 559 | " print(i)" 560 | ] 561 | }, 562 | { 563 | "cell_type": "code", 564 | "execution_count": null, 565 | "metadata": { 566 | "collapsed": false 567 | }, 568 | "outputs": [], 569 | "source": [ 570 | "# tf.linspace(0, 10, 4)\n", 571 | "# TypeError: Value passed to parameter 'start' has DataType int32 not in list of allowed \n", 572 | "# values: float32, float64\n", 573 | "for i in tf.linspace(0.0, 10.0, 4):\n", 574 | " print(i)\n", 575 | "# TypeError: 'Tensor' object is not iterable" 576 | ] 577 | }, 578 | { 579 | "cell_type": "code", 580 | "execution_count": 22, 581 | "metadata": { 582 | "collapsed": false 583 | }, 584 | "outputs": [ 585 | { 586 | "name": "stdout", 587 | "output_type": "stream", 588 | "text": [ 589 | "0\n", 590 | "1\n", 591 | "2\n", 592 | "3\n" 593 | ] 594 | } 595 | ], 596 | "source": [ 597 | "for i in range(4):\n", 598 | " print(i)" 599 | ] 600 | }, 601 | { 602 | "cell_type": "code", 603 | "execution_count": null, 604 | "metadata": { 605 | "collapsed": false 606 | }, 607 | "outputs": [], 608 | "source": [ 609 | "for i in tf.range(4):\n", 610 | " print(i)\n", 611 | "# TypeError: 'Tensor' object is not iterable." 612 | ] 613 | }, 614 | { 615 | "cell_type": "markdown", 616 | "metadata": {}, 617 | "source": [ 618 | "### You can also generate random constants from certain distributions" 619 | ] 620 | }, 621 | { 622 | "cell_type": "code", 623 | "execution_count": null, 624 | "metadata": { 625 | "collapsed": true 626 | }, 627 | "outputs": [], 628 | "source": [ 629 | "# tf.random_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None, name=None)\n", 630 | "# tf.truncated_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None, name=None)\n", 631 | "# tf.random_uniform(shape, minval=0, maxval=None, dtype=tf.float32, seed=None, name=None)\n", 632 | "# tf.random_shuffle(value, seed=None, name=None)\n", 633 | "# tf.random_crop(value, size, seed=None, name=None)\n", 634 | "# tf.multinomial(logits, num_samples, seed=None, name=None)\n", 635 | "# tf.random_gamma(shape, alpha, beta=None, dtype=tf.float32, seed=None, name=None)\n", 636 | "# tf.set_random_seed(seed) # Operations that rely on a random seed actually derive \n", 637 | "#it from two seeds: the graph-level and operation-level seeds. This sets the \n", 638 | "# graph-level seed." 639 | ] 640 | }, 641 | { 642 | "cell_type": "markdown", 643 | "metadata": {}, 644 | "source": [ 645 | "https://www.tensorflow.org/api_docs/python/tf/set_random_seed" 646 | ] 647 | }, 648 | { 649 | "cell_type": "code", 650 | "execution_count": 23, 651 | "metadata": { 652 | "collapsed": false 653 | }, 654 | "outputs": [ 655 | { 656 | "name": "stdout", 657 | "output_type": "stream", 658 | "text": [ 659 | "Session 1\n", 660 | "a1\n", 661 | "[[ 3.10000658 7.7726078 4.36233521]\n", 662 | " [ 4.33057308 0.70752025 5.44950485]]\n", 663 | "a2\n", 664 | "[[ 4.27698374 2.59973884 8.42744923]\n", 665 | " [ 2.16902852 1.45320177 5.87610722]]\n", 666 | "b1\n", 667 | "[[-0.20380068 1.02053642 0.83243173]\n", 668 | " [ 0.12789387 -3.15761328 6.83167458]]\n", 669 | "b2\n", 670 | "[[-3.93487716 5.25653267 2.17159677]\n", 671 | " [ 0.12227363 8.25422764 -7.55906105]]\n", 672 | "Session 2\n", 673 | "a1\n", 674 | "[[ 3.10000658 7.7726078 4.36233521]\n", 675 | " [ 4.33057308 0.70752025 5.44950485]]\n", 676 | "a2\n", 677 | "[[ 4.27698374 2.59973884 8.42744923]\n", 678 | " [ 2.16902852 1.45320177 5.87610722]]\n", 679 | "b1\n", 680 | "[[-0.20380068 1.02053642 0.83243173]\n", 681 | " [ 0.12789387 -3.15761328 6.83167458]]\n", 682 | "b2\n", 683 | "[[-3.93487716 5.25653267 2.17159677]\n", 684 | " [ 0.12227363 8.25422764 -7.55906105]]\n" 685 | ] 686 | } 687 | ], 688 | "source": [ 689 | "g = tf.Graph()\n", 690 | "with g.as_default():\n", 691 | " tf.set_random_seed(123)\n", 692 | " a = tf.random_uniform([2, 3], minval=0.0, maxval=10.0)\n", 693 | " b = tf.random_normal([2, 3], mean=0.0, stddev=4.0)\n", 694 | "\n", 695 | "print(\"Session 1\")\n", 696 | "with tf.Session(graph=g) as sess1:\n", 697 | " print('a1')\n", 698 | " print(sess1.run(a))\n", 699 | " print('a2')\n", 700 | " print(sess1.run(a))\n", 701 | " print('b1')\n", 702 | " print(sess1.run(b))\n", 703 | " print('b2')\n", 704 | " print(sess1.run(b))\n", 705 | "\n", 706 | "print(\"Session 2\")\n", 707 | "with tf.Session(graph=g) as sess2:\n", 708 | " print('a1')\n", 709 | " print(sess2.run(a))\n", 710 | " print('a2')\n", 711 | " print(sess2.run(a))\n", 712 | " print('b1')\n", 713 | " print(sess2.run(b))\n", 714 | " print('b2')\n", 715 | " print(sess2.run(b))" 716 | ] 717 | }, 718 | { 719 | "cell_type": "code", 720 | "execution_count": 24, 721 | "metadata": { 722 | "collapsed": false 723 | }, 724 | "outputs": [ 725 | { 726 | "name": "stdout", 727 | "output_type": "stream", 728 | "text": [ 729 | "[[ 6.77753503 5.03207802 5.6798031 ]\n", 730 | " [ 3.92553631 2.4071212 9.85233095]]\n", 731 | "[[ 1.28136131 -0.3047094 -0.75538523]\n", 732 | " [-1.5691671 -1.22085975 0.65354131]]\n", 733 | "[[ 6.77753503 5.03207802 5.6798031 ]\n", 734 | " [ 3.92553631 2.4071212 9.85233095]]\n", 735 | "[[ 1.28136131 -0.3047094 -0.75538523]\n", 736 | " [-1.5691671 -1.22085975 0.65354131]]\n" 737 | ] 738 | } 739 | ], 740 | "source": [ 741 | "a = tf.constant(np.random.uniform(low=0.0, high=10.0, size=(2, 3)))\n", 742 | "b = tf.constant(np.random.normal(loc=0.0, scale=1.0, size=(2,3)))\n", 743 | "with tf.Session() as sess:\n", 744 | " a1, b1 = sess.run([a, b])\n", 745 | " a2, b2 = sess.run([a, b])\n", 746 | " print(a1)\n", 747 | " print(b1)\n", 748 | " print(a2)\n", 749 | " print(b2)" 750 | ] 751 | }, 752 | { 753 | "cell_type": "markdown", 754 | "metadata": {}, 755 | "source": [ 756 | "### Math Operations\n", 757 | "https://www.tensorflow.org/api_guides/python/math_ops" 758 | ] 759 | }, 760 | { 761 | "cell_type": "code", 762 | "execution_count": 25, 763 | "metadata": { 764 | "collapsed": false 765 | }, 766 | "outputs": [ 767 | { 768 | "name": "stdout", 769 | "output_type": "stream", 770 | "text": [ 771 | "[5 8]\n", 772 | "[ 7 10]\n", 773 | "[ 6 12]\n", 774 | "[[18]]\n", 775 | "[1 3]\n", 776 | "[1 0]\n" 777 | ] 778 | } 779 | ], 780 | "source": [ 781 | "a = tf.constant([3, 6])\n", 782 | "b = tf.constant([2, 2])\n", 783 | "c_0 = tf.add(a, b)\n", 784 | "c_1 = tf.add_n([a, b, b])\n", 785 | "c_2 = tf.multiply(a, b) # element wise\n", 786 | "c_3 = tf.matmul(tf.reshape(a, [1, 2]), tf.reshape(b, [2, 1]))\n", 787 | "c_4 = tf.div(a, b)\n", 788 | "c_5 = tf.mod(a, b)\n", 789 | "with tf.Session() as sess:\n", 790 | " c_0, c_1, c_2, c_3, c_4, c_5 = sess.run([c_0, c_1, c_2, c_3, c_4, c_5])\n", 791 | " print(c_0)\n", 792 | " print(c_1)\n", 793 | " print(c_2)\n", 794 | " print(c_3)\n", 795 | " print(c_4)\n", 796 | " print(c_5)" 797 | ] 798 | }, 799 | { 800 | "cell_type": "markdown", 801 | "metadata": {}, 802 | "source": [ 803 | "### Data Types" 804 | ] 805 | }, 806 | { 807 | "cell_type": "code", 808 | "execution_count": 26, 809 | "metadata": { 810 | "collapsed": false 811 | }, 812 | "outputs": [ 813 | { 814 | "name": "stdout", 815 | "output_type": "stream", 816 | "text": [ 817 | "0\n", 818 | "1\n", 819 | "['' '' '']\n", 820 | "[[False False False]\n", 821 | " [False False False]\n", 822 | " [False False False]]\n", 823 | "[[ True True True]\n", 824 | " [ True True True]\n", 825 | " [ True True True]]\n" 826 | ] 827 | } 828 | ], 829 | "source": [ 830 | "a_0 = 19\n", 831 | "a_1 = tf.zeros_like(a_0)\n", 832 | "a_2 = tf.ones_like(a_0)\n", 833 | "b_0 = [b\"apple\", b\"peach\", b\"grape\"]\n", 834 | "b_1 = tf.zeros_like(b_0)\n", 835 | "#b_2 = tf.ones_like(b_0) # TypeError: Expected string, got 1 of type 'int' instead\n", 836 | "\n", 837 | "c_0 = [[True, False, False],\n", 838 | " [False, False, True],\n", 839 | " [False, True, False]] # treated as a 2-d tensor, or \"matrix\"\n", 840 | "c_1 = tf.zeros_like(c_0)\n", 841 | "c_2 = tf.ones_like(c_0)\n", 842 | "with tf.Session() as sess:\n", 843 | " a_1, a_2, b_1, c_1, c_2 = sess.run([a_1, a_2, b_1, c_1, c_2])\n", 844 | " print(a_1)\n", 845 | " print(a_2)\n", 846 | " print(b_1)\n", 847 | " #print(b_2)\n", 848 | " print(c_1)\n", 849 | " print(c_2)" 850 | ] 851 | }, 852 | { 853 | "cell_type": "markdown", 854 | "metadata": {}, 855 | "source": [ 856 | "### Variables\n", 857 | "Constants have been fun, but I think now you guys are old enough to how about variables.\n", 858 | "The difference between a constant and variable:\n", 859 | "1. A constant is constant. A variable can be assigned to, its value can be changed.\n", 860 | "2. A constant's value is stored in the graph and its value is replicated wherever the graph is loads. A variable is stored separately, and may live on a parameter server." 861 | ] 862 | }, 863 | { 864 | "cell_type": "code", 865 | "execution_count": 27, 866 | "metadata": { 867 | "collapsed": false 868 | }, 869 | "outputs": [ 870 | { 871 | "name": "stdout", 872 | "output_type": "stream", 873 | "text": [ 874 | "node {\n", 875 | " name: \"my_const\"\n", 876 | " op: \"Const\"\n", 877 | " attr {\n", 878 | " key: \"dtype\"\n", 879 | " value {\n", 880 | " type: DT_FLOAT\n", 881 | " }\n", 882 | " }\n", 883 | " attr {\n", 884 | " key: \"value\"\n", 885 | " value {\n", 886 | " tensor {\n", 887 | " dtype: DT_FLOAT\n", 888 | " tensor_shape {\n", 889 | " dim {\n", 890 | " size: 2\n", 891 | " }\n", 892 | " }\n", 893 | " tensor_content: \"\\000\\000\\200?\\000\\000\\000@\"\n", 894 | " }\n", 895 | " }\n", 896 | " }\n", 897 | "}\n", 898 | "versions {\n", 899 | " producer: 21\n", 900 | "}\n", 901 | "\n" 902 | ] 903 | } 904 | ], 905 | "source": [ 906 | "g = tf.Graph()\n", 907 | "with g.as_default():\n", 908 | " my_const = tf.constant([1.0, 2.0], name='my_const')\n", 909 | " print(tf.get_default_graph().as_graph_def())" 910 | ] 911 | }, 912 | { 913 | "cell_type": "markdown", 914 | "metadata": {}, 915 | "source": [ 916 | "#### Declare variables" 917 | ] 918 | }, 919 | { 920 | "cell_type": "code", 921 | "execution_count": 28, 922 | "metadata": { 923 | "collapsed": false 924 | }, 925 | "outputs": [ 926 | { 927 | "name": "stdout", 928 | "output_type": "stream", 929 | "text": [ 930 | "Tensor(\"scalar/read:0\", shape=(), dtype=int32)\n", 931 | "Tensor(\"vector_1/read:0\", shape=(2,), dtype=int32)\n", 932 | "Tensor(\"matrix/read:0\", shape=(2, 2), dtype=int32)\n", 933 | "Tensor(\"Variable/read:0\", shape=(784, 10), dtype=float32)\n" 934 | ] 935 | } 936 | ], 937 | "source": [ 938 | "# create variable a with scalar value\n", 939 | "a = tf.Variable(2, name='scalar')\n", 940 | "print(a)\n", 941 | "# create variable b as a vector\n", 942 | "b = tf.Variable([2, 3], name='vector')\n", 943 | "print(b)\n", 944 | "# create variable c as a 2*2 matrix\n", 945 | "c = tf.Variable([[0, 1], [2, 3]], name='matrix')\n", 946 | "print(c)\n", 947 | "# create variable W as 784*10 tensor, filled with zeros\n", 948 | "W = tf.Variable(tf.zeros([784, 10]))\n", 949 | "print(W)" 950 | ] 951 | }, 952 | { 953 | "cell_type": "code", 954 | "execution_count": 29, 955 | "metadata": { 956 | "collapsed": false 957 | }, 958 | "outputs": [ 959 | { 960 | "name": "stdout", 961 | "output_type": "stream", 962 | "text": [ 963 | "[, , , ]\n", 964 | "4\n", 965 | "5\n" 966 | ] 967 | } 968 | ], 969 | "source": [ 970 | "g = tf.Graph()\n", 971 | "with g.as_default():\n", 972 | " x = tf.Variable(2, name='scalar')\n", 973 | "print(g.get_operations())\n", 974 | "print(g.version)\n", 975 | "\n", 976 | "with tf.Session(graph=g) as sess:\n", 977 | " sess.run(x.initializer)\n", 978 | " print(sess.run(x.assign_add(3)))" 979 | ] 980 | }, 981 | { 982 | "cell_type": "code", 983 | "execution_count": 30, 984 | "metadata": { 985 | "collapsed": false 986 | }, 987 | "outputs": [], 988 | "source": [ 989 | "# You have to initialize variables before using them\n", 990 | "init = tf.global_variables_initializer()\n", 991 | "\n", 992 | "with tf.Session() as sess:\n", 993 | " sess.run(init)" 994 | ] 995 | }, 996 | { 997 | "cell_type": "code", 998 | "execution_count": 31, 999 | "metadata": { 1000 | "collapsed": false 1001 | }, 1002 | "outputs": [], 1003 | "source": [ 1004 | "init_ab = tf.variables_initializer([a, b], name='init_ab')\n", 1005 | "with tf.Session() as sess:\n", 1006 | " sess.run(init_ab)" 1007 | ] 1008 | }, 1009 | { 1010 | "cell_type": "code", 1011 | "execution_count": 32, 1012 | "metadata": { 1013 | "collapsed": false 1014 | }, 1015 | "outputs": [], 1016 | "source": [ 1017 | "# create variable W as 784*10 tensor, filled with zeros\n", 1018 | "W = tf.Variable(tf.zeros([784, 10]))\n", 1019 | "with tf.Session() as sess:\n", 1020 | " sess.run(W.initializer)\n", 1021 | "# another way to initialize a variable is to restore it from a save file" 1022 | ] 1023 | }, 1024 | { 1025 | "cell_type": "markdown", 1026 | "metadata": {}, 1027 | "source": [ 1028 | "#### Evaluate values of variables" 1029 | ] 1030 | }, 1031 | { 1032 | "cell_type": "code", 1033 | "execution_count": 33, 1034 | "metadata": { 1035 | "collapsed": false 1036 | }, 1037 | "outputs": [ 1038 | { 1039 | "name": "stdout", 1040 | "output_type": "stream", 1041 | "text": [ 1042 | "Tensor(\"Variable_2/read:0\", shape=(700, 10), dtype=float32)\n" 1043 | ] 1044 | } 1045 | ], 1046 | "source": [ 1047 | "# W is a random 700*100 variable object\n", 1048 | "W = tf.Variable(tf.truncated_normal([700, 10]))\n", 1049 | "with tf.Session() as sess:\n", 1050 | " sess.run(W.initializer)\n", 1051 | " print(W)" 1052 | ] 1053 | }, 1054 | { 1055 | "cell_type": "code", 1056 | "execution_count": 34, 1057 | "metadata": { 1058 | "collapsed": false 1059 | }, 1060 | "outputs": [ 1061 | { 1062 | "name": "stdout", 1063 | "output_type": "stream", 1064 | "text": [ 1065 | "[[-0.87107468 0.26141784 1.52498627 ..., 0.90238583 0.79691088\n", 1066 | " -0.06048778]\n", 1067 | " [ 0.68131208 -0.42437628 -0.04228624 ..., -1.3219763 0.42711928\n", 1068 | " -1.3788898 ]\n", 1069 | " [-1.32859874 -0.52924287 0.03264492 ..., 0.25225312 -0.655994\n", 1070 | " -1.28667557]\n", 1071 | " ..., \n", 1072 | " [ 0.68829215 1.28704309 0.66447854 ..., 1.02221203 -0.43233114\n", 1073 | " 1.59204376]\n", 1074 | " [ 1.27488673 -1.16340578 0.05163746 ..., -1.20619512 0.31666392\n", 1075 | " 1.4724524 ]\n", 1076 | " [-0.47545099 -1.50535309 -1.44748247 ..., 0.2528604 -1.66840899\n", 1077 | " 0.74078834]]\n" 1078 | ] 1079 | } 1080 | ], 1081 | "source": [ 1082 | "# to get the value of a variable, we need to evaluate it using eval()\n", 1083 | "W = tf.Variable(tf.truncated_normal([700, 10]))\n", 1084 | "with tf.Session() as sess:\n", 1085 | " sess.run(W.initializer)\n", 1086 | " print(W.eval())" 1087 | ] 1088 | }, 1089 | { 1090 | "cell_type": "markdown", 1091 | "metadata": {}, 1092 | "source": [ 1093 | "#### Assign values to variables\n", 1094 | "we can assign a value to a variable using tf.Variable.assign()" 1095 | ] 1096 | }, 1097 | { 1098 | "cell_type": "code", 1099 | "execution_count": 35, 1100 | "metadata": { 1101 | "collapsed": false 1102 | }, 1103 | "outputs": [ 1104 | { 1105 | "name": "stdout", 1106 | "output_type": "stream", 1107 | "text": [ 1108 | "10\n" 1109 | ] 1110 | } 1111 | ], 1112 | "source": [ 1113 | "W = tf.Variable(10)\n", 1114 | "W.assign(100)\n", 1115 | "with tf.Session() as sess:\n", 1116 | " sess.run(W.initializer)\n", 1117 | " print(W.eval())" 1118 | ] 1119 | }, 1120 | { 1121 | "cell_type": "code", 1122 | "execution_count": 36, 1123 | "metadata": { 1124 | "collapsed": false 1125 | }, 1126 | "outputs": [ 1127 | { 1128 | "name": "stdout", 1129 | "output_type": "stream", 1130 | "text": [ 1131 | "100\n" 1132 | ] 1133 | } 1134 | ], 1135 | "source": [ 1136 | "W = tf.Variable(10)\n", 1137 | "assign_op = W.assign(100)\n", 1138 | "with tf.Session() as sess:\n", 1139 | " sess.run(assign_op)\n", 1140 | " print(W.eval())\n", 1141 | "# Note that we don't have initialize W in this case, because assign() does it for us.\n", 1142 | "# In fact, initializer op is the assign op that assigns the variable's initial value\n", 1143 | "# to the variable itself.\n", 1144 | "# in the source code\n", 1145 | "# self._initializer_op = state_ops.assign(self._variable, self._initial_value,\n", 1146 | "# validate_shape=validate_shape).op" 1147 | ] 1148 | }, 1149 | { 1150 | "cell_type": "code", 1151 | "execution_count": 37, 1152 | "metadata": { 1153 | "collapsed": false 1154 | }, 1155 | "outputs": [ 1156 | { 1157 | "name": "stdout", 1158 | "output_type": "stream", 1159 | "text": [ 1160 | "4\n", 1161 | "8\n", 1162 | "16\n" 1163 | ] 1164 | } 1165 | ], 1166 | "source": [ 1167 | "# Intersting example\n", 1168 | "# create a variable whose original value is 2\n", 1169 | "a = tf.Variable(2, name='scalar')\n", 1170 | "# assign a*2 to a and call that op a_times_two\n", 1171 | "a_times_two = a.assign(a * 2)\n", 1172 | "\n", 1173 | "init = tf.global_variables_initializer()\n", 1174 | "\n", 1175 | "with tf.Session() as sess:\n", 1176 | " sess.run(init)\n", 1177 | " # have to initialize a, because a_times_two op depends on the value of a\n", 1178 | " print(sess.run(a_times_two)) # >> 4\n", 1179 | " print(sess.run(a_times_two)) # >> 8\n", 1180 | " print(sess.run(a_times_two)) # >> 16" 1181 | ] 1182 | }, 1183 | { 1184 | "cell_type": "code", 1185 | "execution_count": 38, 1186 | "metadata": { 1187 | "collapsed": false 1188 | }, 1189 | "outputs": [ 1190 | { 1191 | "name": "stdout", 1192 | "output_type": "stream", 1193 | "text": [ 1194 | "20\n", 1195 | "18\n" 1196 | ] 1197 | } 1198 | ], 1199 | "source": [ 1200 | "W = tf.Variable(10)\n", 1201 | "\n", 1202 | "with tf.Session() as sess:\n", 1203 | " sess.run(W.initializer)\n", 1204 | " print(sess.run(W.assign_add(10)))\n", 1205 | " print(sess.run(W.assign_sub(2)))" 1206 | ] 1207 | }, 1208 | { 1209 | "cell_type": "code", 1210 | "execution_count": 39, 1211 | "metadata": { 1212 | "collapsed": false 1213 | }, 1214 | "outputs": [ 1215 | { 1216 | "name": "stdout", 1217 | "output_type": "stream", 1218 | "text": [ 1219 | "20\n", 1220 | "8\n", 1221 | "120\n", 1222 | "-42\n" 1223 | ] 1224 | } 1225 | ], 1226 | "source": [ 1227 | "W = tf.Variable(10)\n", 1228 | "\n", 1229 | "sess1 = tf.Session()\n", 1230 | "sess2 = tf.Session()\n", 1231 | "\n", 1232 | "sess1.run(W.initializer)\n", 1233 | "sess2.run(W.initializer)\n", 1234 | "\n", 1235 | "print(sess1.run(W.assign_add(10)))\n", 1236 | "print(sess2.run(W.assign_sub(2)))\n", 1237 | "\n", 1238 | "print(sess1.run(W.assign_add(100)))\n", 1239 | "print(sess2.run(W.assign_sub(50)))\n", 1240 | "\n", 1241 | "sess1.close()\n", 1242 | "sess2.close()" 1243 | ] 1244 | }, 1245 | { 1246 | "cell_type": "code", 1247 | "execution_count": 40, 1248 | "metadata": { 1249 | "collapsed": false 1250 | }, 1251 | "outputs": [ 1252 | { 1253 | "name": "stdout", 1254 | "output_type": "stream", 1255 | "text": [ 1256 | "[[ 0.15774222 1.6848619 0.75867641 ..., 0.70471513 0.45140892\n", 1257 | " 1.1388737 ]\n", 1258 | " [-1.90446675 0.40704605 -0.52457744 ..., 1.38665187 0.0540167\n", 1259 | " -0.13397592]\n", 1260 | " [-1.99462855 0.87634164 0.38599205 ..., -0.44720611 -1.18347108\n", 1261 | " -1.10491443]\n", 1262 | " ..., \n", 1263 | " [ 0.05733988 -0.88313168 1.40067708 ..., 0.46848473 0.86937541\n", 1264 | " 1.65061593]\n", 1265 | " [ 0.09700993 -0.68789995 1.15506804 ..., -0.14402924 0.50634795\n", 1266 | " -0.30204296]\n", 1267 | " [ 0.13493772 1.31803548 -1.11376512 ..., -0.86277169 0.96434861\n", 1268 | " 0.03455852]]\n", 1269 | "[[ 0.31548443 3.3697238 1.51735282 ..., 1.40943027 0.90281785\n", 1270 | " 2.27774739]\n", 1271 | " [-3.8089335 0.8140921 -1.04915488 ..., 2.77330375 0.1080334\n", 1272 | " -0.26795185]\n", 1273 | " [-3.9892571 1.75268328 0.7719841 ..., -0.89441222 -2.36694217\n", 1274 | " -2.20982885]\n", 1275 | " ..., \n", 1276 | " [ 0.11467977 -1.76626337 2.80135417 ..., 0.93696946 1.73875082\n", 1277 | " 3.30123186]\n", 1278 | " [ 0.19401987 -1.37579989 2.31013608 ..., -0.28805849 1.01269591\n", 1279 | " -0.60408592]\n", 1280 | " [ 0.26987544 2.63607097 -2.22753024 ..., -1.72554338 1.92869723\n", 1281 | " 0.06911704]]\n" 1282 | ] 1283 | } 1284 | ], 1285 | "source": [ 1286 | "# W is a random 700*100 tensor\n", 1287 | "W = tf.Variable(tf.truncated_normal([700, 10]))\n", 1288 | "U = tf.Variable(W * 2)\n", 1289 | "init = tf.global_variables_initializer()\n", 1290 | "\n", 1291 | "with tf.Session() as sess:\n", 1292 | " sess.run(W.initializer)\n", 1293 | " sess.run(U.initializer)\n", 1294 | " print(W.eval())\n", 1295 | " print(U.eval())\n", 1296 | "# In this case, you should use initialized_value() to make sure that W is initialized\n", 1297 | "# before its value is used to initialize W" 1298 | ] 1299 | }, 1300 | { 1301 | "cell_type": "markdown", 1302 | "metadata": {}, 1303 | "source": [ 1304 | "### InteractiveSession\n", 1305 | "You sometimes see InteractiveSession instead of Session. The only difference is an\n", 1306 | "InteractiveSession makes itself the default session so you can call run() or eval()\n", 1307 | "without explicitly call the session. This is convenient in interactive shells and IPyhon notebooks, as it avoids having to pass an explicit Session object to run ops. However, it is complicated when you have multiple sessions to run.\n", 1308 | "\n", 1309 | "tf.get_default_session() returns the default session for the current thread. The returned Session will be the innermost session on which a Session or Session.as_default() context has been entered." 1310 | ] 1311 | }, 1312 | { 1313 | "cell_type": "code", 1314 | "execution_count": 41, 1315 | "metadata": { 1316 | "collapsed": false 1317 | }, 1318 | "outputs": [ 1319 | { 1320 | "name": "stdout", 1321 | "output_type": "stream", 1322 | "text": [ 1323 | "30.0\n" 1324 | ] 1325 | } 1326 | ], 1327 | "source": [ 1328 | "sess = tf.InteractiveSession()\n", 1329 | "a = tf.constant(5.0)\n", 1330 | "b = tf.constant(6.0)\n", 1331 | "c = a * b\n", 1332 | "# we can just use 'c.eval()' without passing 'sess'\n", 1333 | "print(c.eval())\n", 1334 | "sess.close()" 1335 | ] 1336 | }, 1337 | { 1338 | "cell_type": "markdown", 1339 | "metadata": {}, 1340 | "source": [ 1341 | "### Control Dependencies" 1342 | ] 1343 | }, 1344 | { 1345 | "cell_type": "code", 1346 | "execution_count": null, 1347 | "metadata": { 1348 | "collapsed": true 1349 | }, 1350 | "outputs": [], 1351 | "source": [ 1352 | "# # your graph g have 5 ops: a, b, c, d, e\n", 1353 | "# with g.control_dependencies([a, b, c]):\n", 1354 | "# # 'd' and 'e' will only run after 'a', 'b', and 'c' have executed\n", 1355 | "# d = ...\n", 1356 | "# e = ..." 1357 | ] 1358 | }, 1359 | { 1360 | "cell_type": "markdown", 1361 | "metadata": {}, 1362 | "source": [ 1363 | "### Placeholders and feed_dict\n", 1364 | "https://www.tensorflow.org/api_docs/python/tf/placeholder" 1365 | ] 1366 | }, 1367 | { 1368 | "cell_type": "code", 1369 | "execution_count": null, 1370 | "metadata": { 1371 | "collapsed": false 1372 | }, 1373 | "outputs": [], 1374 | "source": [ 1375 | "# to define a placeholder, we use:\n", 1376 | "# tf.placeholder(dtype, shape=None, Name=None)\n", 1377 | "\n", 1378 | "# create a placeholder of type float 32-bit, shape is a vector of 3 elements\n", 1379 | "a = tf.placeholder(tf.float32, shape=[3])\n", 1380 | "\n", 1381 | "# create a constant of type float 32-bit, shape is a vector of 3 elements\n", 1382 | "b = tf.constant([5, 5, 5], tf.float32)\n", 1383 | "\n", 1384 | "# use the placeholder as you would a constant or variable\n", 1385 | "c = a + b # short for tf.add(a, b)\n", 1386 | "\n", 1387 | "# If we try to fetch c, we will run into error.\n", 1388 | "\n", 1389 | "with tf.Session() as sess:\n", 1390 | " print(sess.run(c))\n", 1391 | "# InvalidArgumentError (see above for traceback): You must feed a value for placeh\n", 1392 | "# older tensor 'Placeholder' with dtype float and shape [3]" 1393 | ] 1394 | }, 1395 | { 1396 | "cell_type": "code", 1397 | "execution_count": 46, 1398 | "metadata": { 1399 | "collapsed": false 1400 | }, 1401 | "outputs": [ 1402 | { 1403 | "name": "stdout", 1404 | "output_type": "stream", 1405 | "text": [ 1406 | "[ 6. 7. 8.]\n" 1407 | ] 1408 | } 1409 | ], 1410 | "source": [ 1411 | "# to define a placeholder, we use:\n", 1412 | "# tf.placeholder(dtype, shape=None, Name=None)\n", 1413 | "g = tf.Graph()\n", 1414 | "with g.as_default():\n", 1415 | " # create a placeholder of type float 32-bit, shape is a vector of 3 elements\n", 1416 | " a = tf.placeholder(tf.float32, shape=[3])\n", 1417 | "\n", 1418 | " # create a constant of type float 32-bit, shape is a vector of 3 elements\n", 1419 | " b = tf.constant([5, 5, 5], tf.float32)\n", 1420 | "\n", 1421 | " # use the placeholder as you would a constant or variable\n", 1422 | " c = a + b # short for tf.add(a, b)\n", 1423 | "\n", 1424 | "# If we try to fetch c, we will run into error.\n", 1425 | "\n", 1426 | "with tf.Session(graph=g) as sess:\n", 1427 | " writer = tf.summary.FileWriter('./my_graph', sess.graph)\n", 1428 | " print(sess.run(c, feed_dict={a:[1, 2, 3]}))" 1429 | ] 1430 | }, 1431 | { 1432 | "cell_type": "code", 1433 | "execution_count": 48, 1434 | "metadata": { 1435 | "collapsed": false 1436 | }, 1437 | "outputs": [ 1438 | { 1439 | "name": "stdout", 1440 | "output_type": "stream", 1441 | "text": [ 1442 | "45\n" 1443 | ] 1444 | } 1445 | ], 1446 | "source": [ 1447 | "# we can feed as any data points to placeholder as we want by iterating through the data\n", 1448 | "# set and feed in the value one at a time.\n", 1449 | "# you can feed values to tensors that aren't placeholders. Any tensors that feedable can\n", 1450 | "# be fed. To check if a tensor is feedable or not, use:\n", 1451 | "# tf.Graph.is_feedable(tensor)\n", 1452 | "\n", 1453 | "# create Operations, Tensors, etc(using the default graph)\n", 1454 | "a = tf.add(2, 5)\n", 1455 | "b = tf.multiply(a, 3)\n", 1456 | "\n", 1457 | "with tf.Session() as sess:\n", 1458 | " replace_dict = {a: 15}\n", 1459 | " print(sess.run(b, feed_dict=replace_dict))\n", 1460 | "# feed_dict can be extremely useful to test your model. When you have a large graph and \n", 1461 | "# just want to test out certain parts, you can provide dummy values so TensorFlow won't\n", 1462 | "# waste time doing unnecessary computaions" 1463 | ] 1464 | }, 1465 | { 1466 | "cell_type": "markdown", 1467 | "metadata": {}, 1468 | "source": [ 1469 | "### The trap of lazy loading" 1470 | ] 1471 | }, 1472 | { 1473 | "cell_type": "markdown", 1474 | "metadata": {}, 1475 | "source": [ 1476 | "TensorFlow, it means you defer creating an op until you need to compute it. For example,\n", 1477 | "this is normal loading: you create the op z when you assemble the graph." 1478 | ] 1479 | }, 1480 | { 1481 | "cell_type": "code", 1482 | "execution_count": 56, 1483 | "metadata": { 1484 | "collapsed": false, 1485 | "scrolled": false 1486 | }, 1487 | "outputs": [ 1488 | { 1489 | "name": "stdout", 1490 | "output_type": "stream", 1491 | "text": [ 1492 | "10\n", 1493 | "node {\n", 1494 | " name: \"x/initial_value\"\n", 1495 | " op: \"Const\"\n", 1496 | " attr {\n", 1497 | " key: \"dtype\"\n", 1498 | " value {\n", 1499 | " type: DT_INT32\n", 1500 | " }\n", 1501 | " }\n", 1502 | " attr {\n", 1503 | " key: \"value\"\n", 1504 | " value {\n", 1505 | " tensor {\n", 1506 | " dtype: DT_INT32\n", 1507 | " tensor_shape {\n", 1508 | " }\n", 1509 | " int_val: 10\n", 1510 | " }\n", 1511 | " }\n", 1512 | " }\n", 1513 | "}\n", 1514 | "node {\n", 1515 | " name: \"x\"\n", 1516 | " op: \"VariableV2\"\n", 1517 | " attr {\n", 1518 | " key: \"container\"\n", 1519 | " value {\n", 1520 | " s: \"\"\n", 1521 | " }\n", 1522 | " }\n", 1523 | " attr {\n", 1524 | " key: \"dtype\"\n", 1525 | " value {\n", 1526 | " type: DT_INT32\n", 1527 | " }\n", 1528 | " }\n", 1529 | " attr {\n", 1530 | " key: \"shape\"\n", 1531 | " value {\n", 1532 | " shape {\n", 1533 | " }\n", 1534 | " }\n", 1535 | " }\n", 1536 | " attr {\n", 1537 | " key: \"shared_name\"\n", 1538 | " value {\n", 1539 | " s: \"\"\n", 1540 | " }\n", 1541 | " }\n", 1542 | "}\n", 1543 | "node {\n", 1544 | " name: \"x/Assign\"\n", 1545 | " op: \"Assign\"\n", 1546 | " input: \"x\"\n", 1547 | " input: \"x/initial_value\"\n", 1548 | " attr {\n", 1549 | " key: \"T\"\n", 1550 | " value {\n", 1551 | " type: DT_INT32\n", 1552 | " }\n", 1553 | " }\n", 1554 | " attr {\n", 1555 | " key: \"_class\"\n", 1556 | " value {\n", 1557 | " list {\n", 1558 | " s: \"loc:@x\"\n", 1559 | " }\n", 1560 | " }\n", 1561 | " }\n", 1562 | " attr {\n", 1563 | " key: \"use_locking\"\n", 1564 | " value {\n", 1565 | " b: true\n", 1566 | " }\n", 1567 | " }\n", 1568 | " attr {\n", 1569 | " key: \"validate_shape\"\n", 1570 | " value {\n", 1571 | " b: true\n", 1572 | " }\n", 1573 | " }\n", 1574 | "}\n", 1575 | "node {\n", 1576 | " name: \"x/read\"\n", 1577 | " op: \"Identity\"\n", 1578 | " input: \"x\"\n", 1579 | " attr {\n", 1580 | " key: \"T\"\n", 1581 | " value {\n", 1582 | " type: DT_INT32\n", 1583 | " }\n", 1584 | " }\n", 1585 | " attr {\n", 1586 | " key: \"_class\"\n", 1587 | " value {\n", 1588 | " list {\n", 1589 | " s: \"loc:@x\"\n", 1590 | " }\n", 1591 | " }\n", 1592 | " }\n", 1593 | "}\n", 1594 | "node {\n", 1595 | " name: \"y/initial_value\"\n", 1596 | " op: \"Const\"\n", 1597 | " attr {\n", 1598 | " key: \"dtype\"\n", 1599 | " value {\n", 1600 | " type: DT_INT32\n", 1601 | " }\n", 1602 | " }\n", 1603 | " attr {\n", 1604 | " key: \"value\"\n", 1605 | " value {\n", 1606 | " tensor {\n", 1607 | " dtype: DT_INT32\n", 1608 | " tensor_shape {\n", 1609 | " }\n", 1610 | " int_val: 20\n", 1611 | " }\n", 1612 | " }\n", 1613 | " }\n", 1614 | "}\n", 1615 | "node {\n", 1616 | " name: \"y\"\n", 1617 | " op: \"VariableV2\"\n", 1618 | " attr {\n", 1619 | " key: \"container\"\n", 1620 | " value {\n", 1621 | " s: \"\"\n", 1622 | " }\n", 1623 | " }\n", 1624 | " attr {\n", 1625 | " key: \"dtype\"\n", 1626 | " value {\n", 1627 | " type: DT_INT32\n", 1628 | " }\n", 1629 | " }\n", 1630 | " attr {\n", 1631 | " key: \"shape\"\n", 1632 | " value {\n", 1633 | " shape {\n", 1634 | " }\n", 1635 | " }\n", 1636 | " }\n", 1637 | " attr {\n", 1638 | " key: \"shared_name\"\n", 1639 | " value {\n", 1640 | " s: \"\"\n", 1641 | " }\n", 1642 | " }\n", 1643 | "}\n", 1644 | "node {\n", 1645 | " name: \"y/Assign\"\n", 1646 | " op: \"Assign\"\n", 1647 | " input: \"y\"\n", 1648 | " input: \"y/initial_value\"\n", 1649 | " attr {\n", 1650 | " key: \"T\"\n", 1651 | " value {\n", 1652 | " type: DT_INT32\n", 1653 | " }\n", 1654 | " }\n", 1655 | " attr {\n", 1656 | " key: \"_class\"\n", 1657 | " value {\n", 1658 | " list {\n", 1659 | " s: \"loc:@y\"\n", 1660 | " }\n", 1661 | " }\n", 1662 | " }\n", 1663 | " attr {\n", 1664 | " key: \"use_locking\"\n", 1665 | " value {\n", 1666 | " b: true\n", 1667 | " }\n", 1668 | " }\n", 1669 | " attr {\n", 1670 | " key: \"validate_shape\"\n", 1671 | " value {\n", 1672 | " b: true\n", 1673 | " }\n", 1674 | " }\n", 1675 | "}\n", 1676 | "node {\n", 1677 | " name: \"y/read\"\n", 1678 | " op: \"Identity\"\n", 1679 | " input: \"y\"\n", 1680 | " attr {\n", 1681 | " key: \"T\"\n", 1682 | " value {\n", 1683 | " type: DT_INT32\n", 1684 | " }\n", 1685 | " }\n", 1686 | " attr {\n", 1687 | " key: \"_class\"\n", 1688 | " value {\n", 1689 | " list {\n", 1690 | " s: \"loc:@y\"\n", 1691 | " }\n", 1692 | " }\n", 1693 | " }\n", 1694 | "}\n", 1695 | "node {\n", 1696 | " name: \"Add\"\n", 1697 | " op: \"Add\"\n", 1698 | " input: \"x/read\"\n", 1699 | " input: \"y/read\"\n", 1700 | " attr {\n", 1701 | " key: \"T\"\n", 1702 | " value {\n", 1703 | " type: DT_INT32\n", 1704 | " }\n", 1705 | " }\n", 1706 | "}\n", 1707 | "node {\n", 1708 | " name: \"init\"\n", 1709 | " op: \"NoOp\"\n", 1710 | " input: \"^x/Assign\"\n", 1711 | " input: \"^y/Assign\"\n", 1712 | "}\n", 1713 | "versions {\n", 1714 | " producer: 21\n", 1715 | "}\n", 1716 | "\n" 1717 | ] 1718 | } 1719 | ], 1720 | "source": [ 1721 | "g = tf.Graph()\n", 1722 | "with g.as_default():\n", 1723 | " x = tf.Variable(10, name='x')\n", 1724 | " y = tf.Variable(20, name='y')\n", 1725 | " z = tf.add(x, y)\n", 1726 | "\n", 1727 | "with tf.Session(graph=g) as sess:\n", 1728 | " sess.run(tf.global_variables_initializer())\n", 1729 | " for _ in range(10):\n", 1730 | " sess.run(z)\n", 1731 | " writer = tf.summary.FileWriter('./graph_1', sess.graph)\n", 1732 | " writer.close()\n", 1733 | " #print(sess.graph.get_operations())\n", 1734 | " print(sess.graph.version)\n", 1735 | " print(sess.graph.as_graph_def())\n", 1736 | "# The protobuf for the graph in normal loading has only 1 node “Add”" 1737 | ] 1738 | }, 1739 | { 1740 | "cell_type": "markdown", 1741 | "metadata": {}, 1742 | "source": [ 1743 | "This is what happens when someone decides to be clever and use lazy loading to save one line of code.\n", 1744 | "On the other hand, the protobuf for the graph in lazy loading has 10 copies of the \n", 1745 | "node “Add”. It adds a new node “Add” every time you want to compute z!" 1746 | ] 1747 | }, 1748 | { 1749 | "cell_type": "code", 1750 | "execution_count": 55, 1751 | "metadata": { 1752 | "collapsed": false, 1753 | "scrolled": false 1754 | }, 1755 | "outputs": [ 1756 | { 1757 | "name": "stdout", 1758 | "output_type": "stream", 1759 | "text": [ 1760 | "19\n", 1761 | "node {\n", 1762 | " name: \"x/initial_value\"\n", 1763 | " op: \"Const\"\n", 1764 | " attr {\n", 1765 | " key: \"dtype\"\n", 1766 | " value {\n", 1767 | " type: DT_INT32\n", 1768 | " }\n", 1769 | " }\n", 1770 | " attr {\n", 1771 | " key: \"value\"\n", 1772 | " value {\n", 1773 | " tensor {\n", 1774 | " dtype: DT_INT32\n", 1775 | " tensor_shape {\n", 1776 | " }\n", 1777 | " int_val: 10\n", 1778 | " }\n", 1779 | " }\n", 1780 | " }\n", 1781 | "}\n", 1782 | "node {\n", 1783 | " name: \"x\"\n", 1784 | " op: \"VariableV2\"\n", 1785 | " attr {\n", 1786 | " key: \"container\"\n", 1787 | " value {\n", 1788 | " s: \"\"\n", 1789 | " }\n", 1790 | " }\n", 1791 | " attr {\n", 1792 | " key: \"dtype\"\n", 1793 | " value {\n", 1794 | " type: DT_INT32\n", 1795 | " }\n", 1796 | " }\n", 1797 | " attr {\n", 1798 | " key: \"shape\"\n", 1799 | " value {\n", 1800 | " shape {\n", 1801 | " }\n", 1802 | " }\n", 1803 | " }\n", 1804 | " attr {\n", 1805 | " key: \"shared_name\"\n", 1806 | " value {\n", 1807 | " s: \"\"\n", 1808 | " }\n", 1809 | " }\n", 1810 | "}\n", 1811 | "node {\n", 1812 | " name: \"x/Assign\"\n", 1813 | " op: \"Assign\"\n", 1814 | " input: \"x\"\n", 1815 | " input: \"x/initial_value\"\n", 1816 | " attr {\n", 1817 | " key: \"T\"\n", 1818 | " value {\n", 1819 | " type: DT_INT32\n", 1820 | " }\n", 1821 | " }\n", 1822 | " attr {\n", 1823 | " key: \"_class\"\n", 1824 | " value {\n", 1825 | " list {\n", 1826 | " s: \"loc:@x\"\n", 1827 | " }\n", 1828 | " }\n", 1829 | " }\n", 1830 | " attr {\n", 1831 | " key: \"use_locking\"\n", 1832 | " value {\n", 1833 | " b: true\n", 1834 | " }\n", 1835 | " }\n", 1836 | " attr {\n", 1837 | " key: \"validate_shape\"\n", 1838 | " value {\n", 1839 | " b: true\n", 1840 | " }\n", 1841 | " }\n", 1842 | "}\n", 1843 | "node {\n", 1844 | " name: \"x/read\"\n", 1845 | " op: \"Identity\"\n", 1846 | " input: \"x\"\n", 1847 | " attr {\n", 1848 | " key: \"T\"\n", 1849 | " value {\n", 1850 | " type: DT_INT32\n", 1851 | " }\n", 1852 | " }\n", 1853 | " attr {\n", 1854 | " key: \"_class\"\n", 1855 | " value {\n", 1856 | " list {\n", 1857 | " s: \"loc:@x\"\n", 1858 | " }\n", 1859 | " }\n", 1860 | " }\n", 1861 | "}\n", 1862 | "node {\n", 1863 | " name: \"y/initial_value\"\n", 1864 | " op: \"Const\"\n", 1865 | " attr {\n", 1866 | " key: \"dtype\"\n", 1867 | " value {\n", 1868 | " type: DT_INT32\n", 1869 | " }\n", 1870 | " }\n", 1871 | " attr {\n", 1872 | " key: \"value\"\n", 1873 | " value {\n", 1874 | " tensor {\n", 1875 | " dtype: DT_INT32\n", 1876 | " tensor_shape {\n", 1877 | " }\n", 1878 | " int_val: 20\n", 1879 | " }\n", 1880 | " }\n", 1881 | " }\n", 1882 | "}\n", 1883 | "node {\n", 1884 | " name: \"y\"\n", 1885 | " op: \"VariableV2\"\n", 1886 | " attr {\n", 1887 | " key: \"container\"\n", 1888 | " value {\n", 1889 | " s: \"\"\n", 1890 | " }\n", 1891 | " }\n", 1892 | " attr {\n", 1893 | " key: \"dtype\"\n", 1894 | " value {\n", 1895 | " type: DT_INT32\n", 1896 | " }\n", 1897 | " }\n", 1898 | " attr {\n", 1899 | " key: \"shape\"\n", 1900 | " value {\n", 1901 | " shape {\n", 1902 | " }\n", 1903 | " }\n", 1904 | " }\n", 1905 | " attr {\n", 1906 | " key: \"shared_name\"\n", 1907 | " value {\n", 1908 | " s: \"\"\n", 1909 | " }\n", 1910 | " }\n", 1911 | "}\n", 1912 | "node {\n", 1913 | " name: \"y/Assign\"\n", 1914 | " op: \"Assign\"\n", 1915 | " input: \"y\"\n", 1916 | " input: \"y/initial_value\"\n", 1917 | " attr {\n", 1918 | " key: \"T\"\n", 1919 | " value {\n", 1920 | " type: DT_INT32\n", 1921 | " }\n", 1922 | " }\n", 1923 | " attr {\n", 1924 | " key: \"_class\"\n", 1925 | " value {\n", 1926 | " list {\n", 1927 | " s: \"loc:@y\"\n", 1928 | " }\n", 1929 | " }\n", 1930 | " }\n", 1931 | " attr {\n", 1932 | " key: \"use_locking\"\n", 1933 | " value {\n", 1934 | " b: true\n", 1935 | " }\n", 1936 | " }\n", 1937 | " attr {\n", 1938 | " key: \"validate_shape\"\n", 1939 | " value {\n", 1940 | " b: true\n", 1941 | " }\n", 1942 | " }\n", 1943 | "}\n", 1944 | "node {\n", 1945 | " name: \"y/read\"\n", 1946 | " op: \"Identity\"\n", 1947 | " input: \"y\"\n", 1948 | " attr {\n", 1949 | " key: \"T\"\n", 1950 | " value {\n", 1951 | " type: DT_INT32\n", 1952 | " }\n", 1953 | " }\n", 1954 | " attr {\n", 1955 | " key: \"_class\"\n", 1956 | " value {\n", 1957 | " list {\n", 1958 | " s: \"loc:@y\"\n", 1959 | " }\n", 1960 | " }\n", 1961 | " }\n", 1962 | "}\n", 1963 | "node {\n", 1964 | " name: \"init\"\n", 1965 | " op: \"NoOp\"\n", 1966 | " input: \"^x/Assign\"\n", 1967 | " input: \"^y/Assign\"\n", 1968 | "}\n", 1969 | "node {\n", 1970 | " name: \"Add\"\n", 1971 | " op: \"Add\"\n", 1972 | " input: \"x/read\"\n", 1973 | " input: \"y/read\"\n", 1974 | " attr {\n", 1975 | " key: \"T\"\n", 1976 | " value {\n", 1977 | " type: DT_INT32\n", 1978 | " }\n", 1979 | " }\n", 1980 | "}\n", 1981 | "node {\n", 1982 | " name: \"Add_1\"\n", 1983 | " op: \"Add\"\n", 1984 | " input: \"x/read\"\n", 1985 | " input: \"y/read\"\n", 1986 | " attr {\n", 1987 | " key: \"T\"\n", 1988 | " value {\n", 1989 | " type: DT_INT32\n", 1990 | " }\n", 1991 | " }\n", 1992 | "}\n", 1993 | "node {\n", 1994 | " name: \"Add_2\"\n", 1995 | " op: \"Add\"\n", 1996 | " input: \"x/read\"\n", 1997 | " input: \"y/read\"\n", 1998 | " attr {\n", 1999 | " key: \"T\"\n", 2000 | " value {\n", 2001 | " type: DT_INT32\n", 2002 | " }\n", 2003 | " }\n", 2004 | "}\n", 2005 | "node {\n", 2006 | " name: \"Add_3\"\n", 2007 | " op: \"Add\"\n", 2008 | " input: \"x/read\"\n", 2009 | " input: \"y/read\"\n", 2010 | " attr {\n", 2011 | " key: \"T\"\n", 2012 | " value {\n", 2013 | " type: DT_INT32\n", 2014 | " }\n", 2015 | " }\n", 2016 | "}\n", 2017 | "node {\n", 2018 | " name: \"Add_4\"\n", 2019 | " op: \"Add\"\n", 2020 | " input: \"x/read\"\n", 2021 | " input: \"y/read\"\n", 2022 | " attr {\n", 2023 | " key: \"T\"\n", 2024 | " value {\n", 2025 | " type: DT_INT32\n", 2026 | " }\n", 2027 | " }\n", 2028 | "}\n", 2029 | "node {\n", 2030 | " name: \"Add_5\"\n", 2031 | " op: \"Add\"\n", 2032 | " input: \"x/read\"\n", 2033 | " input: \"y/read\"\n", 2034 | " attr {\n", 2035 | " key: \"T\"\n", 2036 | " value {\n", 2037 | " type: DT_INT32\n", 2038 | " }\n", 2039 | " }\n", 2040 | "}\n", 2041 | "node {\n", 2042 | " name: \"Add_6\"\n", 2043 | " op: \"Add\"\n", 2044 | " input: \"x/read\"\n", 2045 | " input: \"y/read\"\n", 2046 | " attr {\n", 2047 | " key: \"T\"\n", 2048 | " value {\n", 2049 | " type: DT_INT32\n", 2050 | " }\n", 2051 | " }\n", 2052 | "}\n", 2053 | "node {\n", 2054 | " name: \"Add_7\"\n", 2055 | " op: \"Add\"\n", 2056 | " input: \"x/read\"\n", 2057 | " input: \"y/read\"\n", 2058 | " attr {\n", 2059 | " key: \"T\"\n", 2060 | " value {\n", 2061 | " type: DT_INT32\n", 2062 | " }\n", 2063 | " }\n", 2064 | "}\n", 2065 | "node {\n", 2066 | " name: \"Add_8\"\n", 2067 | " op: \"Add\"\n", 2068 | " input: \"x/read\"\n", 2069 | " input: \"y/read\"\n", 2070 | " attr {\n", 2071 | " key: \"T\"\n", 2072 | " value {\n", 2073 | " type: DT_INT32\n", 2074 | " }\n", 2075 | " }\n", 2076 | "}\n", 2077 | "node {\n", 2078 | " name: \"Add_9\"\n", 2079 | " op: \"Add\"\n", 2080 | " input: \"x/read\"\n", 2081 | " input: \"y/read\"\n", 2082 | " attr {\n", 2083 | " key: \"T\"\n", 2084 | " value {\n", 2085 | " type: DT_INT32\n", 2086 | " }\n", 2087 | " }\n", 2088 | "}\n", 2089 | "versions {\n", 2090 | " producer: 21\n", 2091 | "}\n", 2092 | "\n" 2093 | ] 2094 | } 2095 | ], 2096 | "source": [ 2097 | "g = tf.Graph()\n", 2098 | "with g.as_default():\n", 2099 | " x = tf.Variable(10, name='x')\n", 2100 | " y = tf.Variable(20, name='y')\n", 2101 | "\n", 2102 | "with tf.Session(graph=g) as sess:\n", 2103 | " sess.run(tf.global_variables_initializer())\n", 2104 | " for _ in range(10):\n", 2105 | " sess.run(tf.add(x, y)) # # create the op add only when you need to compute it\n", 2106 | " writer = tf.summary.FileWriter('./graph_2', sess.graph)\n", 2107 | " writer.close()\n", 2108 | " #print(sess.graph.get_operations())\n", 2109 | " print(sess.graph.version)\n", 2110 | " print(sess.graph.as_graph_def())" 2111 | ] 2112 | } 2113 | ], 2114 | "metadata": { 2115 | "kernelspec": { 2116 | "display_name": "Python 2", 2117 | "language": "python", 2118 | "name": "python2" 2119 | }, 2120 | "language_info": { 2121 | "codemirror_mode": { 2122 | "name": "ipython", 2123 | "version": 2 2124 | }, 2125 | "file_extension": ".py", 2126 | "mimetype": "text/x-python", 2127 | "name": "python", 2128 | "nbconvert_exporter": "python", 2129 | "pygments_lexer": "ipython2", 2130 | "version": "2.7.12" 2131 | } 2132 | }, 2133 | "nbformat": 4, 2134 | "nbformat_minor": 2 2135 | } 2136 | -------------------------------------------------------------------------------- /3_2_logistic_regression_in_tensorflow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "1. [Understanding Xavier Initialization In Deep Neural Networks](https://prateekvjoshi.com/2016/03/29/understanding-xavier-initialization-in-deep-neural-networks/)\n", 8 | "2. [Understanding the difficulty of training deep feedforward neural networks](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "metadata": { 15 | "collapsed": false 16 | }, 17 | "outputs": [], 18 | "source": [ 19 | "import numpy as np\n", 20 | "import tensorflow as tf\n", 21 | "from tensorflow.examples.tutorials.mnist import input_data\n", 22 | "from wrap_functions import define_scope\n", 23 | "import functools" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "metadata": { 30 | "collapsed": false 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "class Model:\n", 35 | " def __init__(self, input_data, label, input_dim=784, output_dim=10, learning_rate=0.01):\n", 36 | " # config\n", 37 | " self.input_dim = input_dim\n", 38 | " self.output_dim = output_dim\n", 39 | " self.learning_rate = learning_rate\n", 40 | " \n", 41 | " # ops\n", 42 | " self.input_data = input_data\n", 43 | " self.label = label\n", 44 | " self.prediction\n", 45 | " self.optimize\n", 46 | " self.error\n", 47 | " \n", 48 | " @define_scope\n", 49 | " def prediction(self):\n", 50 | " xvaier_stddev = tf.sqrt(1.0 / tf.cast(self.input_dim + self.output_dim, dtype=tf.float32))\n", 51 | " w = tf.Variable(tf.random_normal(shape=[self.input_dim, self.output_dim],\n", 52 | " stddev=xvaier_stddev), name='weights')\n", 53 | " b = tf.Variable(tf.zeros(shape=[self.output_dim], name='bias'))\n", 54 | " logits = tf.matmul(self.input_data, w) + b\n", 55 | " preds = tf.nn.softmax(logits)\n", 56 | " return preds # preds will be set as an attribute named _cache_prediction\n", 57 | " \n", 58 | " @define_scope\n", 59 | " def optimize(self):\n", 60 | " logprob = tf.log(self.prediction + 1e-12)\n", 61 | " cross_entropy = -tf.reduce_sum(self.label * logprob)\n", 62 | " optimizer = tf.train.RMSPropOptimizer(self.learning_rate)\n", 63 | " return optimizer.minimize(cross_entropy)\n", 64 | " \n", 65 | " @define_scope\n", 66 | " def error(self):\n", 67 | " mistakes = tf.not_equal(\n", 68 | " tf.argmax(self.label, 1), tf.argmax(self.prediction, 1))\n", 69 | " return tf.reduce_mean(tf.cast(mistakes, tf.float32))" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 3, 75 | "metadata": { 76 | "collapsed": false 77 | }, 78 | "outputs": [ 79 | { 80 | "name": "stdout", 81 | "output_type": "stream", 82 | "text": [ 83 | "Extracting ./data/mnist/train-images-idx3-ubyte.gz\n", 84 | "Extracting ./data/mnist/train-labels-idx1-ubyte.gz\n", 85 | "Extracting ./data/mnist/t10k-images-idx3-ubyte.gz\n", 86 | "Extracting ./data/mnist/t10k-labels-idx1-ubyte.gz\n" 87 | ] 88 | } 89 | ], 90 | "source": [ 91 | "mnist = input_data.read_data_sets(\"./data/mnist\", one_hot=True)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 4, 97 | "metadata": { 98 | "collapsed": true 99 | }, 100 | "outputs": [], 101 | "source": [ 102 | "lr = 0.01\n", 103 | "batch_size = 128\n", 104 | "n_epochs = 25\n", 105 | "n_batches = int(mnist.train.num_examples/batch_size)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 5, 111 | "metadata": { 112 | "collapsed": false 113 | }, 114 | "outputs": [ 115 | { 116 | "name": "stdout", 117 | "output_type": "stream", 118 | "text": [ 119 | "Epoch 0 : Test error 90.85%\n", 120 | "Epoch 1 : Test error 12.90%\n", 121 | "Epoch 2 : Test error 9.80%\n", 122 | "Epoch 3 : Test error 8.85%\n", 123 | "Epoch 4 : Test error 9.23%\n", 124 | "Epoch 5 : Test error 8.88%\n", 125 | "Epoch 6 : Test error 8.28%\n", 126 | "Epoch 7 : Test error 8.09%\n", 127 | "Epoch 8 : Test error 8.88%\n", 128 | "Epoch 9 : Test error 8.44%\n", 129 | "Epoch 10 : Test error 8.37%\n", 130 | "Epoch 11 : Test error 8.12%\n", 131 | "Epoch 12 : Test error 8.11%\n", 132 | "Epoch 13 : Test error 8.89%\n", 133 | "Epoch 14 : Test error 9.02%\n", 134 | "Epoch 15 : Test error 8.20%\n", 135 | "Epoch 16 : Test error 8.58%\n", 136 | "Epoch 17 : Test error 8.03%\n", 137 | "Epoch 18 : Test error 7.99%\n", 138 | "Epoch 19 : Test error 7.50%\n", 139 | "Epoch 20 : Test error 7.86%\n", 140 | "Epoch 21 : Test error 7.83%\n", 141 | "Epoch 22 : Test error 8.40%\n", 142 | "Epoch 23 : Test error 7.81%\n", 143 | "Epoch 24 : Test error 7.64%\n" 144 | ] 145 | } 146 | ], 147 | "source": [ 148 | "def main():\n", 149 | " g = tf.Graph()\n", 150 | " with g.as_default():\n", 151 | " image = tf.placeholder(tf.float32, [None, 784])\n", 152 | " label = tf.placeholder(tf.float32, [None, 10])\n", 153 | " model = Model(image, label)\n", 154 | "\n", 155 | " config = tf.ConfigProto()\n", 156 | " config.gpu_options.allow_growth = True\n", 157 | "\n", 158 | " with tf.Session(graph=g, config=config) as sess:\n", 159 | " sess.run(tf.global_variables_initializer())\n", 160 | "\n", 161 | " for i in range(n_epochs):\n", 162 | " images, labels = mnist.test.images, mnist.test.labels\n", 163 | " error = sess.run(model.error, {image: images, label: labels})\n", 164 | " print('Epoch {} : Test error {:6.2f}%'.format(i, 100 * error))\n", 165 | " for _ in range(60):\n", 166 | " images, labels = mnist.train.next_batch(batch_size)\n", 167 | " sess.run(model.optimize, {image: images, label: labels})\n", 168 | "\n", 169 | " writer = tf.summary.FileWriter('./my_graph/03/logstic_reg', sess.graph)\n", 170 | " writer.close()\n", 171 | " \n", 172 | "if __name__ == '__main__':\n", 173 | " main()" 174 | ] 175 | } 176 | ], 177 | "metadata": { 178 | "kernelspec": { 179 | "display_name": "Python 2", 180 | "language": "python", 181 | "name": "python2" 182 | }, 183 | "language_info": { 184 | "codemirror_mode": { 185 | "name": "ipython", 186 | "version": 2 187 | }, 188 | "file_extension": ".py", 189 | "mimetype": "text/x-python", 190 | "name": "python", 191 | "nbconvert_exporter": "python", 192 | "pygments_lexer": "ipython2", 193 | "version": "2.7.12" 194 | } 195 | }, 196 | "nbformat": 4, 197 | "nbformat_minor": 2 198 | } 199 | -------------------------------------------------------------------------------- /4_word2vec_visualize.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "from __future__ import absolute_import\n", 12 | "from __future__ import division\n", 13 | "from __future__ import print_function\n", 14 | "\n", 15 | "import os\n", 16 | "import numpy as np\n", 17 | "from tensorflow.contrib.tensorboard.plugins import projector\n", 18 | "import tensorflow as tf\n", 19 | "\n", 20 | "from process_data import process_data\n", 21 | "from wrap_functions import doublewrap\n", 22 | "from wrap_functions import define_scope" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 2, 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "VOCAB_SIZE = 50000\n", 34 | "BATCH_SIZE = 128\n", 35 | "EMBED_SIZE = 128 # dimension of the word embedding vectors\n", 36 | "SKIP_WINDOW = 1 # the context window\n", 37 | "NUM_SAMPLED = 64 # Number of negative examples to sample\n", 38 | "LEARNING_RATE = 1.0\n", 39 | "NUM_TRAIN_STEPS = 10000\n", 40 | "WEIGHTS_FLD = 'processed/'\n", 41 | "SKIP_STEP = 2000" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 3, 47 | "metadata": { 48 | "collapsed": false 49 | }, 50 | "outputs": [], 51 | "source": [ 52 | "class SkipGramModel:\n", 53 | " \"\"\" Build the graph for word2vec model \"\"\"\n", 54 | " def __init__(self, vocab_size, embed_size, batch_size, num_sampled, learning_rate):\n", 55 | " # config\n", 56 | " self.vocab_size = vocab_size\n", 57 | " self.embed_size = embed_size\n", 58 | " self.batch_size = batch_size\n", 59 | " self.num_sampled = num_sampled\n", 60 | " self.learning_rate = learning_rate\n", 61 | " \n", 62 | " # ops\n", 63 | " with tf.device('/gpu:0'):\n", 64 | " self.global_step = tf.Variable(0, dtype=tf.int32, trainable=False, name='global_step')\n", 65 | " self.create_placeholders()\n", 66 | " \n", 67 | " self.prediction\n", 68 | " self.loss\n", 69 | " self.optimize\n", 70 | " self.summaries\n", 71 | " \n", 72 | " def create_placeholders(self):\n", 73 | " with tf.name_scope('data'):\n", 74 | " self.center_words = tf.placeholder(tf.int32, shape=[self.batch_size], name='center_words')\n", 75 | " self.target_words = tf.placeholder(tf.int32, shape=[self.batch_size, 1], name='target_words')\n", 76 | " \n", 77 | " \n", 78 | " @define_scope(scope='embed')\n", 79 | " def prediction(self):\n", 80 | " self.embed_matrix = tf.Variable(tf.random_uniform([self.vocab_size,\n", 81 | " self.embed_size], -1.0, 1.0),\n", 82 | " name='embed_matrix')\n", 83 | " embed_predict = tf.nn.embedding_lookup(self.embed_matrix, self.center_words, name='embed_predict')\n", 84 | " return embed_predict\n", 85 | " \n", 86 | " @define_scope\n", 87 | " def loss(self):\n", 88 | " nce_weight = tf.Variable(tf.truncated_normal([self.vocab_size, self.embed_size],\n", 89 | " stddev=1.0 / (self.embed_size ** 0.5)),\n", 90 | " name='nce_weight')\n", 91 | " nce_bias = tf.Variable(tf.zeros([self.vocab_size]), name='nce_bias')\n", 92 | " \n", 93 | " loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weight,\n", 94 | " biases=nce_bias,\n", 95 | " labels=self.target_words,\n", 96 | " inputs=self.prediction,\n", 97 | " num_sampled=self.num_sampled,\n", 98 | " num_classes=self.vocab_size), name='nce_loss')\n", 99 | " return loss\n", 100 | " \n", 101 | " @define_scope\n", 102 | " def optimize(self):\n", 103 | " optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)\n", 104 | " return optimizer.minimize(self.loss, global_step=self.global_step)\n", 105 | " \n", 106 | " @define_scope\n", 107 | " def summaries(self):\n", 108 | " tf.summary.scalar('loss', self.loss)\n", 109 | " tf.summary.histogram('histogram_loss', self.loss)\n", 110 | " return tf.summary.merge_all()" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 4, 116 | "metadata": { 117 | "collapsed": false 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "def train_model(model, batch_gen, num_train_steps, weights_fld):\n", 122 | " saver = tf.train.Saver()\n", 123 | " \n", 124 | " initial_step = 0\n", 125 | " \n", 126 | " config = tf.ConfigProto()\n", 127 | " config.gpu_options.allow_growth = True\n", 128 | " config.allow_soft_placement = True\n", 129 | " with tf.Session(config=config) as sess:\n", 130 | " sess.run(tf.global_variables_initializer())\n", 131 | " ckpt = tf.train.get_checkpoint_state(os.path.dirname('./checkpoints/checkpoint'))\n", 132 | " if ckpt and ckpt.model_checkpoint_path:\n", 133 | " saver.restore(sess, ckpt.model_checkpoint_path)\n", 134 | " \n", 135 | " total_loss = 0.0\n", 136 | " writer = tf.summary.FileWriter('./improved_graph/lr' + str(LEARNING_RATE), sess.graph)\n", 137 | " initial_step = model.global_step.eval()\n", 138 | " for index in xrange(initial_step, initial_step + num_train_steps):\n", 139 | " centers, targets = batch_gen.next()\n", 140 | " feed_dict = {model.center_words: centers, model.target_words: targets}\n", 141 | " loss_batch, _, summary = sess.run([model.loss, model.optimize, model.summaries],\n", 142 | " feed_dict=feed_dict)\n", 143 | " writer.add_summary(summary, global_step=index)\n", 144 | " total_loss += loss_batch\n", 145 | " if(index + 1) % SKIP_STEP == 0:\n", 146 | " print('Average loss at step{}:{:5.1f}'.format(index, total_loss / SKIP_STEP))\n", 147 | " total_loss = 0.0\n", 148 | " saver.save(sess, './checkpoints/skip-gram', index)\n", 149 | " \n", 150 | " ####################\n", 151 | " # code to visualize the embeddings. uncomment the below to visualize embeddings\n", 152 | " final_embed_matrix = sess.run(model.embed_matrix)\n", 153 | "\n", 154 | " # it has to variable. constants don't work here. you can't reuse model.embed_matrix\n", 155 | " embedding_var = tf.Variable(final_embed_matrix[:1000], name='embedding')\n", 156 | " sess.run(embedding_var.initializer)\n", 157 | "\n", 158 | " config = projector.ProjectorConfig()\n", 159 | " summary_writer = tf.summary.FileWriter('processed')\n", 160 | "\n", 161 | " # add embedding to the config file\n", 162 | " embedding = config.embeddings.add()\n", 163 | " embedding.tensor_name = embedding_var.name\n", 164 | "\n", 165 | " # link this tensor to its metadata file, in this case the first 500 words of vocab\n", 166 | " embedding.metadata_path = 'processed/vocab_1000.tsv'\n", 167 | "\n", 168 | " # saves a configuration file that TensorBoard will read during startup.\n", 169 | " projector.visualize_embeddings(summary_writer, config)\n", 170 | " saver_embed = tf.train.Saver([embedding_var])\n", 171 | " saver_embed.save(sess, 'processed/model3.ckpt', 1)" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 5, 177 | "metadata": { 178 | "collapsed": false, 179 | "scrolled": false 180 | }, 181 | "outputs": [ 182 | { 183 | "name": "stdout", 184 | "output_type": "stream", 185 | "text": [ 186 | "Dataset ready\n", 187 | "Average loss at step241999: 4.0\n", 188 | "Average loss at step243999: 4.0\n", 189 | "Average loss at step245999: 4.0\n", 190 | "Average loss at step247999: 4.0\n", 191 | "Average loss at step249999: 4.0\n" 192 | ] 193 | } 194 | ], 195 | "source": [ 196 | "def main():\n", 197 | "\n", 198 | " model = SkipGramModel(VOCAB_SIZE, EMBED_SIZE, BATCH_SIZE, NUM_SAMPLED, LEARNING_RATE)\n", 199 | " \n", 200 | " batch_gen = process_data(VOCAB_SIZE, BATCH_SIZE, SKIP_WINDOW)\n", 201 | " train_model(model, batch_gen, NUM_TRAIN_STEPS, WEIGHTS_FLD)\n", 202 | "\n", 203 | "if __name__ == '__main__':\n", 204 | " main()" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "![china_embed](./china_embed.png)" 212 | ] 213 | } 214 | ], 215 | "metadata": { 216 | "kernelspec": { 217 | "display_name": "Python 2", 218 | "language": "python", 219 | "name": "python2" 220 | }, 221 | "language_info": { 222 | "codemirror_mode": { 223 | "name": "ipython", 224 | "version": 2 225 | }, 226 | "file_extension": ".py", 227 | "mimetype": "text/x-python", 228 | "name": "python", 229 | "nbconvert_exporter": "python", 230 | "pygments_lexer": "ipython2", 231 | "version": "2.7.12" 232 | } 233 | }, 234 | "nbformat": 4, 235 | "nbformat_minor": 2 236 | } 237 | -------------------------------------------------------------------------------- /5_randomization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import tensorflow as tf" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## Set random seed at operation level" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "metadata": { 25 | "collapsed": false 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "a = tf.random_uniform([], -10, 10, seed=2)" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 3, 35 | "metadata": { 36 | "collapsed": false 37 | }, 38 | "outputs": [ 39 | { 40 | "name": "stdout", 41 | "output_type": "stream", 42 | "text": [ 43 | "3.57493\n", 44 | "-5.97319\n" 45 | ] 46 | } 47 | ], 48 | "source": [ 49 | "with tf.Session() as sess:\n", 50 | " print(sess.run(a))\n", 51 | " print(sess.run(a))" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 4, 57 | "metadata": { 58 | "collapsed": false 59 | }, 60 | "outputs": [ 61 | { 62 | "name": "stdout", 63 | "output_type": "stream", 64 | "text": [ 65 | "3.57493\n", 66 | "3.57493\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "with tf.Session() as sess:\n", 72 | " print(sess.run(a))\n", 73 | "\n", 74 | "with tf.Session() as sess:\n", 75 | " print(sess.run(a))" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 5, 81 | "metadata": { 82 | "collapsed": false 83 | }, 84 | "outputs": [ 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "3.57493\n", 90 | "3.57493\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "b = tf.random_uniform([], -10, 10, seed=2)\n", 96 | "c = tf.random_uniform([], -10, 10, seed=2)\n", 97 | "\n", 98 | "with tf.Session() as sess:\n", 99 | " print(sess.run(b))\n", 100 | " print(sess.run(c))" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "## set random seed at graph level with tf.Graph.seed" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 6, 113 | "metadata": { 114 | "collapsed": false 115 | }, 116 | "outputs": [ 117 | { 118 | "name": "stdout", 119 | "output_type": "stream", 120 | "text": [ 121 | "-4.00752\n", 122 | "-2.98339\n" 123 | ] 124 | } 125 | ], 126 | "source": [ 127 | "g = tf.Graph()\n", 128 | "with g.as_default():\n", 129 | " tf.set_random_seed(2)\n", 130 | " d = tf.random_uniform([], -10, 10)\n", 131 | " e = tf.random_uniform([], -10, 10)\n", 132 | "\n", 133 | "with tf.Session(graph=g) as sess:\n", 134 | " print(sess.run(d))\n", 135 | " print(sess.run(e))\n", 136 | "# If the graph structure changes,this result would be different." 137 | ] 138 | } 139 | ], 140 | "metadata": { 141 | "kernelspec": { 142 | "display_name": "Python 2", 143 | "language": "python", 144 | "name": "python2" 145 | }, 146 | "language_info": { 147 | "codemirror_mode": { 148 | "name": "ipython", 149 | "version": 2 150 | }, 151 | "file_extension": ".py", 152 | "mimetype": "text/x-python", 153 | "name": "python", 154 | "nbconvert_exporter": "python", 155 | "pygments_lexer": "ipython2", 156 | "version": "2.7.12" 157 | } 158 | }, 159 | "nbformat": 4, 160 | "nbformat_minor": 2 161 | } 162 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cs20si 2 | standford cs20si learning notes by lightaime 3 | 4 | Email: lightaime@gmail.com 5 | -------------------------------------------------------------------------------- /china_embed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lightaime/cs20si/f96bb769bdffa6214f819928770f26f5499928fc/china_embed.png -------------------------------------------------------------------------------- /process_data.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | from collections import Counter 6 | import random 7 | import os 8 | import zipfile 9 | 10 | import numpy as np 11 | from six.moves import urllib 12 | import tensorflow as tf 13 | 14 | # Parameters for downloading data 15 | DOWNLOAD_URL = 'http://mattmahoney.net/dc/' 16 | EXPECTED_BYTES = 31344016 17 | DATA_FOLDER = './' 18 | 19 | FILE_NAME = 'text8.zip' 20 | 21 | def download(file_name, expected_bytes): 22 | """ Download the dataset text8 if it's not already downloaded """ 23 | file_path = DATA_FOLDER + file_name 24 | if os.path.exists(file_path): 25 | print("Dataset ready") 26 | return file_path 27 | file_name, _ = urllib.request.urlretrieve(DOWNLOAD_URL + file_name, file_path) 28 | file_stat = os.stat(file_path) 29 | if file_stat.st_size == expected_bytes: 30 | print('Successfully downloaded the file', file_name) 31 | else: 32 | raise Exception('File ' + file_name + 33 | ' might be corrupted. You should try downloading it with a browser.') 34 | return file_path 35 | 36 | def read_data(file_path): 37 | """ Read data into a list of tokens 38 | There should be 17,005,207 tokens 39 | """ 40 | with zipfile.ZipFile(file_path) as f: 41 | words = tf.compat.as_str(f.read(f.namelist()[0])).split() 42 | # tf.compat.as_str() converts the input into the string 43 | return words 44 | 45 | def build_vocab(words, vocab_size): 46 | """ Build vocabulary of VOCAB_SIZE most frequent words """ 47 | dictionary = dict() 48 | count = [('UNK', -1)] 49 | count.extend(Counter(words).most_common(vocab_size - 1)) 50 | index = 0 51 | with open('processed/vocab_1000.tsv', "w") as f: 52 | # f.write("Name\n") 53 | for word, _ in count: 54 | dictionary[word] = index 55 | if index < 1000: 56 | f.write(word + "\n") 57 | index += 1 58 | index_dictionary = dict(zip(dictionary.values(), dictionary.keys())) 59 | return dictionary, index_dictionary 60 | 61 | def convert_words_to_index(words, dictionary): 62 | """ Replace each word in the dataset with its index in the dictionary """ 63 | return [dictionary[word] if word in dictionary else 0 for word in words] 64 | 65 | def generate_sample(index_words, context_window_size): 66 | """ Form training pairs according to the skip-gram model. """ 67 | for index, center in enumerate(index_words): 68 | context = random.randint(1, context_window_size) 69 | # get a random target before the center word 70 | for target in index_words[max(0, index - context): index]: 71 | yield center, target 72 | # get a random target after the center wrod 73 | for target in index_words[index + 1: index + context + 1]: 74 | yield center, target 75 | 76 | def get_batch(iterator, batch_size): 77 | """ Group a numerical stream into batches and yield them as Numpy arrays. """ 78 | while True: 79 | center_batch = np.zeros(batch_size, dtype=np.int32) 80 | target_batch = np.zeros([batch_size, 1]) 81 | for index in range(batch_size): 82 | center_batch[index], target_batch[index] = next(iterator) 83 | yield center_batch, target_batch 84 | 85 | def process_data(vocab_size, batch_size, skip_window): 86 | file_path = download(FILE_NAME, EXPECTED_BYTES) 87 | words = read_data(file_path) 88 | dictionary, _ = build_vocab(words, vocab_size) 89 | index_words = convert_words_to_index(words, dictionary) 90 | del words # to save memory 91 | single_gen = generate_sample(index_words, skip_window) 92 | return get_batch(single_gen, batch_size) 93 | 94 | def get_index_vocab(vocab_size): 95 | file_path = download(FILE_NAME, EXPECTED_BYTES) 96 | words = read_data(file_path) 97 | return build_vocab(words, vocab_size) 98 | -------------------------------------------------------------------------------- /wrap_functions.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import tensorflow as tf 3 | from tensorflow.examples.tutorials.mnist import input_data 4 | 5 | def doublewrap(function): 6 | """ 7 | A decorator decorator, allowing to use the decorator to be used without 8 | parentheses if not arguments are provided. All arguments must be optional. 9 | """ 10 | @functools.wraps(function) 11 | def decorator(*args, **kwargs): 12 | if len(args) == 1 and len(kwargs) == 0 and callable(args[0]): 13 | return function(args[0]) 14 | else: 15 | return lambda wrapee: function(wrapee, *args, **kwargs) 16 | return decorator 17 | 18 | @doublewrap 19 | def define_scope(function, scope=None, *args, **kwargs): 20 | """ 21 | A decorator for functions that define TensorFlow operations. The wrapped 22 | function will only be executed once. Subsequent calls to it will directly 23 | return the result so that operations are added to the graph only once. 24 | The operations added by the function live within a tf.variable_scope(). If 25 | this decorator is used with arguments, they will be forwarded to the 26 | variable scope. The scope name defaults to the name of the wrapped 27 | function. 28 | """ 29 | attribute = '_cache_' + function.__name__ 30 | name = scope or function.__name__ 31 | @property 32 | @functools.wraps(function) 33 | def decorator(self): 34 | if not hasattr(self, attribute): 35 | with tf.variable_scope(name, *args, **kwargs): 36 | setattr(self, attribute, function(self)) 37 | return getattr(self, attribute) 38 | return decorator 39 | 40 | class Model: 41 | 42 | def __init__(self, image, label): 43 | self.image = image 44 | self.label = label 45 | self.prediction # call function without parantheses 46 | self.optimize 47 | self.error 48 | 49 | @define_scope(initializer=tf.contrib.slim.xavier_initializer()) 50 | def prediction(self): 51 | x = self.image 52 | x = tf.contrib.slim.fully_connected(x, 200) 53 | x = tf.contrib.slim.fully_connected(x, 200) 54 | x = tf.contrib.slim.fully_connected(x, 10, tf.nn.softmax) 55 | return x 56 | 57 | @define_scope 58 | def optimize(self): 59 | logprob = tf.log(self.prediction + 1e-12) 60 | cross_entropy = -tf.reduce_sum(self.label * logprob) 61 | optimizer = tf.train.RMSPropOptimizer(0.03) 62 | return optimizer.minimize(cross_entropy) 63 | 64 | @define_scope 65 | def error(self): 66 | mistakes = tf.not_equal( 67 | tf.argmax(self.label, 1), tf.argmax(self.prediction, 1)) 68 | return tf.reduce_mean(tf.cast(mistakes, tf.float32)) 69 | 70 | 71 | def main(): 72 | mnist = input_data.read_data_sets('./mnist/', one_hot=True) 73 | image = tf.placeholder(tf.float32, [None, 784]) 74 | label = tf.placeholder(tf.float32, [None, 10]) 75 | model = Model(image, label) 76 | 77 | config = tf.ConfigProto() 78 | config.gpu_options.allow_growth=True 79 | sess = tf.Session(config=config) 80 | sess.run(tf.global_variables_initializer()) 81 | 82 | for _ in range(10): 83 | images, labels = mnist.test.images, mnist.test.labels 84 | error = sess.run(model.error, {image: images, label: labels}) 85 | print('Test error {:6.2f}%'.format(100 * error)) 86 | for _ in range(60): 87 | images, labels = mnist.train.next_batch(100) 88 | sess.run(model.optimize, {image: images, label: labels}) 89 | 90 | 91 | if __name__ == '__main__': 92 | main() --------------------------------------------------------------------------------