├── .flake8 ├── .gitignore ├── 02__up_and_running ├── hello_world.py └── softmax.py ├── 03__tensorflow_basics └── Chapter3.ipynb ├── 04__convolutional_neural_networks ├── __init__.py ├── cifar_cnn.py ├── layers.py └── mnist_cnn.py ├── 05__text_and_visualizations ├── BasicRNNCell.py ├── LSTM_supervised_embeddings.py ├── scan_example.py └── vanilla_rnn_with_tfboard.py ├── 06__word_embeddings_and_rnns ├── GRU_pretrained_GloVe.py └── word2vec.py ├── 07__abstractions └── Chapter7.ipynb ├── 08__queues_threads ├── queue_basic.py ├── tfrecords_end_to_end.py └── tfrecords_read_write.py ├── 09__distributed_tensorflow ├── distribute-run.py └── distribute.py ├── 10__serving └── Chapter10.ipynb ├── LICENSE └── README.md /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /02__up_and_running/hello_world.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | h = tf.constant("Hello") 4 | w = tf.constant(" World!") 5 | hw = h + w 6 | 7 | with tf.Session() as sess: 8 | ans = sess.run(hw) 9 | 10 | print(ans) 11 | -------------------------------------------------------------------------------- /02__up_and_running/softmax.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | 4 | 5 | DATA_DIR = '/tmp/data' 6 | NUM_STEPS = 1000 7 | MINIBATCH_SIZE = 100 8 | 9 | 10 | data = input_data.read_data_sets(DATA_DIR, one_hot=True) 11 | 12 | x = tf.placeholder(tf.float32, [None, 784]) 13 | W = tf.Variable(tf.zeros([784, 10])) 14 | 15 | y_true = tf.placeholder(tf.float32, [None, 10]) 16 | y_pred = tf.matmul(x, W) 17 | 18 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits( 19 | logits=y_pred, labels=y_true)) 20 | 21 | gd_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) 22 | 23 | correct_mask = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1)) 24 | accuracy = tf.reduce_mean(tf.cast(correct_mask, tf.float32)) 25 | 26 | with tf.Session() as sess: 27 | 28 | # Train 29 | sess.run(tf.global_variables_initializer()) 30 | for _ in range(NUM_STEPS): 31 | batch_xs, batch_ys = data.train.next_batch(MINIBATCH_SIZE) 32 | sess.run(gd_step, feed_dict={x: batch_xs, y_true: batch_ys}) 33 | 34 | # Test 35 | ans = sess.run(accuracy, feed_dict={x: data.test.images, y_true: data.test.labels}) 36 | 37 | print("Accuracy: {:.4}%".format(ans*100)) 38 | -------------------------------------------------------------------------------- /03__tensorflow_basics/Chapter3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Chapter 3" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "### Our first TensorFlow graph" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": { 21 | "collapsed": false 22 | }, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | "outs = 5\n" 29 | ] 30 | } 31 | ], 32 | "source": [ 33 | "import tensorflow as tf \n", 34 | "a = tf.constant(5) \n", 35 | "b = tf.constant(2)\n", 36 | "c = tf.constant(3)\n", 37 | "d = tf.multiply(a,b) \n", 38 | "e = tf.add(c,b) \n", 39 | "f = tf.subtract(d,e) \n", 40 | "sess = tf.Session() \n", 41 | "outs = sess.run(f) \n", 42 | "sess.close() \n", 43 | "print(\"outs = {}\".format(outs))" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": {}, 49 | "source": [ 50 | "### Constructing and managing our graph " 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 2, 56 | "metadata": { 57 | "collapsed": false 58 | }, 59 | "outputs": [ 60 | { 61 | "name": "stdout", 62 | "output_type": "stream", 63 | "text": [ 64 | "\n", 65 | "\n" 66 | ] 67 | } 68 | ], 69 | "source": [ 70 | "import tensorflow as tf\n", 71 | "print(tf.get_default_graph())\n", 72 | "\n", 73 | "g = tf.Graph()\n", 74 | "print(g)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 3, 80 | "metadata": { 81 | "collapsed": false 82 | }, 83 | "outputs": [ 84 | { 85 | "name": "stdout", 86 | "output_type": "stream", 87 | "text": [ 88 | "False\n", 89 | "True\n" 90 | ] 91 | } 92 | ], 93 | "source": [ 94 | "a = tf.constant(5) \n", 95 | "\n", 96 | "print(a.graph is g)\n", 97 | "print(a.graph is tf.get_default_graph())" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 4, 103 | "metadata": { 104 | "collapsed": false 105 | }, 106 | "outputs": [ 107 | { 108 | "name": "stdout", 109 | "output_type": "stream", 110 | "text": [ 111 | "True\n", 112 | "False\n", 113 | "True\n" 114 | ] 115 | } 116 | ], 117 | "source": [ 118 | "g1 = tf.get_default_graph() \n", 119 | "g2 = tf.Graph() \n", 120 | "\n", 121 | "print(g1 is tf.get_default_graph())\n", 122 | "\n", 123 | "with g2.as_default(): \n", 124 | " print(g1 is tf.get_default_graph())\n", 125 | "\n", 126 | "print(g1 is tf.get_default_graph())" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": {}, 132 | "source": [ 133 | "### Fetches " 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 5, 139 | "metadata": { 140 | "collapsed": false 141 | }, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "outs = [5, 2, 3, 10, 5, 5]\n", 148 | "\n" 149 | ] 150 | } 151 | ], 152 | "source": [ 153 | "with tf.Session() as sess:\n", 154 | " fetches = [a,b,c,d,e,f]\n", 155 | " outs = sess.run(fetches) \n", 156 | "\n", 157 | "print(\"outs = {}\".format(outs))\n", 158 | "print(type(outs[0]))" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "### Nodes are operations, edges are Tensor objects " 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 6, 171 | "metadata": { 172 | "collapsed": false 173 | }, 174 | "outputs": [ 175 | { 176 | "name": "stdout", 177 | "output_type": "stream", 178 | "text": [ 179 | "Tensor(\"Const_4:0\", shape=(), dtype=float32)\n" 180 | ] 181 | } 182 | ], 183 | "source": [ 184 | "c = tf.constant(4.0)\n", 185 | "print(c)" 186 | ] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "metadata": {}, 191 | "source": [ 192 | "### Data types " 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 7, 198 | "metadata": { 199 | "collapsed": false 200 | }, 201 | "outputs": [ 202 | { 203 | "name": "stdout", 204 | "output_type": "stream", 205 | "text": [ 206 | "Tensor(\"Const_5:0\", shape=(), dtype=float64)\n", 207 | "\n" 208 | ] 209 | } 210 | ], 211 | "source": [ 212 | "c = tf.constant(4.0, dtype=tf.float64)\n", 213 | "print(c)\n", 214 | "print(c.dtype)" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": 9, 220 | "metadata": { 221 | "collapsed": false 222 | }, 223 | "outputs": [ 224 | { 225 | "name": "stdout", 226 | "output_type": "stream", 227 | "text": [ 228 | "\n", 229 | "\n" 230 | ] 231 | } 232 | ], 233 | "source": [ 234 | "x = tf.constant([1,2,3],name='x',dtype=tf.float32) \n", 235 | "print(x.dtype)\n", 236 | "x = tf.cast(x,tf.int64)\n", 237 | "print(x.dtype)" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": {}, 243 | "source": [ 244 | "### Tensor arrays and Shapes " 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": 27, 250 | "metadata": { 251 | "collapsed": false 252 | }, 253 | "outputs": [ 254 | { 255 | "name": "stdout", 256 | "output_type": "stream", 257 | "text": [ 258 | "Python List input: (2, 3)\n", 259 | "3d Numpy array input: (2, 2, 3)\n" 260 | ] 261 | } 262 | ], 263 | "source": [ 264 | "import numpy as np \n", 265 | "\n", 266 | "c = tf.constant([[1,2,3],\n", 267 | " [4,5,6]]) \n", 268 | "print(\"Python List input: {}\".format(c.get_shape()))\n", 269 | "\n", 270 | "c = tf.constant(np.array([\n", 271 | " [[1,2,3], \n", 272 | " [4,5,6]], \n", 273 | "\n", 274 | " [[1,1,1], \n", 275 | " [2,2,2]]\n", 276 | " ])) \n", 277 | "\n", 278 | "print(\"3d Numpy array input: {}\".format(c.get_shape()))" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 4, 284 | "metadata": { 285 | "collapsed": false 286 | }, 287 | "outputs": [ 288 | { 289 | "name": "stdout", 290 | "output_type": "stream", 291 | "text": [ 292 | "(1, 50000)\n" 293 | ] 294 | }, 295 | { 296 | "data": { 297 | "image/png": "iVBORw0KGgoAAAANSUhEUgAABDAAAAFOCAYAAABwhO0AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmYHGW5sPH7IawBwiabBISIC4uIigIKJiCgiAgiCAoo\nHo9RlO3ghwICBlwARUEExRxUFhVEFgWFEIKHTUCIx+WwBFHWsEkgC5CwJc/3R1VPOp2eradnuid9\n/66rrumqeuutp6Yyle6n3yUyE0mSJEmSpHa2VKsDkCRJkiRJ6o0JDEmSJEmS1PZMYEiSJEmSpLZn\nAkOSJEmSJLU9ExiSJEmSJKntmcCQJEmSJEltzwSGJGmJFBEbRkSWy3lNqvOhsr6HmlFfL+eqxH5D\nN/snVJUZN9jxDIXBuGftYEm9LkmShtrSrQ5AktQ5IiIrrzMzWhkLQEQcBGwIkJkTWhmLJEmSemYC\nQ5LUyQ4CxpavJ7QuDEmSJPXGBIYkaYmUmQ8BTW3lkZkbNrO+gShbjExocRiSJElDxjEwJEmSJElS\n2zOBIUmSJEmS2p4JDElS24iIcVWzNUwot20QEd+NiGkR8UJEzIqIWyPiCxHRbVfInmZ+iIgbygFF\nx1ZtyzrLhJrjep2FpIz3kIj4dUTcFxHPR8TLEfHv8rxfiYhVGvoFLXqebmchqfk99nWZUP9MXdf0\nzYi4IyKeLq/nyYi4LiIOjohl+xjzWyLiJxHxcES8GBFPRMTkiNhvYL+NrvrXKGPLiPhzH4/5cNXv\n4Lt19r85Io6KiCsj4oGImBsRL5WxTyr/HS7fhNh7nHVmAGW3jYgfRcQ95d/OixHxSET8KiJ268Px\nIyLiwIi4KiIeLY+fV77+34j4eUR8KiJW7NuVSpLUOMfAkCS1rYj4AHARsGrNrm3LZc+I2D0zXxry\n4OooEwl/oP7YG2tSJEzGAl+KiL0y85YhDK8hEXEM8DVguZpda5fLTsB/RcSHMvMfPdRzMPB9YJmq\nzeuUy84R8THg6IHEmpnPRMQ1wIeBt0fEJpl5by+HHVD1+sKamD8FnNfNcZXY3w8cWf477O1cQ6ZM\nKPwE2LfO7vXL5WMR8Xvg45n5XJ06XgNcDbyzTh2jy+VtwP7AbOA3zYlekqT6TGBIktrVlsBRFMmA\nHwO3AS8BWwGfB1YEdga+CpzQz7qPA14DfAPYrNz2kTrlpvWz3uXLeO8G/ge4F3im3L4+sCfwDopk\nxu8iYstysNFmu4v611MtgO8Ary/XZy1WIOJ04Iiq/RcDdwLPAetSXM8OwBuAGyPibZn5ZJ16Pgb8\nsGrTNcBvyzrfDHymD/H21YUUCQwokhNf7a5gRIwCdi9X78rMv9YUWQFI4M/ATcB9wExgFPA6iuTA\nGyl+h9eU93Ox3+NQi4jlgCnANuWmfwG/ovj3+AqwMfBJith3A34TETtn5oKaqv6bhcmLf1IkE/8B\nzKP4HbwJeC+w9aBdjCRJVUxgSJLa1R7AI8BOmXl/1faLI+IS4I8U/48dEhHf7E8rjErLh4g4ompb\nM749vhfYIjP/r5v934iIjwM/B1ahaNnw6SacdxGZOYNevg2PiK+zMHlxA3B2zf49WJi8mALsl5nP\n1FRzZkR8DjiHojXCGcAi3UEiYtWquhMYn5nn1pT5HvB7mpPEuIqiNcAqwCci4rjMzG7K7k2RXILi\nntS6GXh9Zj5Y7+Cy282RFImg1wGHAyc2HnrTnMLC5MV3gGMz89XqAhHxbeBcikTGjsB4ivtY2b8W\nxd8gwFRgXGa+UO9kEfG6pkYvSVI3HANDktTODqhJXgCQmXdQfKMMsBrwriGNqhuZ+XAPyYtKmYtY\n+GF534hYpqfygyEiPkHRCgWKb9Y/mpmv1BQ7qfz5KLBnneQFAJn5YxZ2vdg7ItavKXIQRWsXgPNr\nkxdlHc9RtGZYrBtDf5WJrF+XqxsC2/VQvNJ9ZAHwizp13d1d8qLcvyAzT6NonQFwYL8DbrKIWBf4\nQrl6eWZ+uTZ5AVDe7/8EHig3HVlTZAwLu0L9srvkRVnXw5n58MAilySpdyYwJEnt6i+ZeXMP+/9Q\n9XrTwQ6myW4tf64AbDGUJ46IbSjGRoCiC8fumflsTZm3VsX1o54+vJYqCZkRwPtq9lW3qlhskMyK\nzHyC+q0gGlE9lsUB9QpExGgWDuJ6Q2ZOH8D5Kvfz9eW4Ea30MaAyqOppPRUskxiVROAbImLDqt1z\nq15vhiRJbcAuJJKkdnV7L/sfq3q92mAG0l8RsTXFB+dtKL7JXplFB6+sNppijIWhiGsDiq4lywOv\nAh/LzHrjfGxf9Xq5iNizl6rXq3q9SdX5gmLMD4B/Z+ZdvdRzPXBwL2X64mbgYYpuHftExKGZ+XJN\nmU+w8IucHhMnEbETRdeYdwIbUNzPEd0UXw+Y0WDczVB970b34d5V/+1sAjxUvr4beBx4LfCZ8l7+\nN3BHnbEyJEkaEiYwJEntqrcPgdVjXgx4GstmKKcTPZf+dSUYNUjhLCIiVgKupJg5BOCIzLyum+Ib\nVr3+Wj9PVf2BeBWKwVah6KrSm76U6VVmZkT8Aji2jGc34IqaYpWWGfOAy+rVE8V0t5cAu/Tj9ENy\nP3uwYdXrS/p5bNe9y8z55fgml1G06PiPcpkVEbcBtwDXZuaQJN8kSQITGJKk9jUcv+U9m4XJi5co\npqC8k6K1yAvA/HLfjsCh5evuvslvmohYimKMh7eWm36YmWf3cMgqAzjdslWvV6p6Pbe2YB29dVXp\njwspEhhQJCu6EhgRsQXwlnL1t5k5p5s6LqWYJhaK8TmuAv4KPEFxPZV/o/uxcLrSQb+fvWjWvSMz\nfxcR7wImUCSBlqGY0njXcvlmRNwFHJWZkwZwXkmS+sQEhiRJTVCOH/CZcnU6MDYzH+im7Hr1tg+i\nU1g4tegUitkyevJ81esdM/N/GjxvdT0j+1B+xd6L9E1mTouIqRTT7u4WEatWTXFaPS7GhYsfDRHx\nXhYmL/4G7JyZT3dT9j1NCrtXZTKqJ5XfeQJLD7S7R2b+DfhIRKwMvAd4N8XUqe+mSGhsDlwdEQdm\n5mIDoUqS1EwO4ilJUnPsyMJZG07pLnlRGrJpJyPi08BR5ep9wD71ZqWoUT2+yOgBnH42C1tVbNyH\n8n0p0x+VsS2WA/aBrgTAx8vt/wYmd3PsTlWvv9pd8qLUjPtZGaNj2R5LLZzRpTuVexcsOjbJgGTm\nc5k5KTNPyMxxwLrA6VXn+l5EtLr1iSRpCWcCQ5LUybq+nS4HKRyItate/6uXsu8f4Ln6JCK2B84p\nV58FPlTVCqEnN1a97s/4D4vIzASmlqtrRURvs1nUzmAyUBdRDFYKC1tdjGNhUubiHpI5fbqf5bgn\nOwwgxorKfXltL+W27mV/U+5dbzLzmcw8kqr7C7xhsM4nSRKYwJAkdbbqLg4D7b5QPcbD67srFBF7\nMARTp0bEGOByim/0XwH2zsy+DpI5lWIWCoB9+5B46En14JlHdlcoItYG9h/AeRaTmdUtLLYvZ2Hp\ntftIqU/3k2LWlGZMnXpP+fN15b3rzmG91HMxC1tzfCUimtYtpxsPVb22a7IkaVCZwJAkdbIHq16/\nfYB13Vn1+v9FxGJTu5bTq/50gOfpVUSMohhwsvLB+pD+jGNRtpw4plxdhmKMg3f2cs5NI+JHdXad\nz8IZZT4dEQfVOXYlig/egzGDR6UbSVCMUfLRcn1aZk6tfwiw6P08ISKWqy0QEbtTjC/SDNWDYJ5a\nr0VQRJzEol1bFpOZjwI/KFffAFwVEet0Vz4iloqInSLiuJrt74+Iw8uZWLo7dmNg53L1eXpveSRJ\n0oCYKZckdbLrWfiN9k8i4nTgYRbOFvLPfrRauA34M/AOiqksp0XEORTjTqxAMUZGZaaKX9Dk1gY1\nvg5sWr6+E/h3ROzZyzHTMnNaZSUzryo/MJ8AbAD8KSImUwwCOp1ikMg1gM0oumVsSvF7O7i60syc\nFRFfBH5FkUT4WUTsTTGl62zgTRSJhQ0oWmt8pMFr7s5vKGYQWRk4moVjTPy82yMKV1CMJ7Ee8C7g\nnoj4CfAAxUwcHwR2pxjj4zIWJkYa9VPgy8DqwN7AzeVUsDMofjf7UQxIenH5uifHAFtSdMnZAXgg\nIi6j+Df6NMXvYB2KWWl2Ll9fD3yjqo51gTOAb0fE/wB/orj2uRSJsXcCH2Nhy6UzMnNeg9cuSVKf\nmMCQJHWy3wO3ANtRDCBZO7XoiRRTSPYqMzMi9gP+AKxPMSbACTXFXgS+SDH2xmAmMKq/NX8ni3bj\n6M5i15qZX4uIR4HvUrSOeD89j98xvd7GzLwkIl4DfJ/ivcdu5VLtMooP3k1NYGTmvPLD+0EsTF4k\nvSQwyuP2ppgKdzVgDPDNmmKzKO7juxhgAiMzn46IAym6/SxHMeNH7ewmvwP+g14SGJn5SkR8kOK+\nHUyRQDuARbvP1HqsZj3Ln8vS831P4Ezgaz3FJElSM9iFRJLUsTJzPsU30EdTfDs9k4WtLxqp75/A\n24CTgXspEhbPU7TCOAt4R2YOeheSZsrMcylm2fgSxXgSjwMvlcuTwE3Adyi+7e927IbM/CFFN53z\ngEcpxml4iqJFxycyc2+KsToGQ+1YF7dk5sO9HZSZt1O0UjiLonvEyxStRu4CTgXemplXNyvIsq63\nAj8DHinP9zTwP8CBwIf72sohM1/OzEOBN1N0c/lTWderFK0oHqRIzhwLbJGZn6qp4gKKAUO/StEd\n6Z8UrU3mU/wO/srCf9NHDHS6VkmS+iKKbq6SJEmSJEntyxYYkiRJkiSp7ZnAkCRJkiRJbc8EhiRJ\nkiRJansmMCRJkiRJUtszgSFJkiRJktqeCQxJkiRJktT2TGBIkiRJkqS2ZwJDkiRJkiS1PRMYkiRJ\nkiSp7ZnAkCRJkiRJbc8EhiRJkiRJansmMCRJkiRJUtszgSFJkiRJktqeCQxJkiRJktT2TGBIkiRJ\nkqS2ZwJDkiRJkiS1PRMYkiRJkiSp7ZnAkCRJkiRJbc8EhiRJkiRJansmMCRJkiRJUtszgSFJkiRJ\nktqeCQxJkiRJktT2TGBIkiRJkqS2ZwJDkiRJkiS1PRMYkiRJkiSp7ZnAkCRJkiRJbc8EhiRJkiRJ\nansmMCRJkiRJUtszgSFJkiRJktqeCQxJkiRJktT2TGBI6lVETIiIjIhxrY5FkpohIlYqn2u/a3Us\nw1FEnFb+/rZqdSySehYRN0RE1tm+TEScGBH3R8RL5d/0nq2IUeorExhSN8qHeEbEwxGxfDdlHirL\nLD3U8UkanqqeLX1dDmp1zEOlnT8UVyU8MiLu6+65HxEzIuLFoY5P0vASEePK58kNPZTZsCzz0CCF\n8SXgBOBx4DTgRGDaIJ1Lago/dEm92wA4Ajil1YFIWiKcWGfbEcAqwPeBWTX7/jroEam/3gh8Hjir\n1YFIUh98EhhZZ/uHgOeBnTPz5aENSWqMCQypZzOBBI6OiHMzc0arA5I0vGXmhNptZSuLVYAzMvOh\nIQ5J/fMUxQeBEyLigsyc0+qAJKknmflIN7teCzxj8kLDiV1IpJ7NBb5O8cHia/05MCI+FhE3RcTs\niJgXEf8XEcdExHJ1yj5ULqMi4nvl61ciYkK5v2sMioj4eET8OSLmRsTjZfnlynI7lv0c50TEzIi4\nMCLWqHO+HSJiYkTcU5adFxF3RcTXuusuI6m1ImJqRDwfEStExDci4p8R8XJEnFXu77b7RURsXu47\nq2b7peX2NSPi8PKZ8GJEPBERZ0XEit3EsmFE/DAi/lWWnxERt0fEUTXldomIn0bEtIh4rnxu/b18\nFi5TU3YGRXNmgDurums8X1NupYg4oXymzi3rvTki9uom1uUj4uvlc/WlMubjafxLnFnAt4E1gWP6\nc2BEHBgRfyyfu3Mj4m8R8aXa30VZdkb5XF49In4QEY9ExKsR8f/K/V33OyIOioi/ls/y6RHxrUoX\nl4jYNSJuKf/tPFPej1XqnK/P90pS69S8J9w7Iu4o/16fjYiLI2K9OscsMgZGRJxXrm8EvK7qeftQ\nzXFt+15WncsWGFLvzgYOAT4XEWdm5v29HRAR36J4YzsD+CVF87xdgW8B74+IXepku5cF/gCsDkwG\n5gAP1pQ5tKznN8ANwC7AfwGrR8RvgYuB3wMTgXcDBwCvKY+p9hXgzcCtZfnlgfcAE4BxEbFTZs7v\n7TolDbmlgN8BbwKuBZ4BHm5CvWcD76N4HkwCdga+CLwO2L26YERsV8awCnA98GtgZWAz4HjgO1XF\nTwDWAf4E/BZYCdie4lm4XUR8KDMrb6q/DewJbAv8N0WfbICuZ2VErEnx7NsUuKMstyzFM+6yiDgm\nM0+pKr8UcGV5PfcBZ1K0njgUGMg4G9+l6EJyRET8MDMf7e2AiDizPO9TwAXAixS/29OA90XE7nWe\nuyOBm8pr/D0wD6g91zEU/xf8luL/kF3LbaMi4nbgXOAq4HZgLPBpivu1T009/blXklrvC8CHKZ5x\nNwJbA/sCb42ILTPzpR6O/Q3wEEX3RYAzyp9dXRiHwXtZdarMdHFxqbNQdB2ZXr7eu1y/vKbMQ+X2\npau2bVtuewRYp2r70hRvIhM4tpt6pgAr1ollQrl/NrBJ1fblgLuB+RQfZMZW7VsKuK48bsua+sYA\nUec8Xy/L79vN+ce1+r64uCyJS9UzYMMeykwty9wBrFpn/2nl/q3q7Nu83HdWzfZLy+3/ANat2r5s\n1fk2rdo+EngCWADsUec8o2vWx3RzLaeXde/W12uoifeLNdtHUryBfxV4Y9X28WX564FlqravDUwv\n9/2uj/dopbL8tHL9oHL9gppyM4AXa7btXJa9H1ij5vc8pdx3WJ16kuLDyfI93O+ngdfX/C4eBF4p\n63hX1b4RwB/L+/eGwbxXLi4uPS/AuPJv6IYeymxYlnmoatuEctsc4C015X9Z7vtYzfYbgKxT/0PV\ndVdtb/v3si6du9iFROqDzLwUuA34SPntY0/+o/z5jcx8sqqOVymaRy8A/rObY7+UmS/0UPeZmXlv\nVZ0vAb+ieMD/PjNvrNq3APh5ufrWmut5IDPrfZN2evnz/T3EIKm1jsnM2oE+B+qEzHyispLFt2rn\nl6vvqiq3D8W39Bdl5m9rK8nM6TXrD3Rzvn4/ayJiNLAXxZv9s2vOMxc4luID+n5Vuz5d/jw6M1+p\nKv8UAx+Y+QLgb8ABEfG2XspW/l/4WmY+UxXHyyzsNtPd/wv/lZk9zWpyWmb+q6rOuRStYpYGLsnM\nO6r2zaf4gBPAFtWVNPNeSRoSZ2bm/9Vs++/y57tqC/dT27+XVeeyC4nUd1+i6HJxGrBND+XeXv78\nQ+2OzPxHREwHNoqIVTJzdtXuF4G/9xLD1DrbKs2s/1xn32Plz9HVG6Po13448BGK0fRXpnhDW7FY\n/0lJbeOO3ov0W71nS6WrwmpV2yrPvmv6UmlEjKJoGrwHsDFFK4ZGnzXblMcuU+lTXaMyXscmVdve\nBszNzDvrlL+hH+deTGYuiGLMj8mU3UB6KN7T/wt/i4hngc0jYpnqRAvwbHVyohvN+n+hmfdK0uDr\n63O7EW3/XladywSG1EeZeVtEXArsHRH7ZuavuilaGRztiW72P0ExNeuqFM3oKv7dTauIarPrbHu1\nD/u6BmArB2P7A0V2/i6KrPfTFM2NoRisdLHBmSS1hbmZ+dwg1FuvRUfl+TGiatuq5c/H6EUUAwLf\nAryFoqXCLymaB79C0XXiGPr3rKkM4vaecunOSlXnX46iWXM9T3azvc8y87qImAR8ICJ2y8zfd1N0\nFYpvLJ/qZv8TFH3GR1H8jvoTYzP+X2j2vZLUuwXlz55axFf2Laizr6/P7Ua09XtZdTYTGFL/HEPx\n7dTJEXFFN2UqD991gHrfnK1bU65iqAZH24MieXFeZn66ekdErEs/Z1uRNKR6ek5U3uDW+7991Trb\nGlF5w9yXb+P3o/hAfHZmHlK9IyLeQD9n8GDhM/PrmXlCb4Uz88WIeAlYq5si6/Tz/N35MsUgdN8u\nkxn1zKYYd2Mt6icx1qW4t7XJqaH6f6HZ90pS7yrPtJ5m2HhN+bPZ3QZ70+7vZdXBHAND6ofM/Cfw\nQ4pppw7tpthfyp/jandExMYUTeAeHIQ+7H21cfnz8jr7xg5lIJKaamb5c/06+wYy40a128uffRkN\nvvKsuazOvu6eNZVZOOp9e1g59/Z9OHfFX4CREfHOOvvG9aOebpV90H9GMTNKd33Ce/p/YQuK1hd3\n5eIj+g+VRu6VpIG5D3gJeGMP04RuW/7829CE1KXd38uqg5nAkPrvJIpM+FcpmyrX+Gn587hyyj8A\nImIERT/ppYCfDHaQPXio/DmuemNEjAFOHepgJDVNZWyMz5TThwJdf9vN+gb91xTdGj4eER+u3VkO\ntFnxUPlzXE2ZN1HMeFRPpfvEBrU7MvMh4AqKqZ6PrL7GqrrfGBHVCZyflT9PKbvPVcqtDRzdTQyN\nOB54ATiR+l0tKv8vTIiIrtYwZUynlavt+P9CT/dK0gCUg/NeTNFq7jsRUT3mTOV5elS5et7QRtf2\n72XVwexCIvVTZj5bzo397W723xoR36ZoVnxXOW7GCxTfWG5O0c/4O0MVbx1XAf8EjoyIt1Bk2TcA\nPkQx7/ZiHxwkDQv/QzE42vuB2yPiJopmvntQ/G1/bKAnyMy5EfGxsr7fRsSU8pwrAptRtPSo9J2+\nFDgBOD4itqIYc2dDYHeKqUH3rXOKyoBxp0fEuyiaJ7+cmZXn7WcpWsB9F/jPiLiVYqrQ15bnf3tZ\nf2Ugu59QzJyyE/D3iPgdsALF7+I2mjQwZWY+ERHfLa8Xim9Vq/dPjogfAl8A7omIyygGu9sdeBPF\nQKBnNSOWBjVyryQN3JeAd1LMmLRtRFxHMT3q6yie3SsDp1bPzDEUhsF7WXUwW2BIjTmT7geGIzO/\nAnwcuB/4JHAYxd/bccDOLWwmTDm11Y4Ug7RtVsa2BcW3bAe0Ki5JA1NON/dBiulPNwIOofgb/wJN\n/BY9M28GtgTOpZjF6EsUz7sVKFogVMrNovhG/1KKxMJhFN0sjgHGd1P3VIokxUyKbnpfZ2FSgHIK\n0m3Lcz5HkYg4gqKbwzPlMbdUlZ8PfBj4JjCyjGFXimTBQQ3/Eur7Nj0MupmZXyzP+TDFh5VDgJcp\nPiDsXsbaEo3cK0kDVz7TtqZo1fs8xTPiK8DOwI3AbpnZzNZi/Ymtbd/LqrNF7wPFSpIkSZIktZYt\nMCRJkiRJUtszgSFJkiRJktqeCQxJkiRJktT2TGBIkiRJkqS25zSq/fCBD3wgJ02a1OowJGmoRO9F\neuezU1KH8dkpSf3Xp2enLTD6YcaMGa0OQZKGHZ+dktR/PjslaXEmMCRJkiRJUtszgSFJkiRJktqe\nCQxJkiRJktT2TGBIkiRJkqS2ZwJDkiRJkiS1PRMYUhuZOLFYJEmSJEmLMoEhSZIkSZLangkMSZIk\nSZLU9kxgSJIkSZKktmcCQxpGHCNDkiRJUqcygSFJkiRJktqeCQxJkiRJktT2TGBIkiRJkqS2ZwJD\nanOVcS8c+0Ldueuuu4iIrmXEiBE8+uijrQ5LktrCQQcdtMgzst6y7LLLsuaaa7LVVltx8MEHc8MN\nN5CZrQ5dklRj6VYHIGlxJivUH+edd94i6wsWLOCCCy7gq1/9amsCkqRh5pVXXmHGjBnMmDGDP//5\nz5xzzjmMHTuWn/3sZ2y00UatDk+S2sI999zDpEmTmDJlCg8++CAzZsxg1qxZjBo1ijXWWIMtttiC\nbbbZhr322osxY8YMSgwmMKQWM1mhgXj11Vf5xS9+sdj2888/3wSGJNVYfvnlGTt27GLb582bx/Tp\n03nggQe6tt144428973v5bbbbmP06NFDGaYktZW//OUvHHfccVx99dV19z/77LM8++yz3H///Vx2\n2WV8+ctfZvvtt+fEE09k3LhxTY3FBIYkDWOTJk3iySefBGCzzTZjzpw5PProo9x///388Y9/5D3v\neU+LI5Sk9rH22mszadKkbvc/8MADHHXUUVx++eUATJ8+nSOOOIJLL710qEKUpLZy2mmn8ZWvfIUF\nCxYssn311Vdn9OjRrLHGGsyePZsnn3ySxx9/HIDM5KabbmKHHXbg1FNP5ctf/nLT4nEMDGkYclwM\nVVR3H/nkJz/J/vvvX3efJKl3Y8aM4de//jU77rhj17bf/OY3PPPMMy2MSpJa49BDD+Woo47qSl4s\nu+yyHH744UydOpUZM2bwt7/9jT/84Q/8+c9/5rHHHuOBBx7g+9//Pq9//eu76qgkNZrFBIbUAiYg\n1AzPPvssV111FQBLLbUUn/jEJzjggAO69l9yySXMmzevVeFJ0rC01FJLcdhhh3Wtz58/n6lTp7Yw\nIkkaeueeey5nnXVW1/ob3/hG7r77bs444wze8Y53EBGLHbPRRhtx2GGHce+993LWWWcxcuTIpsdl\nAkOShqmLLrqIl19+GYBx48YxevRoNttsM972trcBMGfOHK644opWhihJw9Kb3/zmRdZtgSGpk/zr\nX/9aJJE7ZswYbrnlFjbeeOM+Hb/MMsvwxS9+kdtuu22R1hjNYAJDkoap6i4iBx54YN3XdiORpP6r\nJIcrVlpppRZFIklD7zvf+U5XK96lllqK8847jzXXXLPf9WyxxRYceuihTY3NBIYkDUN33313V5Pm\nFVZYgY9+9KNd+z7+8Y8zYsQIAK6//nqmT5/ekhglabi6/fbbF1nffPPNWxSJJA2tGTNmcP7553et\n77rrrmy//fYtjGhRJjAkaRiqblmxxx57sPLKK3etr7POOuy0004ALFiwgAsuuGCow5OkYeuZZ57h\nlFNO6VrfZpttGDNmTAsjkqShM3nyZF588cWu9c9+9rMtjGZxTqMqDaHBGLSzXp3jxzf/PGof8+fP\n5+c//3nXenWXkept1157LQDnn38+xx577JDFJ0nDzUsvvcT06dOZMmUKJ598Mg8//DAAK664Imef\nfXaLo5OkoXPzzTd3vY4IdthhhxZGszgTGJI0zEyaNIknn3wSgLXWWotddtllsTJ77rknK664Ii+8\n8AL/+Mfr3/wmAAAgAElEQVQ/uPXWW3n3u9891KFKUlt5+OGH646cX8+4ceM4/fTT2XLLLQc5Kklq\nH9WzLr3xjW9k1KhRLYxmcXYhkaRhprpf4n777cfSSy+ei15xxRXZa6+9utYdzFOS+m677bbjc5/7\nnGNfSOo4Tz/9dNfrDTbYoIWR1GcLDEkaRmbOnMmVV17ZtX7AAQd0W/aAAw7gwgsvBOCSSy7hzDPP\nZPnllx/0GCWpXS2//PKMHTt2se0LFixg9uzZ3HfffcyePZtbbrmFW265hW9+85tccsklbLLJJi2I\nVpKGXvW00ausskoLI6mv5S0wImLjiPhxRPw9IuZHxA11yjwUEVmzPFmn3KYRcX1EzI2IxyPipIgY\nUVMmIuLYiHg0IuZFxE0RYdtAScPCRRddxEsvvQTAm970Jt75znd2W/Z973sf6667LgCzZ8/miiuu\nGJIYJaldrb322kyaNGmxZfLkyfzpT39i5syZXHvttWy66aYA3HXXXYwbN65rTAxJWtJV3mcCLLfc\nci2MpL6WJzCAzYAPAvcB/+ih3C+BbauWD1bvjIjVgClAAnsAJwFfAk6sqedo4HjgVGB34HlgSkSs\nM9ALkaTBVt0VpN7gndVGjBjBJz7xibrHSpIWFxHssssu3HzzzV1Np//9739z8MEHtzgySRoaq666\natfrOXPmtDCS+tohgXFVZq6fmfsAd/dQ7onMvL1q+d+a/Z8HVgD2yszrMvMciuTFkRExCiAilqdI\nYJycmWdl5hRgH4qkxyHNvjBJaqZ77rmHO++8s2v95JNPZtVVV+1x+dGPftRVfsqUKTz22GOtCF2S\nhpXVV1+dI444omv9mmuu4V//+lcLI5KkobH66qt3vX722WdbGEl9LU9gZOaCJlW1K3BtZlaniS6m\nSGpUOju+GxgFXFJ1/heAq8rjJalt1bageOGFF5g9e3aPy9y5c7vKL1iwgAsuuGCIo5ak4Wm77bZb\nZL16akFJWlKNGTOm6/Xdd99NZrYwmsW1PIHRD5+JiJcjYnZEXBoRr6vZ/2ZgWvWGzHwEmFvuq5SZ\nD9xfc+y9VWWkJcrEiQsXDV/z58/n5z//+YDrqZ7BRJLUvepm1EDX9NWStCTbfvvtu17PmjWL++67\nr4XRLG64JDB+C3wBeB9wFMUYGDdHRPWwqKsBs+ocO7PcVynzfGbOr1NmZEQsW3twRIyPiKkRMbV6\nShmpnZm0WPJMnjyZJ554AijGtvj3v/9NZvZpmT59OhEBwH333cftt98+6PH67JQ03M2cOXOR9RVW\nWGHQz+mzU1KrjRs3bpH1iy++uDWBdGNYJDAy8/DMvCgzb87MicD7gdcCnx6Cc0/MzK0yc6s111xz\nsE8nSXVVdx8ZN24c/Xkerbfeerz73e+uW9dg8dkpabi76aabFll/3etqG/82n89OSa227bbbsvnm\nm3etn3vuucybN6+FES1qWCQwamXmXRSzlry9avNMoN5EtauV+yplVqqdWrUsMzczX252rJI0ULNm\nzeK3v/1t1/ree+/d7zr22Wefrte/+tWvePHFF5sSmyQtiWbMmMEZZ5zRtb7sssuy4447tjAiSRo6\nRx11VNfrxx57jGOPPbbhuu6/v3b0hoEZlgmMUpZLxTRqxrGIiPWBkSwcG2MaMALYuKauxcbPkAbC\nLhxqposuuqhrTu4RI0aw11579buOj370o13dSGbNmsVvfvObpsYoSUuCzGTy5Mlsv/32i8zadOih\nhzJq1KgWRiZJQ2f//fdn7NixXevf//73Ofvss/tVx7x58/jkJz/Z7+N6s3RTaxsiEbE5RdKh+uPh\nNcBREbFyZj5XbtsXmAfcWK7fCsyhmDr1G2VdI4Hda+qSmsYkhgaqusvH9ttvz1prrdXvOkaPHs22\n227Lrbfe2lXnfvvt16wQJWlYeOqpp/jABz6w2PYFCxYwZ84cpk2bxuzZsxfZ9973vpeTTjppqEKU\npJYbMWIEF110EVtuuWXXuGuHHHIIf/3rX/n617/OOuus0+2x8+fP54ILLmDChAk88sgjHH744U2N\nreUJjDKB8MFydT1gVERU2kdfDewAHAD8DnicInFxHPAIcF5VVecAhwGXR8SpwBhgAvC9ytSqmfli\nRJwCHB8RMylaXRxJ0RLlB4N0iZLUsHvvvZc77rija726K0h/7bPPPl0JjOuuu47HHnuM9dZbb8Ax\nStJw8eKLL3Lttdf2qeyIESP44he/yLe+9S1Gjhw5yJFJUntZd911ueGGG9htt9148MEHgWI8jF/8\n4hfsvPPO7LTTTmywwQasvvrqzJ49myeeeII//vGPXH311QzmIMQtT2AAawG/rtlWWd8IeLQscwaw\nKvAMMAk4tpKYAMjMmRHxPuAs4CqKGUlOp0hiVDuFImFxDLAGMBXYOTOfat4lSVJzVE97utRSSzXU\nfaRi77335sgjjyQzWbBgARdeeCFHH310M8KUpGEtIlhxxRVZffXVectb3sJ2223HAQccwOjRo1sd\nmiS1zCabbMLtt9/O+PHju8ZjmzdvHldeeSVXXnllr8fvvPPOjB8/vqkxRWb2XkoAbLXVVjl16tRW\nh6FhoF27jTT5+aElXzSjEp+dkjqMz05JS5w//elPfPe732Xy5MmLdbWrtsoqq7DXXnvxmc98hve8\n5z39OUWfnp3t0AJDkiRJkiS1qa233ppLLrmE+fPnM3XqVB588EGefvpp5syZw8orr8xrXvMa3vrW\nt7LJJpuw1FKDN1eICQxJkiRJktSrESNGsPXWW7P11lu35PzDeRpVSZIkSZLUIUxgSJIkSZKktmcX\nEqmDVA8u6oCekiRJkoYTW2BIkiRJkqS2ZwJDkiRJkiS1PRMYUoeaOHHRLiWSJEmS1M4cA0NqEpMB\nkiRJkjR4+t0CIyJMekiSJEmSpCHVSBeSRyPimxGxUdOjkSRJkiRJqqORBMZywDHA/RFxTUTsERGO\npSFJkiRJkgZNI4mHdYGDgNuB9wOXA49ExIkRsX4TY5MkSZIkSQIaSGBk5kuZeUFmbgdsDpwFrAAc\nDzwQEVdGxG4REU2OVZIkSZIkdagBdf3IzHsy83DgtSxslfEh4ErgoYg4LiLWHnCUkiRJkiSpozVl\nRpHMfCkirgBWAjYE1gPWB04CvhoRZwPHZubLzTif1E6cPlWSJEmSBt+AB9+MiK0i4r+Bx4EfAKOA\nHwJbAeOBB4D/Ar430HNJkiRJkqTO1FALjIhYEdgf+BywJRDA34EfAT/PzBfKov8bET8DJgP7AocM\nOGJJkiRJktRx+p3AiIgfA/tRdBd5BbgI+GFm3lqvfGbOj4g/AOMGEKckSZIkSepgjbTA+CzwEPAt\n4CeZOaMPx9xYlpckSZIkSeq3RhIYHwKuyczs6wGZeQtwSwPnkiRJkiRJ6n8CIzOvHoxAJLVG9Swq\n48e3Lg5JkiRJ6km/ZyGJiB0iYmJErNvN/teW+9878PAkSZIkSZIa60JyGLBpZj5Rb2dmPl4mL1YD\nbhpIcJIkSZIkSdBYAuMdwJReytwC7NxA3dKwUN3tQpIkSZI0+PrdhQRYC3i8lzJPluUkSZIkSZIG\nrJEExmxgdC9lRgMvNFC3JEmSJEnSYhpJYNwJ7BkRa9fbGRHrAHuW5SRJkiRJkgaskQTGWcAo4KaI\n+GBELA0QEUtHxG7AjcDKwA+aF6YkSZIkSepk/R7EMzMnRcTJwDHAVcCCiJgBvIYiIRLAyZl5dVMj\nlSRJkiRJHauRFhhk5leBDwGTgecoBux8DrgW2K3cL0mSJEmS1BSNTKMKQNnCwlYWkiRJkiRp0DXU\nAkOSJEmSJGkoNdwCAyAilgNWBUbU25+Zjw+kfkmSJEmSJGgwgRERHwe+AmxOMWhnPdlo/ZJaY+LE\nha/Hj29dHJIkSZJUq98Jhog4EDgfWADcDjwKvNrkuCRJkiRJkro00kLiy8BsYPvMvKvJ8UiSJEmS\nJC2mkUE83wBcYvJCkiRJkiQNlUZaYMwE5jU7EKndVY8PIUmSJEkaWo20wPg9MC4iuhu8U5IkSZIk\nqakaSWAcDawInB0RI5scj6Q2MXGirU4kSZIktY9GupD8kmIQz88B+0fEfcCsOuUyM98/kOAkSZIk\nSZKgsRYYOwFvBwJYGdiq3FZv6VVEbBwRP46Iv0fE/Ii4oU6ZiIhjI+LRiJgXETdFxJZ1ym0aEddH\nxNyIeDwiToqIEY3UJUmSJEmS2kcjCYxl+rgs28f6NgM+CNwH/KObMkcDxwOnArsDzwNTImKdSoGI\nWA2YAiSwB3AS8CXgxP7WJUmSJEmS2ku/ExiZOb+vSx+rvCoz18/MfYC7a3dGxPIUSYeTM/OszJwC\n7EORqDikqujngRWAvTLzusw8hyJ5cWREjOpnXZIkSZIkqY000gKjqTJzQS9F3g2MAi6pOuYF4Cpg\n16pyuwLXZuacqm0XUyQ1xvazLkmSJEmS1EYaSmCU40gcHBG3RMQzEfFi1b4tI+LMiHhDk2J8MzAf\nuL9m+73lvupy06oLZOYjwNyqcn2tS5IkSZIktZF+JzAiYhngWuAsYFPgJYoxLyoeBsYD+zcjQGA1\n4Pk6XVJmAiMjYtmqcvVmQ5lZ7utPXV0iYnxETI2IqU8//XTDFyFJncRnpyT1n89OSepZIy0w/h/F\nDCPfANYEJlbvzMyZwM3AEjGFamZOzMytMnOrNddcs9XhSNKw4LNTkvrPZ6ck9WzpBo45ALgtM78G\nEBFZp8wDFDN8NMNMYKWIGFHTcmI1YG5mvlxVbpU6x69W7utPXZJKE6tSlOPHty4OSZIkSZ2tkQTG\nGIruIz15FlijgbrrmQaMADammGq1onbMi2nUjGMREesDI6vK9bUuqcvEib2XkSRJkiQNrkYSGC9S\nv6VDtQ2oPx5FI24F5lBMd/oNgIgYSdHCo/qj5TXAURGxcmY+V27bF5gH3NjPutThTFpIkiRJUntp\nJIHxV2DniFi2XpeLiBgF7ALc3pfKygTCB8vV9YBREbF3uX51Zs6NiFOA4yNiJkVLiSMpxu/4QVVV\n5wCHAZdHxKkULUUmAN+rTK2amS/2sS5JddidRJIkSVKrNJLAOBe4EDg/Ij5bvaNMXvwUWB34cR/r\nWwv4dc22yvpGwEPAKRRJhmMouqZMBXbOzKcqB2TmzIh4H0X3lqsoWoCcTpHEqNZrXZIkSZIkqb30\nO4GRmb+IiF2AA4E9KQfIjIjbgbcAKwA/zszf9bG+h4DopUwC3yyXnsrdA+zYjLokSZIkSVL7aGQa\nVTLzU8B44J/AOhQJiHcBjwCfy8yDmxahJEmSJEnqeI10IQEgM88Fzo2IlSi6jMzOzNlNi0ySJEmS\nJKnUcAKjIjOfB55vQiySJEmSJEl1NdSFRJIkSZIkaSj1uwVGRPyjj0UzM9/U3/olSZIkSZJqNdKF\nZCSQdbavAqxUvn4KeLXRoCRJkiRJkqr1uwtJZo7OzPXrLKOATYHrgHuBjZsdrCRJUm8mTiyW6nVJ\nkjT8NXUMjMycBnwE2BA4vpl1S5IkSZKkztX0QTwzcy5wLXBAs+uWJEnqia0tJElacg3WLCSvAOsM\nUt2SJEmSJKnDND2BERGrU3Qjmd7suiVJkqBvLS1qx8IYSF2SJKn1GplG9dge6lqfInmxGnDcAOKS\n1OYqb/jHj29tHJI6m88iSZI6RyPTqH6jl/3PA6dk5skN1C21jN/ASdLwNXGiSQxJkpZ0jSQwdu5m\n+wJgJnBPZr7ceEiSJEn9159EtAkPSZKGn34nMDLz+sEIRJIkaSjZ8k6SpOFlsGYhkSRJGlSNJiBM\nXEiSNDw1Mojnaxs9WWY+3uixkiSpszUj8WDyQpKk4auRMTCmA9nAcdng+SRJUgcayhlGqhMbjo0h\nSVJ7aiSh8EtgA2A74Dng78CTwDrAFsDKwM3AI02KUZIkdbiBtpyw5YUkScNfIwmME4HbgB8AX8vM\nWZUdEbEq8HXg48BnMvOfTYlSkiRpiDhDiSRJ7amRQTxPBe7NzMOrkxcAmTkrMw8FppXlpLY3caLf\nzElSu6l+LrfiGe3/C5IktZ9GEhhjgZt6KXNTWU7SEq6SAPLNvqQljc81SZLaSyMJjOWAtXspsw6w\nfAN1S5KkDteOiYN2jEmSpE7TSALjb8B+EbFFvZ0RsSWwL/CXgQQmSZI6g8kBSZLUF40M4nkS8Hvg\njoi4gKK7yFMUrTLGAgeW9Z7UrCAlSZJaweSKJEnto98JjMy8NiL2B84B/hP4TNXuAGYDn8/M65oT\noiRJkiRJ6nSNtMAgM38VEVcDHwHeDqxCkbj4X+CKzHyueSFKkqQlXaWlg9OXSpKk7jSUwAAokxQX\nlIskSdISbeJEEyySJLVSwwmMiohYGVgpM59oQjzSkLBPsySpESYxJElqnUZmISEiRkbEqRExHZgF\nPFq1710RcWU5G4kkSZIkSdKA9bsFRtni4mZgC+AuYA7wpqoidwM7AtOAvzYhRkmS1CFsISdJkrrT\nSBeS4yiSF/+ZmT+NiAnA8ZWdmflCRNwIvK85IUrN4ZtiSZIkSRq+GulC8lFgcmb+tFzPOmUeAkY3\nGpQkSZIkSVK1RhIYo4G/9VLmeYqpVSVJkrpl6zhJktRXjSQwngfW7KXMRsCMBuqWJEmSJElaTCMJ\njDuBD0XESvV2RsQ6wK7ArQMJTJIkSZIkqaKRBMaZwGuA30XEG6p3lOu/AlYoy0mSJNU1XLuPTJw4\nfGOXJGk46/csJJl5TUR8g2I2kmnASwAR8SRF15IAvpqZtzQzUEmSJEmS1LkamUaVzDwhIm4GDgO2\nAZYrl8nA9zLzuuaFKGm46O4byfHjhzYOSe1p4sTiebAktV6oXJMkSRp8DSUwAMokhYkKSZLUkZak\nRIwkScNBv8fAiIjJETFhEGKRBoV9lSVJktRMvreUWqORQTy3A5ZtdiCSJEnDkYlySZKGRiMJjH8C\n6zc7EEmSpOHMRIakJZnPN7WDRhIYPwE+GBGjmx1MdyLioIjIOsvnq8pERBwbEY9GxLyIuCkitqxT\n16YRcX1EzI2IxyPipIgYMVTXIklSJ/MNsCQt5DNR6p9GEhiXAX8E/hgRn4+Id0TEehHx2tqlybEC\n7AhsW7VcXrXvaOB44FRgd+B5YEpErFMpEBGrAVOABPYATgK+BJw4CLFKkiRJGsaGomVVs+vva8xD\n2WrMRI2apZFZSB6hSAAEcHYP5bLB+ntyZ2Y+X7sxIpanSGCcnJlnldtuAx4CDgGOK4t+HlgB2Csz\n5wDXRcQoYEJEfLvcJkmSmsApRiVJUjM10gLjl+Xyi6rX9ZaLmhRjX7wbGAVcUtmQmS8AVwG7VpXb\nFbi2JlFxMUVSY+wQxClJUkepfMPXSd++ddK1SmpMo8/F6uOqj++urv6coy8x9fRMrxdXT/X0tD4U\nfFYPT/1OYGTmAZl5YF+WQYj3XxHxakTcFxGfq9r+ZmA+cH9N+XvLfdXlptVczyPA3Jpykpqo0z68\nSPJvXlLnacaH8tqkxECepa06tnJ8vd9HM5I2fUmMd3cvfE86/DXSAqMVnqAY3+JAivEtbgfOiYj/\nKvevBjyfmfNrjpsJjIyIZavKzapT/8xy32IiYnxETI2IqU8//fQAL0OSOoPPTknqP5+dS7ZGWyh0\nlwzobl9f6+2u/nrJgkY1u6VGvf0mKzpLn8aoiIhPAn/NzL8Pcjx1Zea1wLVVm64px704LiK+P8jn\nnghMBNhqq61yMM+l5vFhJbWWz05J6j+fne2j3ofiypg+fWlp0ZfuHX09d6P1DPS4vtTbzHGOBmvc\npL4mURyzaXjo6yCb5wETgK4ERkR8CvhUZu7Y/LD65FLgY8CGFC0oVoqIETWtMFYD5mbmy+X6TGCV\nOnWtVu6TJEkaMN8US0uewRy3oVl1NaNVw2Cfv6fyPT07e+sy4rO2MwykC8mGtHbgy6z6OQ0YAWxc\nU6Z2zItp1Ix1ERHrAyNrykmSJEnqQO2YmGi3c/XnnO3WckTDW7OnOR1KewMzgIcpxsiYA+wDfAMg\nIkZSjJdR/U//GuCoiFg5M58rt+0LzANuHKK4JUlaYvmGU1InW1Kfgf0ZMLNVmjHwqK042t+wSGBE\nxGXAHRRdWEZQJB32BQ7LzAXAixFxCnB8RMykaE1xJEULkx9UVXUOcBhweUScCoyh6BrzvZqpVSVJ\nkpqi9k21b5AlSWrMsEhgAPcB/wGsDwRwD/DJzLywqswpFAmLY4A1gKnAzpn5VKVAZs6MiPcBZwFX\nUcxIcjpFEkOSJEmSBk27tFboRP7ulwz9SWC0bCTkzDwWOLaXMgl8s1x6KncP0KqBR6WO5qB2kiRJ\nkhrVnwTGhIiYULsxIubXKQtFTmG4tPCQJEkD5LdbkiRpMPVnFpLo5zKQGU4kSVIbM1khSZKGWp9a\nSGSmyQi1Pd9MS5IkSdKSyy4ekiRpQEwgS5KkoWDLCkktMXHiwkXS8OLf7cD5O5Qkqf9MYEiSJA0h\nkxeSJDXGLiSSJKkhfhCXJElDyQSGhjXfPEuShquJE2H8+FZHIUnS8GEXEkmSJEmS1PZsgaFhyZYX\nktQaPn8lSVKr2AJDkiRJkiS1PRMYkiRJLWbLFkmSemcXEkmSpBYxcSFJUt/ZAkNSy02c6Jt4SZIk\nST2zBYYkSeqVSUZJktRqtsCQJEmSJEltzxYYkiSpW7a8kCRJ7cIEhiRJUpuoThiNH9+6OCRJakcm\nMCRJ0mJseTH0/J1LktQzExgaNnxjt+Tzm0ep9SZO9O+vnXg/JElayEE8JUnSIkwYtwfvgyRJi7IF\nhqS2ZGsMSZIkSdVMYKit+e2TJKnTVf4vNJkrSep0diGR1PYmTjSZJUmSJHU6ExiSJAkwUdjuvD+S\npE5nAkOSpA5nK6fhw/skSepkJjAkSZIkSVLbM4EhSVIH8xt9SZI0XJjAkDRsVJq5+4FLGjj/joYv\n750kqVOZwFBb8kOqJA0+n7PDn/dQktRJTGBIkiQNYyYxJEmdYulWByBV+AZMkgbPxIkwfrzP2iWF\n91GS1IlsgSFJkiRJktqeCQxJw5IDekr959/Lkst7K0nqBHYhUUtU3miNH9/aOLRk8N+TVJ9/G52l\nOonhPZckLYlMYKil/MZIkgafz1pJkrQkMIGhIeMbaA02v32UJEmSllyOgSFJkiRJktqeCQxJkiRJ\nktT27EIiaYnUXZclu5ZIkiRJw5MtMCRJkiRJUtuzBYaazoEU1c6cVlKSJEkankxgaFA584gkSZIk\nqRk6MoEREZsCPwC2BWYB5wInZub8lgYmacjYUkiSJEkaXjougRERqwFTgHuAPYDXA9+lGA/kuBaG\nNmzYqkKdwASHJEmS1F46LoEBfB5YAdgrM+cA10XEKGBCRHy73KY6TFxoSeW/bUmSJKn9dWICY1fg\n2ppExcXAqcBY4KqWRNWm/GAn2RpDkiRJagedmMB4M/CH6g2Z+UhEzC33DasERm8zKpiAkJqr3t9U\n9d9fb/sbOZdJE0mSJAkiM1sdw5CKiFeAozLzjJrt04ELMvPYmu3jgcrHhzcB9w1JoPW9BpjRwvO3\nUidfO3T29XfytUNrr39GZn6gkQN9draNTr526Ozr99pbx2fn8NfJ1w6dff1ee+v06dlpAmPh9roJ\njHYSEVMzc6tWx9EKnXzt0NnX38nXDl5/M3Ty77CTrx06+/q99s689mbq5N9jJ187dPb1e+3tf+1L\ntTqAFpgJrFJn+2rlPkmSJEmS1GY6MYExjWKsiy4RsT4wstwnSZIkSZLaTCcmMK4B3h8RK1dt2xeY\nB9zYmpD6rJOH5Ozka4fOvv5Ovnbw+puhk3+HnXzt0NnX77VroDr599jJ1w6dff1ee5vrxDEwVgPu\nAe6imDp1DPA94IzMPK6VsUmSpP/f3r3HWlaWdxz//gS0RQFHFFSKZYi2SKupeAsabqVR0HREEQwp\nNUUhqFGIoigGZaRRg1o7BdoYZpDpHyXG2nBLAJGqNFOj0xKaYJR4qxoRlMvIRRB0ePxjrTNzzmbP\nmbPJ2Xvtvc73k0z2nHe9++znnGH/3p2Hd60lSZI03IprYAAkORi4GDgU+BWwAVhbVVs7LUySJEmS\nJA21IhsYkiRJkiRptqzEa2D0SpIzk1SSL3VdyyQk2TPJx5JsTnJfkjuTXJHkT7qubbklOTjJfyZ5\nKMnPk5yfZJeu6xq3JCckuTrJ7UkeTHJzkpO6rqsrSfZrfw+V5Gld19MXZqfZ2Tdm53bm5viYnWZn\n35id281KdtrAmGFJ9gHWAnd1XMokPQ84Dfgy8GbgdOA5wLfau8n0QnutlhuBAt4AnA+cBXysy7om\n5H3Ag8B7gTXA14DLk7yn06q682ma34eWidlpdvaU2bmduTkGZqfZ2VNm53YzkZ2eQjLDklwKPBnY\nH7i7qt7ccUljl+SpwGNV9fC8sWcAPwU+XVW9CNok5wBnA39cVfe3Y2fTfHB49txYHyV5ZlXdPTB2\nOXBoVa3uqKxOJDkcuBL4BM2iskdVTf3CMu3Mzm1jZmePmJ0Nc3N8zM5tY2Znj5idjVnKTndgzKgk\nrwBOBD7UdS2TVFW/nr+ItGP3Aj8BnttNVWNxLPDlgQXjC8AfAkd0U9JkDC4irVvo17/vTrXbNi+i\n+b8gw34negLMzgVjZmePmJ3m5jiZnQvGzM4eMTtnLzttYMygJKH5j+xTVXV71/V0LcmzgOcD3+u6\nlmV0EHDb/IGq+inwUHtspTmUfv37LsU7gKcA/9x1IX1hdi5kdq4IKy07zc0xMDsXMjtXBLNziu3a\ndQF6Qk4B9gU+03UhU+IfaM7X2thxHctpFc0tfgdtaY+tGEmOBo4D3tZ1LZOSZG/g74GTq+q3zWdH\nLQOzcyGzs8dWWnaam2Nldi5kdvaY2Tn92WkDYwok2YvmgkCLqqrb2rmfBN4zuKVtVo3y8w957juB\nk4Hjq+qeMZSnDiU5ALgcuKqqNnZazGR9HPhmVV3bdSHTzOw0OzXcCs1Oc3OJzE6zU8OZnbPBBsZ0\nOFtsGsIAAAc9SURBVAFYv4R5AT5Mc+GgG5I8vR3fFdit/fqBqto6njLHZpSff/sXyRqaLY0frKor\nxlFYh7YAew0ZX9Ue6732IlnX0Zxn+jcdlzMxSf6Mput/+Lz3+O7t415JtvblQ+QyMDvNzkFm5wrM\nTnNzZGan2TnI7DQ7ZyY7vQvJjElyJc3tjXbksKraNKl6upLk1cBXgM9X1bu7rme5Jfkv4PaqOmne\n2P40HyLWVNU1nRU3AUl2p7md1740V4H+ZcclTUyS44DFPhhdWlWnTqqevjA7G2an2dlH5ub4mJ0N\ns9Ps7KNZzU53YMyec4F1A2PrgPuA84BbJ17RhLXdwmuA64EzOi5nXK4DPpBkj6p6oB17C/AwcFN3\nZY1fkl2BfwdeALxqpSwi82wCjhoYOwb4IPA64EcTr6gfzE6z0+zsL3NzfMxOs9Ps7K+ZzE53YPRA\nkq+zcu7HvQ9wM1DAW4HfzDt8f1V9p5PCllmSVcB3gG8DFwAHAp8F1lXVuV3WNm5JLgFOA84ENg8c\nvqWqHpl8Vd1K8nfAZUzxPblnkdm5jdnZA2bnQubm+Jid25idPWB2LjQL2ekODM2ag4E/av/+tYFj\nNwFHTrSaMamqLe1VkC+m6fr/CvhHYG2XdU3Ia9rHfxpybDXw48mVIvWG2dl/Zqe0/MzO/jM7Z4w7\nMCRJkiRJ0tR7UtcFSJIkSZIk7YwNDEmSJEmSNPVsYEiSJEmSpKlnA0OSJEmSJE09GxiSJEmSJGnq\n2cCQJEmSJElTzwaGNAWSPD9JJdnQdS2SNCvMTkkandmpWWYDQ1pEkn9rA/5dS5h7Qzv3jZOoTZKm\nldkpSaMzO6Wds4EhLW59+3jqYpOSHAD8FXAHcM14S5KkqWd2StLozE5pJ2xgSIuoqq8D3wNekuSQ\nRaa+HQhwWVX9bhK1SdK0MjslaXRmp7RzNjCknZvrhp827GCSXYBTgAI2tGP7JTkvyTeS3Jnk0SS3\nt1sDD1rqCyfZlGTowpTk1Hbr4MlDju2f5F+S/CjJI0nuSXJVkpcOmbtnW+u3kzzQ/vlBki8keclS\na5WkAWanJI3O7JQWYQND2rl/BR4FTkqy+5DjxwL7ATdW1f+3Y0cBZwP3Av8BrAM2AycCm5P8+biK\nTfIy4P+AdwC3ARfSbC88EvhGktfMmxvgBmAtcB/Novk54H/a+a8cV52Ses/slKTRmZ3SInbtugBp\n2lXVXUmupFkETgQ2DkyZ65BfMm/sK8C+VfXg/IltZ3kT8Engr5e71iS7AV8EdgcOr6pN846dS7NA\nfD7JgVX1KPAXNIvFl6rqhIHvtQuw53LXKGllMDslaXRmp7Q4d2BISzO3SCy4qFKS5wCvA34JXDU3\nXlW/GFxE2vFbgJuAo9ugXm5rgNXAuvmLSPvaPwM+Q9O1P3LgeQ8PqXVrVW0ZQ42SVg6zU5JGZ3ZK\nO+AODGlpvgr8EHh1khdW1Xfb8VNo3kcbq+q385+QZA1wOvBSYG8e/357BnDXMtd5aPu4OsnaIcf/\ntH18Ic0WvlvbP3+bZDVwNU2n/n8Hfx5JegLMTkkandkp7YANDGkJqqqSbKDZgncqcFZ7Ht/baS6i\ntH7+/CRn0XSd7wVuBH5C020u4E3Ai4CnjKHUvdvHt+xk3tMAqup3SY4CPgocD3yqPX5/ko3Ah6vq\n12OoU9IKYHZK0ujMTmnHbGBIS3cZcD7w1iTnAIcBBwJfraofzE1qzwc8D/g5cEhV/WL+N0ly2Aiv\n+VjzlDypqh4bOPb0IfPvax9fX1XXLuUFquoe4EzgzCQvoNnmdzpwBs25iKeMUK8kDTI7JWl0Zqc0\nhNfAkJaoXRCuBp4JHMf28xIvGZi6L7AHsGnIIrInMMotorbQvE/3G3LsZUPGvtk+jrJYbVNV36+q\n9cARNJ37457I95GkOWanJI3O7JSGs4EhjWZuy95ZwBuBu4ErBubcATwCvDzJU+cGkzwZuAhYNcLr\nbW4fF9wLvL0l1QmPn84VwI+BM5K8dtg3TPKqJH/Q/v3AJAcMmbYK2A14aIRaJWlHzE5JGp3ZKQ3w\nFBJpNDfQBPUr2q8vbm8LtU1VbU1yEfB+4NYkV9Ocd/iXwF40V4M+YomvdynNovWR9lZY3wUOAo6h\nWTSOH3jtR5K8CbgeuD7Jf9Pcm/th4HnAy2muFv0s4DfAIcAXk2xuv/cdwD7AG2jy4YIl1ilJizE7\nJWl0Zqc0wB0Y0giqqoAN84bW72DqOcDZNB3x02m2xH2LJsh/NsLr3Umz6FxPc47gu2i2CR7djg17\nzi3Ai2kujLQKeBvwTppF42bgZJotgrQ1XUBzzuOxNIvWa2k68MdU1YVLrVWSdsTslKTRmZ3S46V5\nX0iSJEmSJE0vd2BIkiRJkqSpZwNDkiRJkiRNPRsYkiRJkiRp6tnAkCRJkiRJU88GhiRJkiRJmno2\nMCRJkiRJ0tSzgSFJkiRJkqaeDQxJkiRJkjT1bGBIkiRJkqSp93vs4wiaAwr3QgAAAABJRU5ErkJg\ngg==\n", 298 | "text/plain": [ 299 | "" 300 | ] 301 | }, 302 | "metadata": {}, 303 | "output_type": "display_data" 304 | } 305 | ], 306 | "source": [ 307 | "import matplotlib.pyplot as plt \n", 308 | "% matplotlib inline\n", 309 | "\n", 310 | "sess = tf.InteractiveSession()\n", 311 | "\n", 312 | "# === Noramal and Truncated normal distributions ===\n", 313 | "mean = 0\n", 314 | "std = 1\n", 315 | "x_normal = tf.random_normal((1,50000),mean,std).eval()\n", 316 | "x_truncated = tf.truncated_normal((1,50000),mean,std).eval()\n", 317 | "\n", 318 | "# === Uniform distribution\n", 319 | "minval = -2 \n", 320 | "maxval = 2\n", 321 | "x_uniform = tf.random_uniform((1,50000),minval,maxval).eval()\n", 322 | "\n", 323 | "sess.close()\n", 324 | "\n", 325 | "\n", 326 | "def simpleaxis(ax):\n", 327 | " ax.spines['top'].set_visible(False)\n", 328 | " ax.spines['right'].set_visible(False)\n", 329 | " ax.get_xaxis().tick_bottom()\n", 330 | " ax.get_yaxis().tick_left()\n", 331 | "# ax.set_ylim([-1.1,1.1])\n", 332 | " ax.tick_params(axis='both', which='major', labelsize=15)\n", 333 | " \n", 334 | "def get_axis_limits(ax, scale=.8):\n", 335 | " return ax.get_xlim()[1]*scale, ax.get_ylim()[1]*scale\n", 336 | "\n", 337 | "f,axarr = plt.subplots(1,3,figsize=[15,4],sharey=True)\n", 338 | "titles = ['Normal','Truncated Normal','Uniform']\n", 339 | "\n", 340 | "print(x_normal.shape)\n", 341 | "for i,x in enumerate([x_normal,x_truncated,x_uniform]):\n", 342 | " ax = axarr[i]\n", 343 | " ax.hist(x[0],bins=100,color='b',alpha=0.4)\n", 344 | " ax.set_title(titles[i],fontsize=20)\n", 345 | " ax.set_xlabel('Values',fontsize=20)\n", 346 | " ax.set_xlim([-5,5])\n", 347 | " ax.set_ylim([0,1800])\n", 348 | " \n", 349 | " simpleaxis(ax)\n", 350 | " \n", 351 | " \n", 352 | "axarr[0].set_ylabel('Frequency',fontsize=20)\n", 353 | "plt.suptitle('Initialized values',fontsize=30, y=1.15)\n", 354 | "\n", 355 | "\n", 356 | "for ax,letter in zip(axarr,['A','B','C']):\n", 357 | " simpleaxis(ax)\n", 358 | " ax.annotate(letter, xy=get_axis_limits(ax),fontsize=35)\n", 359 | "\n", 360 | "plt.tight_layout()\n", 361 | "\n", 362 | "plt.savefig('histograms.png', bbox_inches='tight', format='png', dpi=200, pad_inches=0,transparent=True)\n", 363 | "\n", 364 | "plt.show()" 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": 12, 370 | "metadata": { 371 | "collapsed": false 372 | }, 373 | "outputs": [ 374 | { 375 | "name": "stdout", 376 | "output_type": "stream", 377 | "text": [ 378 | "The content of 'c':\n", 379 | " [ 0. 1. 2. 3. 4.]\n", 380 | "\n" 381 | ] 382 | } 383 | ], 384 | "source": [ 385 | "sess = tf.InteractiveSession()\n", 386 | "c = tf.linspace(0.0, 4.0, 5) \n", 387 | "print(\"The content of 'c':\\n {}\\n\".format(c.eval()))\n", 388 | "sess.close()" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "### Matrix multiplication" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": 5, 401 | "metadata": { 402 | "collapsed": false 403 | }, 404 | "outputs": [ 405 | { 406 | "name": "stdout", 407 | "output_type": "stream", 408 | "text": [ 409 | "()\n", 410 | "(3,)\n", 411 | "(3, 1)\n", 412 | "matmul result:\n", 413 | " [[ 4]\n", 414 | " [10]]\n" 415 | ] 416 | } 417 | ], 418 | "source": [ 419 | "A = tf.constant([ [1,2,3],\n", 420 | " [4,5,6] ])\n", 421 | "print(a.get_shape())\n", 422 | "\n", 423 | "x = tf.constant([1,0,1])\n", 424 | "print(x.get_shape())\n", 425 | "\n", 426 | "x = tf.expand_dims(x,1)\n", 427 | "print(x.get_shape())\n", 428 | "\n", 429 | "b = tf.matmul(A,x)\n", 430 | "\n", 431 | "sess = tf.InteractiveSession()\n", 432 | "print('matmul result:\\n {}'.format(b.eval()))\n", 433 | "sess.close()" 434 | ] 435 | }, 436 | { 437 | "cell_type": "markdown", 438 | "metadata": {}, 439 | "source": [ 440 | "### Names " 441 | ] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": 18, 446 | "metadata": { 447 | "collapsed": false 448 | }, 449 | "outputs": [ 450 | { 451 | "name": "stdout", 452 | "output_type": "stream", 453 | "text": [ 454 | "c:0\n", 455 | "c_1:0\n" 456 | ] 457 | } 458 | ], 459 | "source": [ 460 | "with tf.Graph().as_default():\n", 461 | " c1 = tf.constant(4,dtype=tf.float64,name='c') \n", 462 | " c2 = tf.constant(4,dtype=tf.int32,name='c') \n", 463 | "print(c1.name)\n", 464 | "print(c2.name)" 465 | ] 466 | }, 467 | { 468 | "cell_type": "markdown", 469 | "metadata": {}, 470 | "source": [ 471 | "### Name scopes" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": 19, 477 | "metadata": { 478 | "collapsed": false 479 | }, 480 | "outputs": [ 481 | { 482 | "name": "stdout", 483 | "output_type": "stream", 484 | "text": [ 485 | "c:0\n", 486 | "prefix_name/c:0\n", 487 | "prefix_name/c_1:0\n" 488 | ] 489 | } 490 | ], 491 | "source": [ 492 | "with tf.Graph().as_default():\n", 493 | " c1 = tf.constant(4,dtype=tf.float64,name='c') \n", 494 | " with tf.name_scope(\"prefix_name\"):\n", 495 | " c2 = tf.constant(4,dtype=tf.int32,name='c') \n", 496 | " c3 = tf.constant(4,dtype=tf.float64,name='c')\n", 497 | "\n", 498 | "print(c1.name)\n", 499 | "print(c2.name)\n", 500 | "print(c3.name)" 501 | ] 502 | }, 503 | { 504 | "cell_type": "markdown", 505 | "metadata": {}, 506 | "source": [ 507 | "### Variables\n" 508 | ] 509 | }, 510 | { 511 | "cell_type": "code", 512 | "execution_count": 20, 513 | "metadata": { 514 | "collapsed": false 515 | }, 516 | "outputs": [ 517 | { 518 | "name": "stdout", 519 | "output_type": "stream", 520 | "text": [ 521 | "pre run: \n", 522 | "\n", 523 | "\n", 524 | "post run: \n", 525 | "[[-0.69993478 1.55560553 0.55664998 -1.7586478 3.34199762]]\n" 526 | ] 527 | } 528 | ], 529 | "source": [ 530 | "init_val = tf.random_normal((1,5),0,1)\n", 531 | "var = tf.Variable(init_val, name='var') \n", 532 | "print(\"pre run: \\n{}\".format(var))\n", 533 | "\n", 534 | "init = tf.global_variables_initializer()\n", 535 | "with tf.Session() as sess:\n", 536 | " sess.run(init)\n", 537 | " post_var = sess.run(var)\n", 538 | "\n", 539 | "print(\"\\npost run: \\n{}\".format(post_var))" 540 | ] 541 | }, 542 | { 543 | "cell_type": "markdown", 544 | "metadata": {}, 545 | "source": [ 546 | "### New variables are created each time" 547 | ] 548 | }, 549 | { 550 | "cell_type": "code", 551 | "execution_count": 21, 552 | "metadata": { 553 | "collapsed": false 554 | }, 555 | "outputs": [ 556 | { 557 | "name": "stdout", 558 | "output_type": "stream", 559 | "text": [ 560 | "pre run: \n", 561 | "\n", 562 | "\n", 563 | "post run: \n", 564 | "[[-0.97382897 -0.38875952 -1.22835159 -0.75114495 0.50170541]]\n" 565 | ] 566 | } 567 | ], 568 | "source": [ 569 | "init_val = tf.random_normal((1,5),0,1)\n", 570 | "var = tf.Variable(init_val, name='var') \n", 571 | "print(\"pre run: \\n{}\".format(var))\n", 572 | "\n", 573 | "init = tf.global_variables_initializer()\n", 574 | "with tf.Session() as sess:\n", 575 | " sess.run(init)\n", 576 | " post_var = sess.run(var)\n", 577 | "\n", 578 | "print(\"\\npost run: \\n{}\".format(post_var))" 579 | ] 580 | }, 581 | { 582 | "cell_type": "markdown", 583 | "metadata": {}, 584 | "source": [ 585 | "### Placeholders" 586 | ] 587 | }, 588 | { 589 | "cell_type": "code", 590 | "execution_count": 22, 591 | "metadata": { 592 | "collapsed": false 593 | }, 594 | "outputs": [ 595 | { 596 | "name": "stdout", 597 | "output_type": "stream", 598 | "text": [ 599 | "outs = 6.63996839523\n" 600 | ] 601 | } 602 | ], 603 | "source": [ 604 | "x_data = np.random.randn(5,10) \n", 605 | "w_data = np.random.randn(10,1) \n", 606 | "\n", 607 | "with tf.Graph().as_default():\n", 608 | " x = tf.placeholder(tf.float32,shape=(5,10))\n", 609 | " w = tf.placeholder(tf.float32,shape=(10,1))\n", 610 | " b = tf.fill((5,1),-1.) \n", 611 | " xw = tf.matmul(x,w)\n", 612 | "\n", 613 | " xwb = xw + b\n", 614 | " s = tf.reduce_max(xwb)\n", 615 | " with tf.Session() as sess:\n", 616 | " outs = sess.run(s,feed_dict={x: x_data,w: w_data}) \n", 617 | "\n", 618 | "print(\"outs = {}\".format(outs))" 619 | ] 620 | }, 621 | { 622 | "cell_type": "markdown", 623 | "metadata": {}, 624 | "source": [ 625 | "### Example 1: Linear Regression" 626 | ] 627 | }, 628 | { 629 | "cell_type": "code", 630 | "execution_count": 23, 631 | "metadata": { 632 | "collapsed": true 633 | }, 634 | "outputs": [], 635 | "source": [ 636 | "# === Create data and simulate results =====\n", 637 | "x_data = np.random.randn(2000,3)\n", 638 | "w_real = [0.3,0.5,0.1]\n", 639 | "b_real = -0.2\n", 640 | "\n", 641 | "noise = np.random.randn(1,2000)*0.1\n", 642 | "y_data = np.matmul(w_real,x_data.T) + b_real + noise" 643 | ] 644 | }, 645 | { 646 | "cell_type": "code", 647 | "execution_count": 24, 648 | "metadata": { 649 | "collapsed": false 650 | }, 651 | "outputs": [ 652 | { 653 | "name": "stdout", 654 | "output_type": "stream", 655 | "text": [ 656 | "(0, [array([[ 0.30298612, 0.48779276, 0.0716765 ]], dtype=float32), -0.18869072])\n", 657 | "(5, [array([[ 0.3005667 , 0.50066561, 0.09561971]], dtype=float32), -0.19749904])\n", 658 | "(10, [array([[ 0.3005667 , 0.50066561, 0.0956197 ]], dtype=float32), -0.19749907])\n" 659 | ] 660 | } 661 | ], 662 | "source": [ 663 | "NUM_STEPS = 10\n", 664 | "\n", 665 | "g = tf.Graph()\n", 666 | "wb_ = []\n", 667 | "with g.as_default():\n", 668 | " x = tf.placeholder(tf.float32,shape=[None,3])\n", 669 | " y_true = tf.placeholder(tf.float32,shape=None)\n", 670 | " \n", 671 | " with tf.name_scope('inference') as scope:\n", 672 | " w = tf.Variable([[0,0,0]],dtype=tf.float32,name='weights')\n", 673 | " b = tf.Variable(0,dtype=tf.float32,name='bias')\n", 674 | " y_pred = tf.matmul(w,tf.transpose(x)) + b\n", 675 | "\n", 676 | " with tf.name_scope('loss') as scope:\n", 677 | " loss = tf.reduce_mean(tf.square(y_true-y_pred))\n", 678 | " \n", 679 | " with tf.name_scope('train') as scope:\n", 680 | " learning_rate = 0.5\n", 681 | " optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n", 682 | " train = optimizer.minimize(loss)\n", 683 | "\n", 684 | " # Before starting, initialize the variables. We will 'run' this first.\n", 685 | " init = tf.global_variables_initializer()\n", 686 | " with tf.Session() as sess:\n", 687 | " sess.run(init) \n", 688 | " for step in range(NUM_STEPS):\n", 689 | " sess.run(train,{x: x_data, y_true: y_data})\n", 690 | " if (step % 5 == 0):\n", 691 | " print(step, sess.run([w,b])) \n", 692 | " wb_.append(sess.run([w,b]))\n", 693 | " \n", 694 | " print(10, sess.run([w,b]))" 695 | ] 696 | }, 697 | { 698 | "cell_type": "markdown", 699 | "metadata": {}, 700 | "source": [ 701 | "### Example 2: Logistic Regression\n" 702 | ] 703 | }, 704 | { 705 | "cell_type": "code", 706 | "execution_count": 25, 707 | "metadata": { 708 | "collapsed": true 709 | }, 710 | "outputs": [], 711 | "source": [ 712 | "N = 20000\n", 713 | "\n", 714 | "def sigmoid(x):\n", 715 | " return 1 / (1 + np.exp(-x))\n", 716 | "# === Create data and simulate results =====\n", 717 | "x_data = np.random.randn(N,3)\n", 718 | "w_real = [0.3,0.5,0.1]\n", 719 | "b_real = -0.2\n", 720 | "wxb = np.matmul(w_real,x_data.T) + b_real\n", 721 | "\n", 722 | "y_data_pre_noise = sigmoid(wxb)\n", 723 | "y_data = np.random.binomial(1,y_data_pre_noise)" 724 | ] 725 | }, 726 | { 727 | "cell_type": "code", 728 | "execution_count": 26, 729 | "metadata": { 730 | "collapsed": false 731 | }, 732 | "outputs": [ 733 | { 734 | "name": "stdout", 735 | "output_type": "stream", 736 | "text": [ 737 | "(0, [array([[ 0.03320302, 0.05792972, 0.01119308]], dtype=float32), -0.020700064])\n", 738 | "(5, [array([[ 0.14649118, 0.2548342 , 0.04982107]], dtype=float32), -0.091209568])\n", 739 | "(10, [array([[ 0.20652841, 0.35850295, 0.07069619]], dtype=float32), -0.12844093])\n", 740 | "(15, [array([[ 0.23972291, 0.41549107, 0.08243967]], dtype=float32), -0.14894117])\n", 741 | "(20, [array([[ 0.25859103, 0.44772011, 0.08921722]], dtype=float32), -0.16054493])\n", 742 | "(25, [array([[ 0.26949188, 0.4662571 , 0.09318593]], dtype=float32), -0.16722204])\n", 743 | "(30, [array([[ 0.27584928, 0.47702512, 0.09552839]], dtype=float32), -0.17110163])\n", 744 | "(35, [array([[ 0.27957708, 0.48331678, 0.09691675]], dtype=float32), -0.17336872])\n", 745 | "(40, [array([[ 0.28176975, 0.48700568, 0.09774131]], dtype=float32), -0.17469805])\n", 746 | "(45, [array([[ 0.28306174, 0.48917308, 0.09823143]], dtype=float32), -0.17547914])\n", 747 | "(50, [array([[ 0.28370172, 0.49024412, 0.09847598]], dtype=float32), -0.17586514])\n" 748 | ] 749 | } 750 | ], 751 | "source": [ 752 | "NUM_STEPS = 50\n", 753 | "\n", 754 | "\n", 755 | "g = tf.Graph()\n", 756 | "wb_ = []\n", 757 | "with g.as_default():\n", 758 | " x = tf.placeholder(tf.float32,shape=[None,3])\n", 759 | " y_true = tf.placeholder(tf.float32,shape=None)\n", 760 | " \n", 761 | " with tf.name_scope('inference') as scope:\n", 762 | " w = tf.Variable([[0,0,0]],dtype=tf.float32,name='weights')\n", 763 | " b = tf.Variable(0,dtype=tf.float32,name='bias')\n", 764 | " y_pred = tf.matmul(w,tf.transpose(x)) + b\n", 765 | "\n", 766 | " with tf.name_scope('loss') as scope:\n", 767 | " loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y_true,logits=y_pred) \n", 768 | " loss = tf.reduce_mean(loss)\n", 769 | " \n", 770 | " with tf.name_scope('train') as scope:\n", 771 | " learning_rate = 0.5\n", 772 | " optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n", 773 | " train = optimizer.minimize(loss)\n", 774 | "\n", 775 | "\n", 776 | "\n", 777 | " # Before starting, initialize the variables. We will 'run' this first.\n", 778 | " init = tf.global_variables_initializer()\n", 779 | " with tf.Session() as sess:\n", 780 | " sess.run(init) \n", 781 | " for step in range(NUM_STEPS):\n", 782 | " sess.run(train,{x: x_data, y_true: y_data})\n", 783 | " if (step % 5 == 0):\n", 784 | " print(step, sess.run([w,b]))\n", 785 | " wb_.append(sess.run([w,b]))\n", 786 | "\n", 787 | " print(50, sess.run([w,b]))" 788 | ] 789 | } 790 | ], 791 | "metadata": { 792 | "kernelspec": { 793 | "display_name": "tf_1_0", 794 | "language": "python", 795 | "name": "tf_1_0" 796 | }, 797 | "language_info": { 798 | "codemirror_mode": { 799 | "name": "ipython", 800 | "version": 2.0 801 | }, 802 | "file_extension": ".py", 803 | "mimetype": "text/x-python", 804 | "name": "python", 805 | "nbconvert_exporter": "python", 806 | "pygments_lexer": "ipython2", 807 | "version": "2.7.12" 808 | } 809 | }, 810 | "nbformat": 4, 811 | "nbformat_minor": 0 812 | } -------------------------------------------------------------------------------- /04__convolutional_neural_networks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Hezi-Resheff/Oreilly-Learning-TensorFlow/adda1fe400aa2b056780797b801aaf33a1dbeee0/04__convolutional_neural_networks/__init__.py -------------------------------------------------------------------------------- /04__convolutional_neural_networks/cifar_cnn.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import tensorflow as tf 6 | from layers import conv_layer, max_pool_2x2, full_layer 7 | 8 | DATA_PATH = "path/to/CIFAR10" 9 | BATCH_SIZE = 50 10 | STEPS = 500000 11 | 12 | 13 | def one_hot(vec, vals=10): 14 | n = len(vec) 15 | out = np.zeros((n, vals)) 16 | out[range(n), vec] = 1 17 | return out 18 | 19 | 20 | def unpickle(file): 21 | with open(os.path.join(DATA_PATH, file), 'rb') as fo: 22 | u = pickle._Unpickler(fo) 23 | u.encoding = 'latin1' 24 | dict = u.load() 25 | return dict 26 | 27 | 28 | def display_cifar(images, size): 29 | n = len(images) 30 | plt.figure() 31 | plt.gca().set_axis_off() 32 | im = np.vstack([np.hstack([images[np.random.choice(n)] for i in range(size)]) 33 | for i in range(size)]) 34 | plt.imshow(im) 35 | plt.show() 36 | 37 | 38 | class CifarLoader(object): 39 | """ 40 | Load and mange the CIFAR dataset. 41 | (for any practical use there is no reason not to use the built-in dataset handler instead) 42 | """ 43 | def __init__(self, source_files): 44 | self._source = source_files 45 | self._i = 0 46 | self.images = None 47 | self.labels = None 48 | 49 | def load(self): 50 | data = [unpickle(f) for f in self._source] 51 | images = np.vstack([d["data"] for d in data]) 52 | n = len(images) 53 | self.images = images.reshape(n, 3, 32, 32).transpose(0, 2, 3, 1)\ 54 | .astype(float) / 255 55 | self.labels = one_hot(np.hstack([d["labels"] for d in data]), 10) 56 | return self 57 | 58 | def next_batch(self, batch_size): 59 | x, y = self.images[self._i:self._i+batch_size], \ 60 | self.labels[self._i:self._i+batch_size] 61 | self._i = (self._i + batch_size) % len(self.images) 62 | return x, y 63 | 64 | def random_batch(self, batch_size): 65 | n = len(self.images) 66 | ix = np.random.choice(n, batch_size) 67 | return self.images[ix], self.labels[ix] 68 | 69 | 70 | class CifarDataManager(object): 71 | def __init__(self): 72 | self.train = CifarLoader(["data_batch_{}".format(i) for i in range(1, 6)])\ 73 | .load() 74 | self.test = CifarLoader(["test_batch"]).load() 75 | 76 | 77 | def run_simple_net(): 78 | cifar = CifarDataManager() 79 | 80 | x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3]) 81 | y_ = tf.placeholder(tf.float32, shape=[None, 10]) 82 | keep_prob = tf.placeholder(tf.float32) 83 | 84 | conv1 = conv_layer(x, shape=[5, 5, 3, 32]) 85 | conv1_pool = max_pool_2x2(conv1) 86 | 87 | conv2 = conv_layer(conv1_pool, shape=[5, 5, 32, 64]) 88 | conv2_pool = max_pool_2x2(conv2) 89 | 90 | conv3 = conv_layer(conv2_pool, shape=[5, 5, 64, 128]) 91 | conv3_pool = max_pool_2x2(conv3) 92 | conv3_flat = tf.reshape(conv3_pool, [-1, 4 * 4 * 128]) 93 | conv3_drop = tf.nn.dropout(conv3_flat, keep_prob=keep_prob) 94 | 95 | full_1 = tf.nn.relu(full_layer(conv3_drop, 512)) 96 | full1_drop = tf.nn.dropout(full_1, keep_prob=keep_prob) 97 | 98 | y_conv = full_layer(full1_drop, 10) 99 | 100 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, 101 | labels=y_)) 102 | train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy) 103 | 104 | correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) 105 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 106 | 107 | def test(sess): 108 | X = cifar.test.images.reshape(10, 1000, 32, 32, 3) 109 | Y = cifar.test.labels.reshape(10, 1000, 10) 110 | acc = np.mean([sess.run(accuracy, feed_dict={x: X[i], y_: Y[i], keep_prob: 1.0}) 111 | for i in range(10)]) 112 | print("Accuracy: {:.4}%".format(acc * 100)) 113 | 114 | with tf.Session() as sess: 115 | sess.run(tf.global_variables_initializer()) 116 | 117 | for i in range(STEPS): 118 | batch = cifar.train.next_batch(BATCH_SIZE) 119 | sess.run(train_step, feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5}) 120 | 121 | if i % 500 == 0: 122 | test(sess) 123 | 124 | test(sess) 125 | 126 | 127 | def build_second_net(): 128 | x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3]) 129 | y_ = tf.placeholder(tf.float32, shape=[None, 10]) 130 | keep_prob = tf.placeholder(tf.float32) 131 | 132 | C1, C2, C3 = 32, 64, 128 133 | F1 = 600 134 | 135 | conv1_1 = conv_layer(x, shape=[3, 3, 3, C1]) 136 | conv1_2 = conv_layer(conv1_1, shape=[3, 3, C1, C1]) 137 | conv1_3 = conv_layer(conv1_2, shape=[3, 3, C1, C1]) 138 | conv1_pool = max_pool_2x2(conv1_3) 139 | conv1_drop = tf.nn.dropout(conv1_pool, keep_prob=keep_prob) 140 | 141 | conv2_1 = conv_layer(conv1_drop, shape=[3, 3, C1, C2]) 142 | conv2_2 = conv_layer(conv2_1, shape=[3, 3, C2, C2]) 143 | conv2_3 = conv_layer(conv2_2, shape=[3, 3, C2, C2]) 144 | conv2_pool = max_pool_2x2(conv2_3) 145 | conv2_drop = tf.nn.dropout(conv2_pool, keep_prob=keep_prob) 146 | 147 | conv3_1 = conv_layer(conv2_drop, shape=[3, 3, C2, C3]) 148 | conv3_2 = conv_layer(conv3_1, shape=[3, 3, C3, C3]) 149 | conv3_3 = conv_layer(conv3_2, shape=[3, 3, C3, C3]) 150 | conv3_pool = tf.nn.max_pool(conv3_3, ksize=[1, 8, 8, 1], strides=[1, 8, 8, 1], padding='SAME') 151 | conv3_flat = tf.reshape(conv3_pool, [-1, C3]) 152 | conv3_drop = tf.nn.dropout(conv3_flat, keep_prob=keep_prob) 153 | 154 | full1 = tf.nn.relu(full_layer(conv3_drop, F1)) 155 | full1_drop = tf.nn.dropout(full1, keep_prob=keep_prob) 156 | 157 | y_conv = full_layer(full1_drop, 10) 158 | 159 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, 160 | labels=y_)) 161 | train_step = tf.train.AdamOptimizer(5e-4).minimize(cross_entropy) # noqa 162 | 163 | correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) 164 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # noqa 165 | 166 | # Plug this into the test procedure as above to continue... 167 | 168 | 169 | def create_cifar_image(): 170 | d = CifarDataManager() 171 | print("Number of train images: {}".format(len(d.train.images))) 172 | print("Number of train labels: {}".format(len(d.train.labels))) 173 | print("Number of test images: {}".format(len(d.test.images))) 174 | print("Number of test labels: {}".format(len(d.test.labels))) 175 | images = d.train.images 176 | display_cifar(images, 10) 177 | 178 | 179 | if __name__ == "__main__": 180 | create_cifar_image() 181 | run_simple_net() 182 | -------------------------------------------------------------------------------- /04__convolutional_neural_networks/layers.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def weight_variable(shape): 5 | initial = tf.truncated_normal(shape, stddev=0.1) 6 | return tf.Variable(initial) 7 | 8 | 9 | def bias_variable(shape): 10 | initial = tf.constant(0.1, shape=shape) 11 | return tf.Variable(initial) 12 | 13 | 14 | def conv2d(x, W): 15 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 16 | 17 | 18 | def max_pool_2x2(x): 19 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], 20 | strides=[1, 2, 2, 1], padding='SAME') 21 | 22 | 23 | def conv_layer(input, shape): 24 | W = weight_variable(shape) 25 | b = bias_variable([shape[3]]) 26 | return tf.nn.relu(conv2d(input, W) + b) 27 | 28 | 29 | def full_layer(input, size): 30 | in_size = int(input.get_shape()[1]) 31 | W = weight_variable([in_size, size]) 32 | b = bias_variable([size]) 33 | return tf.matmul(input, W) + b 34 | -------------------------------------------------------------------------------- /04__convolutional_neural_networks/mnist_cnn.py: -------------------------------------------------------------------------------- 1 | from tensorflow.examples.tutorials.mnist import input_data 2 | import tensorflow as tf 3 | import numpy as np 4 | 5 | from layers import conv_layer, max_pool_2x2, full_layer 6 | 7 | DATA_DIR = '/tmp/data' 8 | MINIBATCH_SIZE = 50 9 | STEPS = 5000 10 | 11 | 12 | mnist = input_data.read_data_sets(DATA_DIR, one_hot=True) 13 | 14 | x = tf.placeholder(tf.float32, shape=[None, 784]) 15 | y_ = tf.placeholder(tf.float32, shape=[None, 10]) 16 | 17 | x_image = tf.reshape(x, [-1, 28, 28, 1]) 18 | conv1 = conv_layer(x_image, shape=[5, 5, 1, 32]) 19 | conv1_pool = max_pool_2x2(conv1) 20 | 21 | conv2 = conv_layer(conv1_pool, shape=[5, 5, 32, 64]) 22 | conv2_pool = max_pool_2x2(conv2) 23 | 24 | conv2_flat = tf.reshape(conv2_pool, [-1, 7*7*64]) 25 | full_1 = tf.nn.relu(full_layer(conv2_flat, 1024)) 26 | 27 | keep_prob = tf.placeholder(tf.float32) 28 | full1_drop = tf.nn.dropout(full_1, keep_prob=keep_prob) 29 | 30 | y_conv = full_layer(full1_drop, 10) 31 | 32 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_)) 33 | train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) 34 | correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) 35 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 36 | 37 | with tf.Session() as sess: 38 | sess.run(tf.global_variables_initializer()) 39 | 40 | for i in range(STEPS): 41 | batch = mnist.train.next_batch(MINIBATCH_SIZE) 42 | 43 | if i % 100 == 0: 44 | train_accuracy = sess.run(accuracy, feed_dict={x: batch[0], y_: batch[1], 45 | keep_prob: 1.0}) 46 | print("step {}, training accuracy {}".format(i, train_accuracy)) 47 | 48 | sess.run(train_step, feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5}) 49 | 50 | X = mnist.test.images.reshape(10, 1000, 784) 51 | Y = mnist.test.labels.reshape(10, 1000, 10) 52 | test_accuracy = np.mean( 53 | [sess.run(accuracy, feed_dict={x: X[i], y_: Y[i], keep_prob: 1.0}) for i in range(10)]) 54 | 55 | print("test accuracy: {}".format(test_accuracy)) 56 | -------------------------------------------------------------------------------- /05__text_and_visualizations/BasicRNNCell.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 20 17:34:43 2016 4 | 5 | @author: tomhope 6 | """ 7 | from __future__ import print_function 8 | import tensorflow as tf 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) 11 | 12 | element_size = 28 13 | time_steps = 28 14 | num_classes = 10 15 | batch_size = 128 16 | hidden_layer_size = 128 17 | 18 | _inputs = tf.placeholder(tf.float32, 19 | shape=[None, time_steps, element_size], 20 | name='inputs') 21 | y = tf.placeholder(tf.float32, shape=[None, num_classes], name='inputs') 22 | 23 | # TensorFlow built-in functions 24 | rnn_cell = tf.contrib.rnn.BasicRNNCell(hidden_layer_size) 25 | outputs, _ = tf.nn.dynamic_rnn(rnn_cell, _inputs, dtype=tf.float32) 26 | 27 | Wl = tf.Variable(tf.truncated_normal([hidden_layer_size, num_classes], 28 | mean=0, stddev=.01)) 29 | bl = tf.Variable(tf.truncated_normal([num_classes], mean=0, stddev=.01)) 30 | 31 | 32 | def get_linear_layer(vector): 33 | return tf.matmul(vector, Wl) + bl 34 | 35 | 36 | last_rnn_output = outputs[:, -1, :] 37 | final_output = get_linear_layer(last_rnn_output) 38 | 39 | softmax = tf.nn.softmax_cross_entropy_with_logits(logits=final_output, labels=y) 40 | cross_entropy = tf.reduce_mean(softmax) 41 | train_step = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cross_entropy) 42 | 43 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(final_output, 1)) 44 | accuracy = (tf.reduce_mean(tf.cast(correct_prediction, tf.float32))) * 100 45 | 46 | sess = tf.InteractiveSession() 47 | sess.run(tf.global_variables_initializer()) 48 | 49 | test_data = mnist.test.images[:batch_size].reshape( 50 | (-1, time_steps, element_size)) 51 | test_label = mnist.test.labels[:batch_size] 52 | 53 | for i in range(3001): 54 | batch_x, batch_y = mnist.train.next_batch(batch_size) 55 | batch_x = batch_x.reshape((batch_size, time_steps, element_size)) 56 | sess.run(train_step, feed_dict={_inputs: batch_x, 57 | y: batch_y}) 58 | if i % 1000 == 0: 59 | acc = sess.run(accuracy, feed_dict={_inputs: batch_x, 60 | y: batch_y}) 61 | loss = sess.run(cross_entropy, feed_dict={_inputs: batch_x, 62 | y: batch_y}) 63 | print("Iter " + str(i) + ", Minibatch Loss= " + 64 | "{:.6f}".format(loss) + ", Training Accuracy= " + 65 | "{:.5f}".format(acc)) 66 | 67 | print("Testing Accuracy:", sess.run( 68 | accuracy, feed_dict={_inputs: test_data, y: test_label})) 69 | -------------------------------------------------------------------------------- /05__text_and_visualizations/LSTM_supervised_embeddings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Dec 29 00:39:23 2016 4 | 5 | @author: tomhope 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | 11 | batch_size = 128 12 | embedding_dimension = 64 13 | num_classes = 2 14 | hidden_layer_size = 32 15 | times_steps = 6 16 | element_size = 1 17 | 18 | digit_to_word_map = {1: "One", 2: "Two", 3: "Three", 4: "Four", 5: "Five", 19 | 6: "Six", 7: "Seven", 8: "Eight", 9: "Nine"} 20 | digit_to_word_map[0] = "PAD" 21 | 22 | even_sentences = [] 23 | odd_sentences = [] 24 | seqlens = [] 25 | for i in range(10000): 26 | rand_seq_len = np.random.choice(range(3, 7)) 27 | seqlens.append(rand_seq_len) 28 | rand_odd_ints = np.random.choice(range(1, 10, 2), 29 | rand_seq_len) 30 | rand_even_ints = np.random.choice(range(2, 10, 2), 31 | rand_seq_len) 32 | 33 | if rand_seq_len < 6: 34 | rand_odd_ints = np.append(rand_odd_ints, 35 | [0]*(6-rand_seq_len)) 36 | rand_even_ints = np.append(rand_even_ints, 37 | [0]*(6-rand_seq_len)) 38 | 39 | even_sentences.append(" ".join([digit_to_word_map[r] for r in rand_odd_ints])) 40 | odd_sentences.append(" ".join([digit_to_word_map[r] for r in rand_even_ints])) 41 | 42 | data = even_sentences+odd_sentences 43 | seqlens *= 2 44 | labels = [1] * 10000 + [0] * 10000 45 | for i in range(len(labels)): 46 | label = labels[i] 47 | one_hot_encoding = [0]*2 48 | one_hot_encoding[label] = 1 49 | labels[i] = one_hot_encoding 50 | 51 | word2index_map = {} 52 | index = 0 53 | for sent in data: 54 | for word in sent.lower().split(): 55 | if word not in word2index_map: 56 | word2index_map[word] = index 57 | index += 1 58 | 59 | index2word_map = {index: word for word, index in word2index_map.items()} 60 | 61 | vocabulary_size = len(index2word_map) 62 | 63 | data_indices = list(range(len(data))) 64 | np.random.shuffle(data_indices) 65 | data = np.array(data)[data_indices] 66 | labels = np.array(labels)[data_indices] 67 | seqlens = np.array(seqlens)[data_indices] 68 | train_x = data[:10000] 69 | train_y = labels[:10000] 70 | train_seqlens = seqlens[:10000] 71 | 72 | test_x = data[10000:] 73 | test_y = labels[10000:] 74 | test_seqlens = seqlens[10000:] 75 | 76 | 77 | def get_sentence_batch(batch_size, data_x, 78 | data_y, data_seqlens): 79 | instance_indices = list(range(len(data_x))) 80 | np.random.shuffle(instance_indices) 81 | batch = instance_indices[:batch_size] 82 | x = [[word2index_map[word] for word in data_x[i].lower().split()] 83 | for i in batch] 84 | y = [data_y[i] for i in batch] 85 | seqlens = [data_seqlens[i] for i in batch] 86 | return x, y, seqlens 87 | 88 | 89 | _inputs = tf.placeholder(tf.int32, shape=[batch_size, times_steps]) 90 | _labels = tf.placeholder(tf.float32, shape=[batch_size, num_classes]) 91 | # seqlens for dynamic calculation 92 | _seqlens = tf.placeholder(tf.int32, shape=[batch_size]) 93 | 94 | with tf.name_scope("embeddings"): 95 | embeddings = tf.Variable( 96 | tf.random_uniform([vocabulary_size, 97 | embedding_dimension], 98 | -1.0, 1.0), name='embedding') 99 | embed = tf.nn.embedding_lookup(embeddings, _inputs) 100 | 101 | 102 | with tf.variable_scope("lstm"): 103 | 104 | lstm_cell = tf.contrib.rnn.BasicLSTMCell(hidden_layer_size, 105 | forget_bias=1.0) 106 | outputs, states = tf.nn.dynamic_rnn(lstm_cell, embed, 107 | sequence_length=_seqlens, 108 | dtype=tf.float32) 109 | 110 | weights = { 111 | 'linear_layer': tf.Variable(tf.truncated_normal([hidden_layer_size, num_classes], 112 | mean=0, stddev=.01)) 113 | } 114 | biases = { 115 | 'linear_layer': tf.Variable(tf.truncated_normal([num_classes], mean=0, stddev=.01)) 116 | } 117 | 118 | # extract the last relevant output and use in a linear layer 119 | final_output = tf.matmul(states[1], 120 | weights["linear_layer"]) + biases["linear_layer"] 121 | 122 | softmax = tf.nn.softmax_cross_entropy_with_logits(logits=final_output, 123 | labels=_labels) 124 | cross_entropy = tf.reduce_mean(softmax) 125 | 126 | train_step = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cross_entropy) 127 | correct_prediction = tf.equal(tf.argmax(_labels, 1), 128 | tf.argmax(final_output, 1)) 129 | accuracy = (tf.reduce_mean(tf.cast(correct_prediction, 130 | tf.float32)))*100 131 | 132 | with tf.Session() as sess: 133 | sess.run(tf.global_variables_initializer()) 134 | 135 | for step in range(1000): 136 | x_batch, y_batch, seqlen_batch = get_sentence_batch(batch_size, 137 | train_x, train_y, 138 | train_seqlens) 139 | sess.run(train_step, feed_dict={_inputs: x_batch, _labels: y_batch, 140 | _seqlens: seqlen_batch}) 141 | 142 | if step % 100 == 0: 143 | acc = sess.run(accuracy, feed_dict={_inputs: x_batch, 144 | _labels: y_batch, 145 | _seqlens: seqlen_batch}) 146 | print("Accuracy at %d: %.5f" % (step, acc)) 147 | 148 | for test_batch in range(5): 149 | x_test, y_test, seqlen_test = get_sentence_batch(batch_size, 150 | test_x, test_y, 151 | test_seqlens) 152 | batch_pred, batch_acc = sess.run([tf.argmax(final_output, 1), accuracy], 153 | feed_dict={_inputs: x_test, 154 | _labels: y_test, 155 | _seqlens: seqlen_test}) 156 | print("Test batch accuracy %d: %.5f" % (test_batch, batch_acc)) 157 | 158 | output_example = sess.run([outputs], feed_dict={_inputs: x_test, 159 | _labels: y_test, 160 | _seqlens: seqlen_test}) 161 | states_example = sess.run([states[1]], feed_dict={_inputs: x_test, 162 | _labels: y_test, 163 | _seqlens: seqlen_test}) 164 | # 165 | # #CODE BLOCK FOR MULTIPLE LSTM 166 | # num_LSTM_layers = 2 167 | # with tf.variable_scope("lstm"): 168 | # 169 | # lstm_cell = tf.contrib.rnn.BasicLSTMCell(hidden_layer_size, 170 | # forget_bias=1.0) 171 | # cell = tf.contrib.rnn.MultiRNNCell(cells=[lstm_cell]*num_LSTM_layers, 172 | # state_is_tuple=True) 173 | # outputs, states = tf.nn.dynamic_rnn(cell, embed, 174 | # sequence_length = _seqlens, 175 | # dtype=tf.float32) 176 | # 177 | # #extract the final state and use in a linear layer 178 | # final_output = tf.matmul(states[num_LSTM_layers-1][1], 179 | # weights["linear_layer"]) + biases["linear_layer"] 180 | -------------------------------------------------------------------------------- /05__text_and_visualizations/scan_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 28 23:20:29 2016 4 | 5 | @author: tomhope 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | 11 | elems = np.array(["T", "e", "n", "s", "o", "r", " ", "F", "l", "o", "w"]) 12 | scan_sum = tf.scan(lambda a, x: a + x, elems) 13 | 14 | sess = tf.InteractiveSession() 15 | sess.run(scan_sum) 16 | -------------------------------------------------------------------------------- /05__text_and_visualizations/vanilla_rnn_with_tfboard.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 20 17:34:43 2016 4 | 5 | @author: tomhope 6 | """ 7 | from __future__ import print_function 8 | import tensorflow as tf 9 | 10 | # Import MINST data 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) 13 | 14 | # Define some parameters 15 | element_size = 28 16 | time_steps = 28 17 | num_classes = 10 18 | batch_size = 128 19 | hidden_layer_size = 128 20 | 21 | # Where to save TensorBoard model summaries 22 | LOG_DIR = "logs/RNN_with_summaries" 23 | 24 | # Create placeholders for inputs, labels 25 | _inputs = tf.placeholder(tf.float32, 26 | shape=[None, time_steps, element_size], 27 | name='inputs') 28 | y = tf.placeholder(tf.float32, shape=[None, num_classes], name='labels') 29 | 30 | 31 | # This helper function taken from official TensorFlow documentation, 32 | # simply add some ops that take care of logging summaries 33 | def variable_summaries(var): 34 | with tf.name_scope('summaries'): 35 | mean = tf.reduce_mean(var) 36 | tf.summary.scalar('mean', mean) 37 | with tf.name_scope('stddev'): 38 | stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) 39 | tf.summary.scalar('stddev', stddev) 40 | tf.summary.scalar('max', tf.reduce_max(var)) 41 | tf.summary.scalar('min', tf.reduce_min(var)) 42 | tf.summary.histogram('histogram', var) 43 | 44 | 45 | # Weights and bias for input and hidden layer 46 | with tf.name_scope('rnn_weights'): 47 | with tf.name_scope("W_x"): 48 | Wx = tf.Variable(tf.zeros([element_size, hidden_layer_size])) 49 | variable_summaries(Wx) 50 | with tf.name_scope("W_h"): 51 | Wh = tf.Variable(tf.zeros([hidden_layer_size, hidden_layer_size])) 52 | variable_summaries(Wh) 53 | with tf.name_scope("Bias"): 54 | b_rnn = tf.Variable(tf.zeros([hidden_layer_size])) 55 | variable_summaries(b_rnn) 56 | 57 | 58 | def rnn_step(previous_hidden_state, x): 59 | 60 | current_hidden_state = tf.tanh( 61 | tf.matmul(previous_hidden_state, Wh) + 62 | tf.matmul(x, Wx) + b_rnn) 63 | 64 | return current_hidden_state 65 | 66 | 67 | # Processing inputs to work with scan function 68 | # Current input shape: (batch_size, time_steps, element_size) 69 | processed_input = tf.transpose(_inputs, perm=[1, 0, 2]) 70 | # Current input shape now: (time_steps,batch_size, element_size) 71 | 72 | 73 | initial_hidden = tf.zeros([batch_size, hidden_layer_size]) 74 | # Getting all state vectors across time 75 | all_hidden_states = tf.scan(rnn_step, 76 | processed_input, 77 | initializer=initial_hidden, 78 | name='states') 79 | 80 | 81 | # Weights for output layers 82 | with tf.name_scope('linear_layer_weights') as scope: 83 | with tf.name_scope("W_linear"): 84 | Wl = tf.Variable(tf.truncated_normal([hidden_layer_size, num_classes], 85 | mean=0, stddev=.01)) 86 | variable_summaries(Wl) 87 | with tf.name_scope("Bias_linear"): 88 | bl = tf.Variable(tf.truncated_normal([num_classes], 89 | mean=0, stddev=.01)) 90 | variable_summaries(bl) 91 | 92 | 93 | # Apply linear layer to state vector 94 | def get_linear_layer(hidden_state): 95 | 96 | return tf.matmul(hidden_state, Wl) + bl 97 | 98 | 99 | with tf.name_scope('linear_layer_weights') as scope: 100 | # Iterate across time, apply linear layer to all RNN outputs 101 | all_outputs = tf.map_fn(get_linear_layer, all_hidden_states) 102 | # Get Last output -- h_28 103 | output = all_outputs[-1] 104 | tf.summary.histogram('outputs', output) 105 | 106 | with tf.name_scope('cross_entropy'): 107 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y)) 108 | tf.summary.scalar('cross_entropy', cross_entropy) 109 | 110 | with tf.name_scope('train'): 111 | # Using RMSPropOptimizer 112 | train_step = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cross_entropy) 113 | 114 | with tf.name_scope('accuracy'): 115 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(output, 1)) 116 | 117 | accuracy = (tf.reduce_mean(tf.cast(correct_prediction, tf.float32)))*100 118 | tf.summary.scalar('accuracy', accuracy) 119 | 120 | # Merge all the summaries 121 | merged = tf.summary.merge_all() 122 | 123 | 124 | # Get a small test set 125 | test_data = mnist.test.images[:batch_size].reshape((-1, time_steps, element_size)) 126 | test_label = mnist.test.labels[:batch_size] 127 | 128 | with tf.Session() as sess: 129 | # Write summaries to LOG_DIR -- used by TensorBoard 130 | train_writer = tf.summary.FileWriter(LOG_DIR + '/train', 131 | graph=tf.get_default_graph()) 132 | test_writer = tf.summary.FileWriter(LOG_DIR + '/test', 133 | graph=tf.get_default_graph()) 134 | 135 | sess.run(tf.global_variables_initializer()) 136 | 137 | for i in range(10000): 138 | 139 | batch_x, batch_y = mnist.train.next_batch(batch_size) 140 | # Reshape data to get 28 sequences of 28 pixels 141 | batch_x = batch_x.reshape((batch_size, time_steps, element_size)) 142 | summary, _ = sess.run([merged, train_step], 143 | feed_dict={_inputs: batch_x, y: batch_y}) 144 | # Add to summaries 145 | train_writer.add_summary(summary, i) 146 | 147 | if i % 1000 == 0: 148 | acc, loss, = sess.run([accuracy, cross_entropy], 149 | feed_dict={_inputs: batch_x, 150 | y: batch_y}) 151 | print("Iter " + str(i) + ", Minibatch Loss= " + 152 | "{:.6f}".format(loss) + ", Training Accuracy= " + 153 | "{:.5f}".format(acc)) 154 | if i % 100 == 0: 155 | # Calculate accuracy for 128 mnist test images and 156 | # add to summaries 157 | summary, acc = sess.run([merged, accuracy], 158 | feed_dict={_inputs: test_data, 159 | y: test_label}) 160 | test_writer.add_summary(summary, i) 161 | 162 | test_acc = sess.run(accuracy, feed_dict={_inputs: test_data, 163 | y: test_label}) 164 | print("Test Accuracy:", test_acc) 165 | -------------------------------------------------------------------------------- /06__word_embeddings_and_rnns/GRU_pretrained_GloVe.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Mar 1 12:18:27 2017 4 | 5 | @author: tomhope 6 | """ 7 | import zipfile 8 | import numpy as np 9 | import tensorflow as tf 10 | 11 | path_to_glove = "c:\\tmp\\data\\glove.840B.300d.zip" 12 | PRE_TRAINED = True 13 | GLOVE_SIZE = 300 14 | batch_size = 128 15 | embedding_dimension = 64 16 | num_classes = 2 17 | hidden_layer_size = 32 18 | times_steps = 6 19 | 20 | digit_to_word_map = {1: "One", 2: "Two", 3: "Three", 4: "Four", 5: "Five", 21 | 6: "Six", 7: "Seven", 8: "Eight", 9: "Nine"} 22 | digit_to_word_map[0] = "PAD_TOKEN" 23 | 24 | even_sentences = [] 25 | odd_sentences = [] 26 | seqlens = [] 27 | for i in range(10000): 28 | rand_seq_len = np.random.choice(range(3, 7)) 29 | seqlens.append(rand_seq_len) 30 | rand_odd_ints = np.random.choice(range(1, 10, 2), 31 | rand_seq_len) 32 | rand_even_ints = np.random.choice(range(2, 10, 2), 33 | rand_seq_len) 34 | 35 | if rand_seq_len < 6: 36 | rand_odd_ints = np.append(rand_odd_ints, 37 | [0]*(6-rand_seq_len)) 38 | rand_even_ints = np.append(rand_even_ints, 39 | [0]*(6-rand_seq_len)) 40 | 41 | even_sentences.append(" ".join([digit_to_word_map[r] 42 | for r in rand_odd_ints])) 43 | odd_sentences.append(" ".join([digit_to_word_map[r] 44 | for r in rand_even_ints])) 45 | 46 | data = even_sentences+odd_sentences 47 | # same seq lengths for even, odd sentences 48 | seqlens *= 2 49 | labels = [1]*10000 + [0]*10000 50 | for i in range(len(labels)): 51 | label = labels[i] 52 | one_hot_encoding = [0]*2 53 | one_hot_encoding[label] = 1 54 | labels[i] = one_hot_encoding 55 | 56 | word2index_map = {} 57 | index = 0 58 | for sent in data: 59 | for word in sent.split(): 60 | if word not in word2index_map: 61 | word2index_map[word] = index 62 | index += 1 63 | 64 | index2word_map = {index: word for word, index in word2index_map.items()} 65 | 66 | vocabulary_size = len(index2word_map) 67 | 68 | 69 | def get_glove(path_to_glove, word2index_map): 70 | 71 | embedding_weights = {} 72 | count_all_words = 0 73 | with zipfile.ZipFile(path_to_glove) as z: 74 | with z.open("glove.840B.300d.txt") as f: 75 | for line in f: 76 | vals = line.split() 77 | word = str(vals[0].decode("utf-8")) 78 | if word in word2index_map: 79 | print(word) 80 | count_all_words += 1 81 | coefs = np.asarray(vals[1:], dtype='float32') 82 | coefs /= np.linalg.norm(coefs) 83 | embedding_weights[word] = coefs 84 | if count_all_words == len(word2index_map) - 1: 85 | break 86 | return embedding_weights 87 | 88 | 89 | word2embedding_dict = get_glove(path_to_glove, word2index_map) 90 | embedding_matrix = np.zeros((vocabulary_size, GLOVE_SIZE)) 91 | 92 | for word, index in word2index_map.items(): 93 | if not word == "PAD_TOKEN": 94 | word_embedding = word2embedding_dict[word] 95 | embedding_matrix[index, :] = word_embedding 96 | 97 | 98 | data_indices = list(range(len(data))) 99 | np.random.shuffle(data_indices) 100 | data = np.array(data)[data_indices] 101 | labels = np.array(labels)[data_indices] 102 | seqlens = np.array(seqlens)[data_indices] 103 | train_x = data[:10000] 104 | train_y = labels[:10000] 105 | train_seqlens = seqlens[:10000] 106 | 107 | test_x = data[10000:] 108 | test_y = labels[10000:] 109 | test_seqlens = seqlens[10000:] 110 | 111 | 112 | def get_sentence_batch(batch_size, data_x, 113 | data_y, data_seqlens): 114 | instance_indices = list(range(len(data_x))) 115 | np.random.shuffle(instance_indices) 116 | batch = instance_indices[:batch_size] 117 | x = [[word2index_map[word] for word in data_x[i].split()] 118 | for i in batch] 119 | y = [data_y[i] for i in batch] 120 | seqlens = [data_seqlens[i] for i in batch] 121 | return x, y, seqlens 122 | 123 | 124 | _inputs = tf.placeholder(tf.int32, shape=[batch_size, times_steps]) 125 | embedding_placeholder = tf.placeholder(tf.float32, [vocabulary_size, 126 | GLOVE_SIZE]) 127 | 128 | _labels = tf.placeholder(tf.float32, shape=[batch_size, num_classes]) 129 | _seqlens = tf.placeholder(tf.int32, shape=[batch_size]) 130 | 131 | if PRE_TRAINED: 132 | embeddings = tf.Variable(tf.constant(0.0, shape=[vocabulary_size, GLOVE_SIZE]), 133 | trainable=True) 134 | # if using pre-trained embeddings, assign them to the embeddings variable 135 | embedding_init = embeddings.assign(embedding_placeholder) 136 | embed = tf.nn.embedding_lookup(embeddings, _inputs) 137 | 138 | else: 139 | embeddings = tf.Variable( 140 | tf.random_uniform([vocabulary_size, 141 | embedding_dimension], 142 | -1.0, 1.0)) 143 | embed = tf.nn.embedding_lookup(embeddings, _inputs) 144 | 145 | with tf.name_scope("biGRU"): 146 | with tf.variable_scope('forward'): 147 | gru_fw_cell = tf.contrib.rnn.GRUCell(hidden_layer_size) 148 | gru_fw_cell = tf.contrib.rnn.DropoutWrapper(gru_fw_cell) 149 | 150 | with tf.variable_scope('backward'): 151 | gru_bw_cell = tf.contrib.rnn.GRUCell(hidden_layer_size) 152 | gru_bw_cell = tf.contrib.rnn.DropoutWrapper(gru_bw_cell) 153 | 154 | outputs, states = tf.nn.bidirectional_dynamic_rnn(cell_fw=gru_fw_cell, 155 | cell_bw=gru_bw_cell, 156 | inputs=embed, 157 | sequence_length=_seqlens, 158 | dtype=tf.float32, 159 | scope="biGRU") 160 | states = tf.concat(values=states, axis=1) 161 | weights = { 162 | 'linear_layer': tf.Variable(tf.truncated_normal([2*hidden_layer_size, 163 | num_classes], 164 | mean=0, stddev=.01)) 165 | } 166 | biases = { 167 | 'linear_layer': tf.Variable(tf.truncated_normal([num_classes], 168 | mean=0, stddev=.01)) 169 | } 170 | 171 | # extract the final state and use in a linear layer 172 | final_output = tf.matmul(states, 173 | weights["linear_layer"]) + biases["linear_layer"] 174 | 175 | softmax = tf.nn.softmax_cross_entropy_with_logits(logits=final_output, 176 | labels=_labels) 177 | cross_entropy = tf.reduce_mean(softmax) 178 | 179 | train_step = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cross_entropy) 180 | correct_prediction = tf.equal(tf.argmax(_labels, 1), 181 | tf.argmax(final_output, 1)) 182 | accuracy = (tf.reduce_mean(tf.cast(correct_prediction, 183 | tf.float32)))*100 184 | 185 | with tf.Session() as sess: 186 | sess.run(tf.global_variables_initializer()) 187 | sess.run(embedding_init, 188 | feed_dict={embedding_placeholder: embedding_matrix}) 189 | for step in range(1000): 190 | x_batch, y_batch, seqlen_batch = get_sentence_batch(batch_size, 191 | train_x, train_y, 192 | train_seqlens) 193 | sess.run(train_step, feed_dict={_inputs: x_batch, _labels: y_batch, 194 | _seqlens: seqlen_batch}) 195 | 196 | if step % 100 == 0: 197 | acc = sess.run(accuracy, feed_dict={_inputs: x_batch, 198 | _labels: y_batch, 199 | _seqlens: seqlen_batch}) 200 | print("Accuracy at %d: %.5f" % (step, acc)) 201 | 202 | norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 203 | 1, keep_dims=True)) 204 | normalized_embeddings = embeddings / norm 205 | normalized_embeddings_matrix = sess.run(normalized_embeddings) 206 | 207 | for test_batch in range(5): 208 | x_test, y_test, seqlen_test = get_sentence_batch(batch_size, 209 | test_x, test_y, 210 | test_seqlens) 211 | batch_pred, batch_acc = sess.run([tf.argmax(final_output, 1), accuracy], 212 | feed_dict={_inputs: x_test, 213 | _labels: y_test, 214 | _seqlens: seqlen_test}) 215 | print("Test batch accuracy %d: %.5f" % (test_batch, batch_acc)) 216 | 217 | ref_word = normalized_embeddings_matrix[word2index_map["Three"]] 218 | 219 | cosine_dists = np.dot(normalized_embeddings_matrix, ref_word) 220 | ff = np.argsort(cosine_dists)[::-1][1:10] 221 | for f in ff: 222 | print(index2word_map[f]) 223 | print(cosine_dists[f]) 224 | -------------------------------------------------------------------------------- /06__word_embeddings_and_rnns/word2vec.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Dec 29 00:39:23 2016 4 | 5 | @author: tomhope 6 | """ 7 | import os 8 | import math 9 | import numpy as np 10 | import tensorflow as tf 11 | from tensorflow.contrib.tensorboard.plugins import projector 12 | 13 | 14 | batch_size = 64 15 | embedding_dimension = 5 16 | negative_samples = 8 17 | LOG_DIR = "logs/word2vec_intro" 18 | 19 | 20 | digit_to_word_map = {1: "One", 2: "Two", 3: "Three", 4: "Four", 5: "Five", 21 | 6: "Six", 7: "Seven", 8: "Eight", 9: "Nine"} 22 | sentences = [] 23 | 24 | # Create two kinds of sentences - sequences of odd and even digits. 25 | for i in range(10000): 26 | rand_odd_ints = np.random.choice(range(1, 10, 2), 3) 27 | sentences.append(" ".join([digit_to_word_map[r] for r in rand_odd_ints])) 28 | rand_even_ints = np.random.choice(range(2, 10, 2), 3) 29 | sentences.append(" ".join([digit_to_word_map[r] for r in rand_even_ints])) 30 | 31 | # Map words to indices 32 | word2index_map = {} 33 | index = 0 34 | for sent in sentences: 35 | for word in sent.lower().split(): 36 | if word not in word2index_map: 37 | word2index_map[word] = index 38 | index += 1 39 | index2word_map = {index: word for word, index in word2index_map.items()} 40 | 41 | vocabulary_size = len(index2word_map) 42 | 43 | # Generate skip-gram pairs 44 | skip_gram_pairs = [] 45 | for sent in sentences: 46 | tokenized_sent = sent.lower().split() 47 | for i in range(1, len(tokenized_sent)-1): 48 | word_context_pair = [[word2index_map[tokenized_sent[i-1]], 49 | word2index_map[tokenized_sent[i+1]]], 50 | word2index_map[tokenized_sent[i]]] 51 | skip_gram_pairs.append([word_context_pair[1], 52 | word_context_pair[0][0]]) 53 | skip_gram_pairs.append([word_context_pair[1], 54 | word_context_pair[0][1]]) 55 | 56 | 57 | def get_skipgram_batch(batch_size): 58 | instance_indices = list(range(len(skip_gram_pairs))) 59 | np.random.shuffle(instance_indices) 60 | batch = instance_indices[:batch_size] 61 | x = [skip_gram_pairs[i][0] for i in batch] 62 | y = [[skip_gram_pairs[i][1]] for i in batch] 63 | return x, y 64 | 65 | 66 | # batch example 67 | x_batch, y_batch = get_skipgram_batch(8) 68 | x_batch 69 | y_batch 70 | [index2word_map[word] for word in x_batch] 71 | [index2word_map[word[0]] for word in y_batch] 72 | 73 | # Input data, labels 74 | train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) 75 | train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) 76 | 77 | # Embedding lookup table currently only implemented in CPU 78 | with tf.name_scope("embeddings"): 79 | embeddings = tf.Variable( 80 | tf.random_uniform([vocabulary_size, embedding_dimension], 81 | -1.0, 1.0), name='embedding') 82 | # This is essentialy a lookup table 83 | embed = tf.nn.embedding_lookup(embeddings, train_inputs) 84 | 85 | # Create variables for the NCE loss 86 | nce_weights = tf.Variable( 87 | tf.truncated_normal([vocabulary_size, embedding_dimension], 88 | stddev=1.0 / math.sqrt(embedding_dimension))) 89 | nce_biases = tf.Variable(tf.zeros([vocabulary_size])) 90 | 91 | 92 | loss = tf.reduce_mean( 93 | tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, inputs=embed, labels=train_labels, 94 | num_sampled=negative_samples, num_classes=vocabulary_size)) 95 | tf.summary.scalar("NCE_loss", loss) 96 | 97 | # Learning rate decay 98 | global_step = tf.Variable(0, trainable=False) 99 | learningRate = tf.train.exponential_decay(learning_rate=0.1, 100 | global_step=global_step, 101 | decay_steps=1000, 102 | decay_rate=0.95, 103 | staircase=True) 104 | train_step = tf.train.GradientDescentOptimizer(learningRate).minimize(loss) 105 | merged = tf.summary.merge_all() 106 | 107 | with tf.Session() as sess: 108 | train_writer = tf.summary.FileWriter(LOG_DIR, 109 | graph=tf.get_default_graph()) 110 | saver = tf.train.Saver() 111 | 112 | with open(os.path.join(LOG_DIR, 'metadata.tsv'), "w") as metadata: 113 | metadata.write('Name\tClass\n') 114 | for k, v in index2word_map.items(): 115 | metadata.write('%s\t%d\n' % (v, k)) 116 | 117 | config = projector.ProjectorConfig() 118 | embedding = config.embeddings.add() 119 | embedding.tensor_name = embeddings.name 120 | # Link this tensor to its metadata file (e.g. labels). 121 | embedding.metadata_path = os.path.join(LOG_DIR, 'metadata.tsv') 122 | projector.visualize_embeddings(train_writer, config) 123 | 124 | tf.global_variables_initializer().run() 125 | 126 | for step in range(1000): 127 | x_batch, y_batch = get_skipgram_batch(batch_size) 128 | summary, _ = sess.run([merged, train_step], 129 | feed_dict={train_inputs: x_batch, 130 | train_labels: y_batch}) 131 | train_writer.add_summary(summary, step) 132 | 133 | if step % 100 == 0: 134 | saver.save(sess, os.path.join(LOG_DIR, "w2v_model.ckpt"), step) 135 | loss_value = sess.run(loss, 136 | feed_dict={train_inputs: x_batch, 137 | train_labels: y_batch}) 138 | print("Loss at %d: %.5f" % (step, loss_value)) 139 | 140 | # Normalize embeddings before using 141 | norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) 142 | normalized_embeddings = embeddings / norm 143 | normalized_embeddings_matrix = sess.run(normalized_embeddings) 144 | 145 | ref_word = normalized_embeddings_matrix[word2index_map["one"]] 146 | 147 | cosine_dists = np.dot(normalized_embeddings_matrix, ref_word) 148 | ff = np.argsort(cosine_dists)[::-1][1:10] 149 | for f in ff: 150 | print(index2word_map[f]) 151 | print(cosine_dists[f]) 152 | -------------------------------------------------------------------------------- /07__abstractions/Chapter7.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Chapter 7" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": { 14 | "collapsed": true 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import tensorflow as tf\n", 19 | "from tensorflow.contrib import learn" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "### Linear Regression For the Boston Housing dataset" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": { 33 | "collapsed": true 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "from sklearn import datasets, metrics, preprocessing\n", 38 | "boston = datasets.load_boston()\n", 39 | "x_data = preprocessing.StandardScaler().fit_transform(boston.data)\n", 40 | "y_data = boston.target" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Native TensorFlow" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": false 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "x = tf.placeholder(tf.float64,shape=(None,13))\n", 59 | "y_true = tf.placeholder(tf.float64,shape=(None))\n", 60 | "\n", 61 | "with tf.name_scope('inference') as scope:\n", 62 | " w = tf.Variable(tf.zeros([1,13],dtype=tf.float64,name='weights'))\n", 63 | " b = tf.Variable(0,dtype=tf.float64,name='bias')\n", 64 | " y_pred = tf.matmul(w,tf.transpose(x)) + b\n", 65 | "\n", 66 | "with tf.name_scope('loss') as scope:\n", 67 | " loss = tf.reduce_mean(tf.square(y_true-y_pred))\n", 68 | "\n", 69 | "with tf.name_scope('train') as scope:\n", 70 | " learning_rate = 0.1\n", 71 | " optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n", 72 | " train = optimizer.minimize(loss)\n", 73 | "\n", 74 | "# Before starting, initialize the variables. We will 'run' this first.\n", 75 | "init = tf.global_variables_initializer()\n", 76 | "with tf.Session() as sess:\n", 77 | " sess.run(init) \n", 78 | " for step in range(200):\n", 79 | " sess.run(train,{x: x_data, y_true: y_data})\n", 80 | " \n", 81 | " MSE = sess.run(loss,{x: x_data, y_true: y_data})\n", 82 | "print(MSE)" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "### contirb.learn" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "metadata": { 96 | "collapsed": false 97 | }, 98 | "outputs": [], 99 | "source": [ 100 | "NUM_STEPS = 200\n", 101 | "MINIBATCH_SIZE = 506\n", 102 | "\n", 103 | "feature_columns = learn.infer_real_valued_columns_from_input(x_data)\n", 104 | "\n", 105 | "reg = learn.LinearRegressor(\n", 106 | " feature_columns=feature_columns,\n", 107 | " optimizer=tf.train.GradientDescentOptimizer(\n", 108 | " learning_rate=0.1)\n", 109 | " )\n", 110 | "\n", 111 | "reg.fit(x_data, boston.target, steps=NUM_STEPS, \n", 112 | " batch_size=MINIBATCH_SIZE)\n", 113 | "\n", 114 | "MSE = reg.evaluate(x_data, boston.target, steps=1)\n", 115 | "\n", 116 | "print(MSE)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "### MNIST with contrib.learn DNN classifier" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": { 130 | "collapsed": false, 131 | "scrolled": true 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "import sys\n", 136 | "import numpy as np\n", 137 | "from tensorflow.examples.tutorials.mnist import input_data\n", 138 | "DATA_DIR = '/tmp/data' if not 'win32' in sys.platform else \"c:\\\\tmp\\\\data\"\n", 139 | "data = input_data.read_data_sets(DATA_DIR, one_hot=False)\n", 140 | "x_data, y_data = data.train.images,data.train.labels.astype(np.int32)\n", 141 | "x_test, y_test = data.test.images,data.test.labels.astype(np.int32)\n", 142 | "\n", 143 | "\n", 144 | "NUM_STEPS = 2000\n", 145 | "MINIBATCH_SIZE = 128\n", 146 | "\n", 147 | "feature_columns = learn.infer_real_valued_columns_from_input(x_data)\n", 148 | "\n", 149 | "dnn = learn.DNNClassifier(\n", 150 | " feature_columns=feature_columns,\n", 151 | " hidden_units=[200],\n", 152 | " n_classes=10,\n", 153 | " optimizer=tf.train.ProximalAdagradOptimizer(\n", 154 | " learning_rate=0.2)\n", 155 | " )\n", 156 | "\n", 157 | "dnn.fit(x=x_data,y=y_data, steps=NUM_STEPS,\n", 158 | " batch_size=MINIBATCH_SIZE)\n", 159 | "\n", 160 | "test_acc = dnn.evaluate(x=x_test,y=y_test, steps=1)[\"accuracy\"]\n", 161 | "print('test accuracy: {}'.format(test_acc))" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": {}, 167 | "source": [ 168 | "### Predict and plot confusion matrix" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "metadata": { 175 | "collapsed": false 176 | }, 177 | "outputs": [], 178 | "source": [ 179 | "import matplotlib.pyplot as plt\n", 180 | "def plot_confusion_matrix(cm, classes,\n", 181 | " normalize=False,\n", 182 | " title='Confusion matrix',\n", 183 | " cmap=plt.cm.Blues):\n", 184 | " \"\"\"\n", 185 | " This function prints and plots the confusion matrix.\n", 186 | " Normalization can be applied by setting `normalize=True`.\n", 187 | " \"\"\"\n", 188 | " plt.imshow(cm, interpolation='nearest', cmap=cmap,aspect='auto')\n", 189 | " plt.title(title)\n", 190 | " plt.colorbar()\n", 191 | " tick_marks = np.arange(len(classes))\n", 192 | " plt.xticks(tick_marks, classes, rotation=45)\n", 193 | " plt.yticks(tick_marks, classes)\n", 194 | "\n", 195 | " thresh = cm.max() / 2.\n", 196 | " for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n", 197 | " plt.text(j, i, cm[i, j],\n", 198 | " horizontalalignment=\"center\",\n", 199 | " color=\"white\" if cm[i, j] > thresh else \"black\")\n", 200 | "\n", 201 | " plt.tight_layout()\n", 202 | " plt.ylabel('True label')\n", 203 | " plt.xlabel('Predicted label')\n", 204 | " \n", 205 | " \n", 206 | " plt.savefig('confusion_mat.png', bbox_inches='tight', format='png', dpi=300, pad_inches=0,transparent=True)\n", 207 | "\n", 208 | " \n", 209 | " plt.show()" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": { 216 | "collapsed": false 217 | }, 218 | "outputs": [], 219 | "source": [ 220 | "from sklearn.metrics import confusion_matrix\n", 221 | "import itertools\n", 222 | "\n", 223 | "y_pred = dnn.predict(x=x_test,as_iterable=False)\n", 224 | "class_names = ['0','1','2','3','4','5','6','7','8','9'] \n", 225 | "cnf_matrix = confusion_matrix(y_test, y_pred)\n", 226 | "\n", 227 | "plot_confusion_matrix(cnf_matrix, class_names)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": {}, 233 | "source": [ 234 | "### Generate example categorical data" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "metadata": { 241 | "collapsed": false 242 | }, 243 | "outputs": [], 244 | "source": [ 245 | "import numpy as np\n", 246 | "import pandas as pd\n", 247 | "N = 10000\n", 248 | "\n", 249 | "weight = np.random.randn(N)*5+70\n", 250 | "spec_id = np.random.randint(0,3,N)\n", 251 | "bias = [0.9,1,1.1]\n", 252 | "height = np.array([weight[i]/100 + bias[b] for i,b in enumerate(spec_id)])\n", 253 | "spec_name = ['Goblin','Human','ManBears']\n", 254 | "spec = [spec_name[s] for s in spec_id]" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "### plot and create data frame" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "metadata": { 268 | "collapsed": false, 269 | "scrolled": true 270 | }, 271 | "outputs": [], 272 | "source": [ 273 | "import matplotlib.pyplot as plt\n", 274 | "\n", 275 | "N = 10000\n", 276 | "\n", 277 | "weight = np.random.randn(N)*5+70\n", 278 | "spec_id = np.random.randint(0,3,N)\n", 279 | "bias = [0.9,1,1.1]\n", 280 | "height = np.array([weight[i]/100 + bias[b] for i,b in enumerate(spec_id)])\n", 281 | "spec_name = ['Goblin','Human','ManBears']\n", 282 | "spec = [spec_name[s] for s in spec_id]\n", 283 | "\n", 284 | "\n", 285 | "\n", 286 | "\n", 287 | "colors = ['r','b','g']\n", 288 | "f,axarr = plt.subplots(1,2,figsize = [7,3])\n", 289 | "ax = axarr[0]\n", 290 | "for ii in range(3):\n", 291 | " ax.hist(height[spec_id == ii],50,color=colors[ii],alpha=0.5)\n", 292 | " ax.set_xlabel('Height')\n", 293 | " ax.set_ylabel('Frequency')\n", 294 | " ax.set_title('Heights distribution')\n", 295 | "# ax.legend(['Goblins','Humans','ManBears'],loc=2, shadow=True,prop={'size':6})\n", 296 | "height = height + np.random.randn(N)*0.015\n", 297 | "ax.text(1.42,150,'Goblins')\n", 298 | "ax.text(1.63,210,'Humans')\n", 299 | "ax.text(1.85,150,'ManBears')\n", 300 | "\n", 301 | "ax.set_ylim([0,260])\n", 302 | "ax.set_xlim([1.38,2.05])\n", 303 | "\n", 304 | "df = pd.DataFrame({'Species':spec,'Weight':weight,'Height':height})\n", 305 | "\n", 306 | "\n", 307 | "ax = axarr[1]\n", 308 | "\n", 309 | "ax.plot(df['Height'],df['Weight'],'o',alpha=0.3,mfc='w',mec='b')\n", 310 | "ax.set_xlabel('Weight')\n", 311 | "ax.set_ylabel('Height')\n", 312 | "ax.set_title('Heights vs. Weights')\n", 313 | " \n", 314 | "plt.tight_layout()\n", 315 | "plt.savefig('test.png', bbox_inches='tight', format='png', dpi=300)\n", 316 | "\n", 317 | "plt.show()" 318 | ] 319 | }, 320 | { 321 | "cell_type": "markdown", 322 | "metadata": {}, 323 | "source": [ 324 | "### Estimate weights with contrib.learn using Feature columns" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": null, 330 | "metadata": { 331 | "collapsed": true 332 | }, 333 | "outputs": [], 334 | "source": [ 335 | "def input_fn(df):\n", 336 | " feature_cols = {}\n", 337 | " feature_cols['Weight'] = tf.constant(df['Weight'].values)\n", 338 | " \n", 339 | " feature_cols['Species'] = tf.SparseTensor(\n", 340 | " indices=[[i, 0] for i in range(df['Species'].size)],\n", 341 | " values=df['Species'].values,\n", 342 | " dense_shape=[df['Species'].size, 1])\n", 343 | " \n", 344 | " labels = tf.constant(df['Height'].values)\n", 345 | "\n", 346 | " return feature_cols, labels" 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": null, 352 | "metadata": { 353 | "collapsed": false 354 | }, 355 | "outputs": [], 356 | "source": [ 357 | "from tensorflow.contrib import layers\n", 358 | "Weight = layers.real_valued_column(\"Weight\")\n", 359 | "\n", 360 | "Species = layers.sparse_column_with_keys(\n", 361 | " column_name=\"Species\", keys=['Goblin','Human','ManBears'])\n", 362 | "\n", 363 | "reg = learn.LinearRegressor(feature_columns=[Weight,Species])\n", 364 | "reg.fit(input_fn=lambda:input_fn(df), steps=50000)" 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": null, 370 | "metadata": { 371 | "collapsed": false 372 | }, 373 | "outputs": [], 374 | "source": [ 375 | "w_w = reg.get_variable_value('linear/Weight/weight')\n", 376 | "print('Estimation for Weight: {}'.format(w_w))\n", 377 | "\n", 378 | "s_w = reg.get_variable_value('linear/Species/weights')\n", 379 | "b = reg.get_variable_value('linear/bias_weight')\n", 380 | "print('Estimation for Species: {}'.format(s_w + b))" 381 | ] 382 | }, 383 | { 384 | "cell_type": "markdown", 385 | "metadata": {}, 386 | "source": [ 387 | "### Create custom CNN Estimator " 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": null, 393 | "metadata": { 394 | "collapsed": true 395 | }, 396 | "outputs": [], 397 | "source": [ 398 | "def model_fn(x, target, mode, params):\n", 399 | " y_ = tf.cast(target, tf.float32)\n", 400 | " x_image = tf.reshape(x, [-1, 28, 28, 1])\n", 401 | "\n", 402 | " # Conv layer 1\n", 403 | " conv1 = layers.convolution2d(x_image, 32, [5,5],\n", 404 | " activation_fn=tf.nn.relu,\n", 405 | " biases_initializer=tf.constant_initializer(0.1),\n", 406 | " weights_initializer=tf.truncated_normal_initializer(stddev=0.1))\n", 407 | " pool1 = layers.max_pool2d(conv1, [2,2])\n", 408 | "\n", 409 | " # Conv layer 2\n", 410 | " conv2 = layers.convolution2d(pool1, 64, [5,5],\n", 411 | " activation_fn=tf.nn.relu,\n", 412 | " biases_initializer=tf.constant_initializer(0.1),\n", 413 | " weights_initializer=tf.truncated_normal_initializer(stddev=0.1))\n", 414 | " pool2 = layers.max_pool2d(conv2, [2,2])\n", 415 | "\n", 416 | " # FC layer\n", 417 | " pool2_flat = tf.reshape(pool2, [-1, 7*7*64])\n", 418 | " fc1 = layers.fully_connected(pool2_flat, 1024,\n", 419 | " activation_fn=tf.nn.relu,\n", 420 | " biases_initializer=tf.constant_initializer(0.1),\n", 421 | " weights_initializer=tf.truncated_normal_initializer(stddev=0.1))\n", 422 | " fc1_drop = layers.dropout(fc1, keep_prob=params[\"dropout\"],\n", 423 | " is_training=(mode == 'train'))\n", 424 | "\n", 425 | " # readout layer\n", 426 | " y_conv = layers.fully_connected(fc1_drop, 10, activation_fn=None)\n", 427 | "\n", 428 | " cross_entropy = tf.reduce_mean(\n", 429 | " tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_))\n", 430 | " train_op = tf.contrib.layers.optimize_loss(\n", 431 | " loss=cross_entropy,\n", 432 | " global_step=tf.contrib.framework.get_global_step(),\n", 433 | " learning_rate=params[\"learning_rate\"],\n", 434 | " optimizer=\"Adam\")\n", 435 | "\n", 436 | " predictions = tf.argmax(y_conv, 1)\n", 437 | " return predictions, cross_entropy, train_op" 438 | ] 439 | }, 440 | { 441 | "cell_type": "code", 442 | "execution_count": null, 443 | "metadata": { 444 | "collapsed": false, 445 | "scrolled": true 446 | }, 447 | "outputs": [], 448 | "source": [ 449 | "from tensorflow.contrib import layers\n", 450 | "\n", 451 | "data = input_data.read_data_sets(DATA_DIR, one_hot=True)\n", 452 | "x_data, y_data = data.train.images,np.int32(data.train.labels)\n", 453 | "tf.cast(x_data,tf.float32)\n", 454 | "tf.cast(y_data,tf.float32)\n", 455 | "\n", 456 | "model_params = {\"learning_rate\": 1e-4, \"dropout\": 0.5}\n", 457 | "\n", 458 | "CNN = tf.contrib.learn.Estimator(\n", 459 | " model_fn=model_fn, params=model_params)\n", 460 | "\n", 461 | "print(\"Starting training for %s steps max\" % 5000)\n", 462 | "CNN.fit(x=data.train.images,\n", 463 | " y=data.train.labels, batch_size=50,\n", 464 | " max_steps=5000)\n", 465 | "\n", 466 | "test_acc = 0\n", 467 | "for ii in range(5):\n", 468 | " batch = data.test.next_batch(2000)\n", 469 | " predictions = list(CNN.predict(batch[0], as_iterable=True))\n", 470 | " test_acc = test_acc + (np.argmax(batch[1],1) == predictions).mean()\n", 471 | "\n", 472 | "print(test_acc/5)" 473 | ] 474 | }, 475 | { 476 | "cell_type": "markdown", 477 | "metadata": {}, 478 | "source": [ 479 | "### TFLearn\n", 480 | "### CNN" 481 | ] 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": null, 486 | "metadata": { 487 | "collapsed": false, 488 | "scrolled": true 489 | }, 490 | "outputs": [], 491 | "source": [ 492 | "import tflearn\n", 493 | "from tflearn.layers.core import input_data, dropout, fully_connected\n", 494 | "from tflearn.layers.conv import conv_2d, max_pool_2d\n", 495 | "from tflearn.layers.normalization import local_response_normalization\n", 496 | "from tflearn.layers.estimator import regression\n", 497 | "\n", 498 | "# Data loading and basic trasformations\n", 499 | "import tflearn.datasets.mnist as mnist\n", 500 | "X, Y, X_test, Y_test = mnist.load_data(one_hot=True)\n", 501 | "X = X.reshape([-1, 28, 28, 1])\n", 502 | "X_test = X_test.reshape([-1, 28, 28, 1])\n", 503 | "\n", 504 | "# Building the network\n", 505 | "CNN = input_data(shape=[None, 28, 28, 1], name='input')\n", 506 | "CNN = conv_2d(CNN, 32, 5, activation='relu', regularizer=\"L2\")\n", 507 | "CNN = max_pool_2d(CNN, 2)\n", 508 | "CNN = local_response_normalization(CNN)\n", 509 | "CNN = conv_2d(CNN, 64, 5, activation='relu', regularizer=\"L2\")\n", 510 | "CNN = max_pool_2d(CNN, 2)\n", 511 | "CNN = local_response_normalization(CNN)\n", 512 | "CNN = fully_connected(CNN, 1024, activation=None)\n", 513 | "CNN = dropout(CNN, 0.5)\n", 514 | "CNN = fully_connected(CNN, 10, activation='softmax')\n", 515 | "CNN = regression(CNN, optimizer='adam', learning_rate=0.0001,\n", 516 | " loss='categorical_crossentropy', name='target')\n", 517 | "\n", 518 | "# Training the network\n", 519 | "model = tflearn.DNN(CNN,tensorboard_verbose=0,tensorboard_dir = 'MNIST_tflearn_board/',\\\n", 520 | " checkpoint_path = 'MNIST_tflearn_checkpoints/checkpoint')\n", 521 | "model.fit({'input': X}, {'target': Y}, n_epoch=3, \n", 522 | " validation_set=({'input': X_test}, {'target': Y_test}),\n", 523 | " snapshot_step=1000,show_metric=True, run_id='convnet_mnist')" 524 | ] 525 | }, 526 | { 527 | "cell_type": "code", 528 | "execution_count": null, 529 | "metadata": { 530 | "collapsed": true 531 | }, 532 | "outputs": [], 533 | "source": [ 534 | "evaluation = model.evaluate({'input': X_test},{'target': Y_test})\n", 535 | "print(evaluation)" 536 | ] 537 | }, 538 | { 539 | "cell_type": "code", 540 | "execution_count": null, 541 | "metadata": { 542 | "collapsed": true 543 | }, 544 | "outputs": [], 545 | "source": [ 546 | "pred = model.predict({'input': X_test})\n", 547 | "print((np.argmax(testY,1)==np.argmax(pred,1)).mean())" 548 | ] 549 | }, 550 | { 551 | "cell_type": "markdown", 552 | "metadata": {}, 553 | "source": [ 554 | "### RNN" 555 | ] 556 | }, 557 | { 558 | "cell_type": "code", 559 | "execution_count": null, 560 | "metadata": { 561 | "collapsed": false 562 | }, 563 | "outputs": [], 564 | "source": [ 565 | "from tflearn.data_utils import to_categorical, pad_sequences\n", 566 | "from tflearn.datasets import imdb\n", 567 | "\n", 568 | "# Load data\n", 569 | "train, test, _ = imdb.load_data(path='imdb.pkl', n_words=10000,\n", 570 | " valid_portion=0.1)\n", 571 | "X_train, Y_train = train\n", 572 | "X_test, Y_test = test\n", 573 | "\n", 574 | "# Sequence padding and Converting labels to binary vectors\n", 575 | "X_train = pad_sequences(X_train, maxlen=100, value=0.)\n", 576 | "X_test = pad_sequences(X_test, maxlen=100, value=0.)\n", 577 | "Y_train = to_categorical(Y_train, nb_classes=2)\n", 578 | "Y_test = to_categorical(Y_test, nb_classes=2)\n", 579 | "\n", 580 | "# Building a LSTM network\n", 581 | "RNN = tflearn.input_data([None, 100])\n", 582 | "RNN = tflearn.embedding(RNN, input_dim=10000, output_dim=128)\n", 583 | "\n", 584 | "RNN = tflearn.lstm(RNN, 128, dropout=0.8)\n", 585 | "RNN = tflearn.fully_connected(RNN, 2, activation='softmax')\n", 586 | "RNN = tflearn.regression(RNN, optimizer='adam', learning_rate=0.001,\n", 587 | " loss='categorical_crossentropy')\n", 588 | "\n", 589 | "# Training the network\n", 590 | "model = tflearn.DNN(RNN, tensorboard_verbose=0)\n", 591 | "model.fit(X_train, Y_train, validation_set=(X_test, Y_test),\n", 592 | " show_metric=True, batch_size=32)" 593 | ] 594 | }, 595 | { 596 | "cell_type": "markdown", 597 | "metadata": {}, 598 | "source": [ 599 | "### Keras\n", 600 | "### Autoencoding for denoising CIFAR10" 601 | ] 602 | }, 603 | { 604 | "cell_type": "code", 605 | "execution_count": null, 606 | "metadata": { 607 | "collapsed": false 608 | }, 609 | "outputs": [], 610 | "source": [ 611 | "import keras\n", 612 | "from keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D\n", 613 | "from keras.models import Model\n", 614 | "from keras.callbacks import TensorBoard, ModelCheckpoint\n", 615 | "from keras.datasets import cifar10\n", 616 | "import numpy as np\n", 617 | "\n", 618 | "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n", 619 | "x_train = x_train[np.where(y_train==1)[0],:,:,:]\n", 620 | "x_test = x_test[np.where(y_test==1)[0],:,:,:]\n", 621 | "\n", 622 | "x_train = x_train.astype('float32') / 255.\n", 623 | "x_test = x_test.astype('float32') / 255.\n", 624 | "\n", 625 | "x_train_n = x_train + 0.5 *\\\n", 626 | " np.random.normal(loc=0.0, scale=0.4, size=x_train.shape) \n", 627 | "\n", 628 | "x_test_n = x_test + 0.5 *\\\n", 629 | " np.random.normal(loc=0.0, scale=0.4, size=x_test.shape) \n", 630 | "\n", 631 | "x_train_n = np.clip(x_train_n, 0., 1.)\n", 632 | "x_test_n = np.clip(x_test_n, 0., 1.)\n", 633 | "\n", 634 | "inp_img = Input(shape=(32, 32, 3)) \n", 635 | "\n", 636 | "img= Conv2D(32, (3, 3), activation='relu', padding='same')(inp_img)\n", 637 | "img = MaxPooling2D((2, 2), padding='same')(img)\n", 638 | "img = Conv2D(32, (3, 3), activation='relu', padding='same')(img)\n", 639 | "img = UpSampling2D((2, 2))(img)\n", 640 | "decoded = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(img)\n", 641 | "\n", 642 | "autoencoder = Model(inp_img, decoded)\n", 643 | "\n", 644 | "autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy')\n", 645 | "\n", 646 | "tensorboard = TensorBoard(log_dir='./models/autoencoder',\\\n", 647 | " histogram_freq=0, write_graph=True, write_images=True)\n", 648 | "model_saver = ModelCheckpoint(\n", 649 | " filepath='./models/autoencoder/autoencoder_model',\\\n", 650 | " verbose=0, period=2)\n", 651 | "\n", 652 | "autoencoder.fit(x_train_n, x_train,\n", 653 | " epochs=10,\n", 654 | " batch_size=64,\n", 655 | " shuffle=True,\n", 656 | " validation_data=(x_test_n, x_test),\n", 657 | " callbacks=[tensorboard, model_saver])" 658 | ] 659 | }, 660 | { 661 | "cell_type": "code", 662 | "execution_count": null, 663 | "metadata": { 664 | "collapsed": false, 665 | "scrolled": false 666 | }, 667 | "outputs": [], 668 | "source": [ 669 | "import matplotlib.pyplot as plt\n", 670 | "% matplotlib inline\n", 671 | "\n", 672 | "n_imgs = 10\n", 673 | "f,axarr = plt.subplots(2,n_imgs,figsize=[20,5])\n", 674 | "decoded_imgs = autoencoder.predict(x_test_n)\n", 675 | "\n", 676 | "for i in range(n_imgs):\n", 677 | " \n", 678 | " ax = axarr[0,i]\n", 679 | " ax.get_yaxis().set_visible(False)\n", 680 | " ax.imshow(x_test_n[i,:,:,:])\n", 681 | " plt.gray()\n", 682 | " ax.get_xaxis().set_visible(False)\n", 683 | " ax.get_yaxis().set_visible(False)\n", 684 | " \n", 685 | " ax = axarr[1,i]\n", 686 | " ax.get_yaxis().set_visible(False)\n", 687 | " ax.imshow(decoded_imgs[i,:,:,:])\n", 688 | " plt.gray()\n", 689 | " ax.get_xaxis().set_visible(False)\n", 690 | " ax.get_yaxis().set_visible(False)\n", 691 | " \n", 692 | "plt.tight_layout()\n", 693 | "plt.show()" 694 | ] 695 | }, 696 | { 697 | "cell_type": "markdown", 698 | "metadata": { 699 | "collapsed": true 700 | }, 701 | "source": [ 702 | "### Load trained model" 703 | ] 704 | }, 705 | { 706 | "cell_type": "code", 707 | "execution_count": null, 708 | "metadata": { 709 | "collapsed": true 710 | }, 711 | "outputs": [], 712 | "source": [ 713 | "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n", 714 | "x_train = x_train[np.where(y_train==1)[0],:,:,:]\n", 715 | "x_test = x_test[np.where(y_test==1)[0],:,:,:]\n", 716 | "\n", 717 | "x_train = x_train.astype('float32') / 255.\n", 718 | "x_test = x_test.astype('float32') / 255.\n", 719 | "\n", 720 | "x_train_n = x_train + 0.5 *\\\n", 721 | " np.random.normal(loc=0.0, scale=0.4, size=x_train.shape) \n", 722 | "\n", 723 | "x_test_n = x_test + 0.5 *\\\n", 724 | " np.random.normal(loc=0.0, scale=0.4, size=x_test.shape) \n", 725 | "\n", 726 | "x_train_n = np.clip(x_train_n, 0., 1.)\n", 727 | "x_test_n = np.clip(x_test_n, 0., 1.)" 728 | ] 729 | }, 730 | { 731 | "cell_type": "code", 732 | "execution_count": null, 733 | "metadata": { 734 | "collapsed": false 735 | }, 736 | "outputs": [], 737 | "source": [ 738 | "import keras\n", 739 | "from keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D\n", 740 | "from keras.models import Model\n", 741 | "from keras.callbacks import TensorBoard, ModelCheckpoint\n", 742 | "from keras.datasets import cifar10\n", 743 | "import numpy as np\n", 744 | "\n", 745 | "inp_img = Input(shape=(32, 32, 3)) \n", 746 | "img= Conv2D(32, (3, 3), activation='relu', padding='same')(inp_img)\n", 747 | "img = MaxPooling2D((2, 2), padding='same')(img)\n", 748 | "img = Conv2D(32, (3, 3), activation='relu', padding='same')(img)\n", 749 | "img = UpSampling2D((2, 2))(img)\n", 750 | "decoded = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(img)\n", 751 | "\n", 752 | "autoencoder = Model(inp_img, decoded)\n", 753 | "Model.load_weights(autoencoder,'./models/autoencoder/autoencoder_model')" 754 | ] 755 | }, 756 | { 757 | "cell_type": "code", 758 | "execution_count": null, 759 | "metadata": { 760 | "collapsed": false 761 | }, 762 | "outputs": [], 763 | "source": [ 764 | "import matplotlib.pyplot as plt\n", 765 | "% matplotlib inline\n", 766 | "\n", 767 | "n_imgs = 10\n", 768 | "f,axarr = plt.subplots(2,n_imgs,figsize=[20,5])\n", 769 | "decoded_imgs = autoencoder.predict(x_test_n)\n", 770 | "\n", 771 | "for i in range(n_imgs):\n", 772 | " \n", 773 | " ax = axarr[0,i]\n", 774 | " ax.get_yaxis().set_visible(False)\n", 775 | " ax.imshow(x_test_n[i,:,:,:])\n", 776 | " plt.gray()\n", 777 | " ax.get_xaxis().set_visible(False)\n", 778 | " ax.get_yaxis().set_visible(False)\n", 779 | " \n", 780 | " ax = axarr[1,i]\n", 781 | " ax.get_yaxis().set_visible(False)\n", 782 | " ax.imshow(decoded_imgs[i,:,:,:])\n", 783 | " plt.gray()\n", 784 | " ax.get_xaxis().set_visible(False)\n", 785 | " ax.get_yaxis().set_visible(False)\n", 786 | " \n", 787 | "plt.tight_layout()\n", 788 | "plt.show()" 789 | ] 790 | }, 791 | { 792 | "cell_type": "markdown", 793 | "metadata": {}, 794 | "source": [ 795 | "### TF-Slim - Pretrained VGG16" 796 | ] 797 | }, 798 | { 799 | "cell_type": "code", 800 | "execution_count": null, 801 | "metadata": { 802 | "collapsed": false 803 | }, 804 | "outputs": [], 805 | "source": [ 806 | "from tensorflow.contrib import slim\n", 807 | "sys.path.append(\"/home/itay/git/models/slim\")\n", 808 | "\n", 809 | "import sys\n", 810 | "from datasets import dataset_utils\n", 811 | "import tensorflow as tf\n", 812 | "import urllib2\n", 813 | "from nets import vgg\n", 814 | "from preprocessing import vgg_preprocessing\n", 815 | "import os\n", 816 | "\n", 817 | "target_dir = '/home/itay/git/checkpoints'\n", 818 | "\n", 819 | "url = (\"http://54.68.5.226/car.jpg\")\n", 820 | "\n", 821 | "im_as_string = urllib2.urlopen(url).read() \n", 822 | "im = tf.image.decode_jpeg(im_as_string, channels=3)\n", 823 | "\n", 824 | "image_size = vgg.vgg_16.default_image_size\n", 825 | "\n", 826 | "processed_im = vgg_preprocessing.preprocess_image(im,\n", 827 | " image_size,\n", 828 | " image_size,\n", 829 | " is_training=False)\n", 830 | "\n", 831 | "processed_images = tf.expand_dims(processed_im, 0)\n", 832 | "\n", 833 | "with slim.arg_scope(vgg.vgg_arg_scope()):\n", 834 | " logits, _ = vgg.vgg_16(processed_images,\n", 835 | " num_classes=1000,\n", 836 | " is_training=False)\n", 837 | "probabilities = tf.nn.softmax(logits)\n", 838 | "\n", 839 | "def vgg_arg_scope(weight_decay=0.0005):\n", 840 | " with slim.arg_scope([slim.conv2d, slim.fully_connected],\n", 841 | " activation_fn=tf.nn.relu,\n", 842 | " weights_regularizer=slim.l2_regularizer(weight_decay),\n", 843 | " biases_initializer=tf.zeros_initializer):\n", 844 | " with slim.arg_scope([slim.conv2d], padding='SAME') as arg_sc:\n", 845 | " return arg_sc\n", 846 | "\n", 847 | "\n", 848 | "\n", 849 | "load_vars = slim.assign_from_checkpoint_fn(\n", 850 | " os.path.join(target_dir, 'vgg_16.ckpt'),\n", 851 | " slim.get_model_variables('vgg_16'))\n", 852 | "\n", 853 | "\n", 854 | "from datasets import imagenet\n", 855 | "imagenet.create_readable_names_for_imagenet_labels()" 856 | ] 857 | }, 858 | { 859 | "cell_type": "markdown", 860 | "metadata": {}, 861 | "source": [ 862 | "### Infer class and probability" 863 | ] 864 | }, 865 | { 866 | "cell_type": "code", 867 | "execution_count": null, 868 | "metadata": { 869 | "collapsed": false 870 | }, 871 | "outputs": [], 872 | "source": [ 873 | "names = []\n", 874 | "with tf.Session() as sess:\n", 875 | " load_vars(sess) \n", 876 | " network_input, probabilities = sess.run([processed_images,\n", 877 | " probabilities])\n", 878 | " probabilities = probabilities[0, 0:]\n", 879 | " names_ = imagenet.create_readable_names_for_imagenet_labels()\n", 880 | " idxs = np.argsort(-probabilities)[:5]\n", 881 | " probs = probabilities[idxs]\n", 882 | " classes = np.array(names_.values())[idxs+1]\n", 883 | " for c,p in zip(classes,probs):\n", 884 | " print('Class: '+ c + ' |Prob: ' + str(p))" 885 | ] 886 | } 887 | ], 888 | "metadata": { 889 | "kernelspec": { 890 | "display_name": "tf_1_0", 891 | "language": "python", 892 | "name": "tf_1_0" 893 | }, 894 | "language_info": { 895 | "codemirror_mode": { 896 | "name": "ipython", 897 | "version": 2.0 898 | }, 899 | "file_extension": ".py", 900 | "mimetype": "text/x-python", 901 | "name": "python", 902 | "nbconvert_exporter": "python", 903 | "pygments_lexer": "ipython2", 904 | "version": "2.7.12" 905 | } 906 | }, 907 | "nbformat": 4, 908 | "nbformat_minor": 0 909 | } -------------------------------------------------------------------------------- /08__queues_threads/queue_basic.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Sat Jan 21 23:52:54 2017 4 | 5 | @author: tomhope 6 | """ 7 | from __future__ import print_function 8 | 9 | import tensorflow as tf 10 | 11 | import threading 12 | import time 13 | 14 | sess = tf.InteractiveSession() 15 | 16 | queue1 = tf.FIFOQueue(capacity=10, dtypes=[tf.string]) 17 | 18 | enque_op = queue1.enqueue(["F"]) 19 | # size is 0 before run 20 | sess.run(queue1.size()) 21 | enque_op.run() 22 | sess.run(queue1.size()) 23 | 24 | enque_op = queue1.enqueue(["I"]) 25 | enque_op.run() 26 | enque_op = queue1.enqueue(["F"]) 27 | enque_op.run() 28 | enque_op = queue1.enqueue(["O"]) 29 | enque_op.run() 30 | 31 | sess.run(queue1.size()) 32 | 33 | 34 | x = queue1.dequeue() 35 | x.eval() 36 | x.eval() 37 | x.eval() 38 | x.eval() 39 | 40 | # Hangs forever if queue empty if running INTERACTIVELY 41 | 42 | # for dequeue many, need to specify shapes in advance... 43 | queue1 = tf.FIFOQueue(capacity=10, dtypes=[tf.string], shapes=[()]) 44 | # .... 45 | inputs = queue1.dequeue_many(4) 46 | 47 | inputs.eval() 48 | 49 | 50 | # single queue, but execute sess.run calls in parallel... 51 | gen_random_normal = tf.random_normal(shape=()) 52 | queue = tf.FIFOQueue(capacity=100, dtypes=[tf.float32], shapes=()) 53 | enque = queue.enqueue(gen_random_normal) 54 | 55 | 56 | def add(): 57 | for i in range(10): 58 | sess.run(enque) 59 | 60 | 61 | # Create 10 threads that run add() 62 | threads = [threading.Thread(target=add, args=()) for i in range(10)] 63 | for t in threads: 64 | t.start() 65 | print(sess.run(queue.size())) 66 | time.sleep(0.01) 67 | print(sess.run(queue.size())) 68 | time.sleep(0.01) 69 | print(sess.run(queue.size())) 70 | 71 | 72 | x = queue.dequeue_many(10) 73 | print(x.eval()) 74 | sess.run(queue.size()) 75 | 76 | 77 | # A coordinator for threads. 78 | # a simple mechanism to coordinate the termination of a set of threads 79 | 80 | gen_random_normal = tf.random_normal(shape=()) 81 | queue = tf.FIFOQueue(capacity=100, dtypes=[tf.float32], shapes=()) 82 | enque = queue.enqueue(gen_random_normal) 83 | 84 | 85 | def add(coord, i): 86 | while not coord.should_stop(): 87 | sess.run(enque) 88 | if i == 1: 89 | coord.request_stop() 90 | 91 | 92 | coord = tf.train.Coordinator() 93 | threads = [threading.Thread(target=add, args=(coord, i)) for i in range(10)] 94 | coord.join(threads) 95 | 96 | for t in threads: 97 | t.start() 98 | 99 | print(sess.run(queue.size())) 100 | time.sleep(0.01) 101 | print(sess.run(queue.size())) 102 | time.sleep(0.01) 103 | print(sess.run(queue.size())) 104 | 105 | 106 | gen_random_normal = tf.random_normal(shape=()) 107 | queue = tf.RandomShuffleQueue(capacity=100, dtypes=[tf.float32], 108 | min_after_dequeue=1) 109 | enqueue_op = queue.enqueue(gen_random_normal) 110 | 111 | qr = tf.train.QueueRunner(queue, [enqueue_op] * 4) 112 | coord = tf.train.Coordinator() 113 | enqueue_threads = qr.create_threads(sess, coord=coord, start=True) 114 | coord.request_stop() 115 | coord.join(enqueue_threads) 116 | 117 | print(sess.run(queue.size())) 118 | -------------------------------------------------------------------------------- /08__queues_threads/tfrecords_end_to_end.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | 4 | """ 5 | Created on Thu Jan 26 00:41:43 2017 6 | 7 | @author: tomhope 8 | """ 9 | 10 | import os 11 | import tensorflow as tf 12 | from tensorflow.contrib.learn.python.learn.datasets import mnist 13 | 14 | #### WRITE TFRECORDS # noqa 15 | save_dir = "D:\\mnist" 16 | 17 | # Download data to save_Dir 18 | data_sets = mnist.read_data_sets(save_dir, 19 | dtype=tf.uint8, 20 | reshape=False, 21 | validation_size=1000) 22 | 23 | data_splits = ["train", "test", "validation"] 24 | for d in range(len(data_splits)): 25 | print("saving " + data_splits[d]) 26 | data_set = data_sets[d] 27 | 28 | filename = os.path.join(save_dir, data_splits[d] + '.tfrecords') 29 | writer = tf.python_io.TFRecordWriter(filename) 30 | for index in range(data_set.images.shape[0]): 31 | image = data_set.images[index].tostring() 32 | example = tf.train.Example(features=tf.train.Features(feature={ 33 | 'height': tf.train.Feature(int64_list=tf.train.Int64List( 34 | value=[data_set.images.shape[1]])), 35 | 'width': tf.train.Feature(int64_list=tf.train.Int64List( 36 | value=[data_set.images.shape[2]])), 37 | 'depth': tf.train.Feature(int64_list=tf.train.Int64List( 38 | value=[data_set.images.shape[3]])), 39 | 'label': tf.train.Feature(int64_list=tf.train.Int64List( 40 | value=[int(data_set.labels[index])])), 41 | 'image_raw': tf.train.Feature(bytes_list=tf.train.BytesList( 42 | value=[image]))})) 43 | writer.write(example.SerializeToString()) 44 | writer.close() 45 | 46 | 47 | # READ 48 | NUM_EPOCHS = 10 49 | 50 | filename = os.path.join("D:\\mnist", "train.tfrecords") 51 | 52 | filename_queue = tf.train.string_input_producer( 53 | [filename], num_epochs=NUM_EPOCHS) 54 | 55 | 56 | reader = tf.TFRecordReader() 57 | _, serialized_example = reader.read(filename_queue) 58 | features = tf.parse_single_example( 59 | serialized_example, 60 | features={ 61 | 'image_raw': tf.FixedLenFeature([], tf.string), 62 | 'label': tf.FixedLenFeature([], tf.int64), 63 | }) 64 | 65 | image = tf.decode_raw(features['image_raw'], tf.uint8) 66 | image.set_shape([784]) 67 | 68 | 69 | image = tf.cast(image, tf.float32) * (1. / 255) - 0.5 70 | 71 | label = tf.cast(features['label'], tf.int32) 72 | 73 | 74 | # Shuffle the examples + batch 75 | images_batch, labels_batch = tf.train.shuffle_batch( 76 | [image, label], batch_size=128, 77 | capacity=2000, 78 | min_after_dequeue=1000) 79 | 80 | 81 | W = tf.get_variable("W", [28*28, 10]) 82 | y_pred = tf.matmul(images_batch, W) 83 | loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y_pred, labels=labels_batch) 84 | 85 | loss_mean = tf.reduce_mean(loss) 86 | 87 | train_op = tf.train.AdamOptimizer().minimize(loss) 88 | 89 | sess = tf.Session() 90 | init = tf.global_variables_initializer() 91 | sess.run(init) 92 | init = tf.local_variables_initializer() 93 | sess.run(init) 94 | 95 | # coordinator 96 | coord = tf.train.Coordinator() 97 | try: 98 | step = 0 99 | while not coord.should_stop(): 100 | step += 1 101 | sess.run([train_op]) 102 | if step % 500 == 0: 103 | loss_mean_val = sess.run([loss_mean]) 104 | print(step) 105 | print(loss_mean_val) 106 | except tf.errors.OutOfRangeError: 107 | print('Done training for %d epochs, %d steps.' % (NUM_EPOCHS, step)) 108 | finally: 109 | # When done, ask the threads to stop. 110 | coord.request_stop() 111 | 112 | 113 | # example -- get image,label 114 | img1, lbl1 = sess.run([image, label]) 115 | 116 | # example - get random batch 117 | labels, images = sess.run([labels_batch, images_batch]) 118 | -------------------------------------------------------------------------------- /08__queues_threads/tfrecords_read_write.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Feb 23 15:59:07 2017 4 | 5 | @author: tomhope 6 | """ 7 | 8 | from __future__ import print_function 9 | import os 10 | import tensorflow as tf 11 | from tensorflow.contrib.learn.python.learn.datasets import mnist 12 | import numpy as np 13 | 14 | 15 | save_dir = "D:\\mnist" 16 | 17 | # Download data to save_dir 18 | data_sets = mnist.read_data_sets(save_dir, 19 | dtype=tf.uint8, 20 | reshape=False, 21 | validation_size=1000) 22 | 23 | data_splits = ["train", "test", "validation"] 24 | for d in range(len(data_splits)): 25 | print("saving " + data_splits[d]) 26 | data_set = data_sets[d] 27 | 28 | filename = os.path.join(save_dir, data_splits[d] + '.tfrecords') 29 | writer = tf.python_io.TFRecordWriter(filename) 30 | for index in range(data_set.images.shape[0]): 31 | image = data_set.images[index].tostring() 32 | example = tf.train.Example(features=tf.train.Features( 33 | feature={ 34 | 'height': tf.train.Feature( 35 | int64_list=tf.train.Int64List( 36 | value=[data_set.images.shape[1]])), 37 | 'width': tf.train.Feature( 38 | int64_list=tf.train.Int64List( 39 | value=[data_set.images.shape[2]])), 40 | 'depth': tf.train.Feature( 41 | int64_list=tf.train.Int64List( 42 | value=[data_set.images.shape[3]])), 43 | 'label': tf.train.Feature( 44 | int64_list=tf.train.Int64List( 45 | value=[int(data_set.labels[index])])), 46 | 'image_raw': tf.train.Feature( 47 | bytes_list=tf.train.BytesList( 48 | value=[image]))})) 49 | writer.write(example.SerializeToString()) 50 | writer.close() 51 | 52 | 53 | filename = os.path.join(save_dir, 'train.tfrecords') 54 | record_iterator = tf.python_io.tf_record_iterator(filename) 55 | seralized_img_example = next(record_iterator) 56 | 57 | example = tf.train.Example() 58 | example.ParseFromString(seralized_img_example) 59 | image = example.features.feature['image_raw'].bytes_list.value 60 | label = example.features.feature['label'].int64_list.value[0] 61 | width = example.features.feature['width'].int64_list.value[0] 62 | height = example.features.feature['height'].int64_list.value[0] 63 | 64 | img_flat = np.fromstring(image[0], dtype=np.uint8) 65 | img_reshaped = img_flat.reshape((height, width, -1)) 66 | -------------------------------------------------------------------------------- /09__distributed_tensorflow/distribute-run.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | subprocess.Popen('python distribute.py --job_name="ps" --task_index=0', 3 | shell=True) 4 | subprocess.Popen('python distribute.py --job_name="worker" --task_index=0', 5 | shell=True) 6 | subprocess.Popen('python distribute.py --job_name="worker" --task_index=1', 7 | shell=True) 8 | subprocess.Popen('python distribute.py --job_name="worker" --task_index=2', 9 | shell=True) 10 | -------------------------------------------------------------------------------- /09__distributed_tensorflow/distribute.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.contrib import slim 3 | from tensorflow.examples.tutorials.mnist import input_data 4 | 5 | 6 | BATCH_SIZE = 50 7 | TRAINING_STEPS = 5000 8 | PRINT_EVERY = 100 9 | LOG_DIR = "/tmp/log" 10 | 11 | 12 | parameter_servers = ["localhost:2222"] 13 | workers = ["localhost:2223", 14 | "localhost:2224", 15 | "localhost:2225"] 16 | 17 | cluster = tf.train.ClusterSpec({"ps": parameter_servers, "worker": workers}) 18 | 19 | 20 | tf.app.flags.DEFINE_string("job_name", "", "'ps' / 'worker'") 21 | tf.app.flags.DEFINE_integer("task_index", 0, "Index of task") 22 | FLAGS = tf.app.flags.FLAGS 23 | 24 | 25 | server = tf.train.Server(cluster, 26 | job_name=FLAGS.job_name, 27 | task_index=FLAGS.task_index) 28 | 29 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True) 30 | 31 | 32 | def net(x): 33 | x_image = tf.reshape(x, [-1, 28, 28, 1]) 34 | net = slim.layers.conv2d(x_image, 32, [5, 5], scope='conv1') 35 | net = slim.layers.max_pool2d(net, [2, 2], scope='pool1') 36 | net = slim.layers.conv2d(net, 64, [5, 5], scope='conv2') 37 | net = slim.layers.max_pool2d(net, [2, 2], scope='pool2') 38 | net = slim.layers.flatten(net, scope='flatten') 39 | net = slim.layers.fully_connected(net, 500, scope='fully_connected') 40 | net = slim.layers.fully_connected(net, 10, activation_fn=None, scope='pred') 41 | return net 42 | 43 | 44 | if FLAGS.job_name == "ps": 45 | server.join() 46 | 47 | elif FLAGS.job_name == "worker": 48 | 49 | with tf.device(tf.train.replica_device_setter( 50 | worker_device="/job:worker/task:%d" % FLAGS.task_index, 51 | cluster=cluster)): 52 | 53 | global_step = tf.get_variable('global_step', [], 54 | initializer=tf.constant_initializer(0), 55 | trainable=False) 56 | 57 | x = tf.placeholder(tf.float32, shape=[None, 784], name="x-input") 58 | y_ = tf.placeholder(tf.float32, shape=[None, 10], name="y-input") 59 | y = net(x) 60 | 61 | cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y, 62 | labels=y_)) 63 | 64 | train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy, 65 | global_step=global_step) 66 | 67 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 68 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 69 | 70 | init_op = tf.global_variables_initializer() 71 | 72 | sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0), 73 | logdir=LOG_DIR, 74 | global_step=global_step, 75 | init_op=init_op) 76 | 77 | with sv.managed_session(server.target) as sess: 78 | step = 0 79 | 80 | while not sv.should_stop() and step <= TRAINING_STEPS: 81 | 82 | batch_x, batch_y = mnist.train.next_batch(BATCH_SIZE) 83 | 84 | _, acc, step = sess.run([train_step, accuracy, global_step], 85 | feed_dict={x: batch_x, y_: batch_y}) 86 | 87 | if step % PRINT_EVERY == 0: 88 | print("Worker : {}, Step: {}, Accuracy (batch): {}". 89 | format(FLAGS.task_index, step, acc)) 90 | 91 | test_acc = sess.run(accuracy, 92 | feed_dict={x: mnist.test.images, y_: mnist.test.labels}) 93 | print("Test-Accuracy: {}".format(test_acc)) 94 | 95 | sv.stop() 96 | -------------------------------------------------------------------------------- /10__serving/Chapter10.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "collapsed": true 7 | }, 8 | "source": [ 9 | "# Chapter 10" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Assigning loaded ​weights - Single weight array" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "### Saving the weights of the trained model" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "collapsed": false 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "import tensorflow as tf\n", 35 | "from tensorflow.examples.tutorials.mnist import input_data\n", 36 | "\n", 37 | "DATA_DIR = '/tmp/data'\n", 38 | "NUM_STEPS = 1000\n", 39 | "MINIBATCH_SIZE = 100\n", 40 | "\n", 41 | "\n", 42 | "data = input_data.read_data_sets(DATA_DIR, one_hot=True)\n", 43 | "\n", 44 | "x = tf.placeholder(tf.float32, [None, 784])\n", 45 | "W = tf.Variable(tf.zeros([784, 10]))\n", 46 | "\n", 47 | "y_true = tf.placeholder(tf.float32, [None, 10])\n", 48 | "y_pred = tf.matmul(x, W)\n", 49 | "\n", 50 | "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n", 51 | " logits=y_pred, labels=y_true))\n", 52 | "\n", 53 | "\n", 54 | "gd_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n", 55 | "\n", 56 | "correct_mask = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\n", 57 | "accuracy = tf.reduce_mean(tf.cast(correct_mask, tf.float32))\n", 58 | "\n", 59 | "with tf.Session() as sess:\n", 60 | "\n", 61 | " # Train\n", 62 | " sess.run(tf.global_variables_initializer())\n", 63 | "\n", 64 | " for _ in range(NUM_STEPS):\n", 65 | " batch_xs, batch_ys = data.train.next_batch(MINIBATCH_SIZE)\n", 66 | " sess.run(gd_step, feed_dict={x: batch_xs, y_true: batch_ys})\n", 67 | "\n", 68 | " weights = sess.run(W)\n", 69 | "\n" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": { 76 | "collapsed": false 77 | }, 78 | "outputs": [], 79 | "source": [ 80 | "import numpy as np\n", 81 | "import os \n", 82 | "path = 'tmp//'\n", 83 | "\n", 84 | "np.savez(os.path.join(path, 'weight_storage'), weights)" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "### Load weights" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": { 98 | "collapsed": false 99 | }, 100 | "outputs": [], 101 | "source": [ 102 | "loaded_w = np.load(path + 'weight_storage.npz')\n", 103 | "loaded_w = loaded_w.items()[0][1]\n", 104 | "\n", 105 | "x = tf.placeholder(tf.float32, [None, 784])\n", 106 | "W = tf.Variable(tf.zeros([784, 10]))\n", 107 | "y_true = tf.placeholder(tf.float32, [None, 10])\n", 108 | "y_pred = tf.matmul(x, W)\n", 109 | "cross_entropy = tf.reduce_mean(\n", 110 | " tf.nn.softmax_cross_entropy_with_logits(logits=y_pred, labels=y_true))\n", 111 | "gd_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n", 112 | "correct_mask = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\n", 113 | "accuracy = tf.reduce_mean(tf.cast(correct_mask, tf.float32))\n", 114 | "\n", 115 | "with tf.Session() as sess:\n", 116 | " # assigning loaded weights\n", 117 | " sess.run(W.assign(loaded_w))\n", 118 | " acc = sess.run(accuracy, feed_dict={x: data.test.images, \n", 119 | " y_true: data.test.labels})\n", 120 | "\n", 121 | "print(\"Accuracy: {}\".format(acc))\n" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "### Saving multiple weight arrays" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "metadata": { 135 | "collapsed": true 136 | }, 137 | "outputs": [], 138 | "source": [ 139 | "class simple_cnn:\n", 140 | " def __init__(self, x_image,keep_prob, weights=None, sess=None):\n", 141 | " \n", 142 | " self.parameters = []\n", 143 | " self.x_image = x_image\n", 144 | "\n", 145 | " conv1 = self.conv_layer(x_image, shape=[5, 5, 1, 32])\n", 146 | " conv1_pool = self.max_pool_2x2(conv1)\n", 147 | "\n", 148 | " conv2 = self.conv_layer(conv1_pool, shape=[5, 5, 32, 64])\n", 149 | " conv2_pool = self.max_pool_2x2(conv2)\n", 150 | "\n", 151 | " conv2_flat = tf.reshape(conv2_pool, [-1, 7*7*64])\n", 152 | " full_1 = tf.nn.relu(self.full_layer(conv2_flat, 1024))\n", 153 | "\n", 154 | " full1_drop = tf.nn.dropout(full_1, keep_prob=keep_prob)\n", 155 | "\n", 156 | " self.y_conv = self.full_layer(full1_drop, 10)\n", 157 | " \n", 158 | " if weights is not None and sess is not None:\n", 159 | " self.load_weights(weights, sess)\n", 160 | " \n", 161 | " def weight_variable(self,shape):\n", 162 | " initial = tf.truncated_normal(shape, stddev=0.1)\n", 163 | " return tf.Variable(initial,name='weights')\n", 164 | "\n", 165 | "\n", 166 | " def bias_variable(self,shape):\n", 167 | " initial = tf.constant(0.1, shape=shape)\n", 168 | " return tf.Variable(initial,name='biases')\n", 169 | "\n", 170 | "\n", 171 | " def conv2d(self,x, W):\n", 172 | " return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], \n", 173 | " padding='SAME')\n", 174 | "\n", 175 | "\n", 176 | " def max_pool_2x2(self,x):\n", 177 | " return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n", 178 | " strides=[1, 2, 2, 1], padding='SAME')\n", 179 | "\n", 180 | "\n", 181 | " def conv_layer(self,input, shape):\n", 182 | " W = self.weight_variable(shape)\n", 183 | " b = self.bias_variable([shape[3]])\n", 184 | " self.parameters += [W, b]\n", 185 | "\n", 186 | " return tf.nn.relu(self.conv2d(input, W) + b)\n", 187 | "\n", 188 | "\n", 189 | " def full_layer(self,input, size):\n", 190 | " in_size = int(input.get_shape()[1])\n", 191 | " W = self.weight_variable([in_size, size])\n", 192 | " b = self.bias_variable([size])\n", 193 | " self.parameters += [W, b]\n", 194 | " return tf.matmul(input, W) + b\n", 195 | " \n", 196 | "\n", 197 | " def load_weights(self, weights, sess):\n", 198 | " for i,w in enumerate(weights):\n", 199 | " print(\"Weight index: {}\".format(i), \n", 200 | " \"Weight shape: {}\".format(w.shape))\n", 201 | " sess.run(self.parameters[i].assign(w))" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": { 208 | "collapsed": false 209 | }, 210 | "outputs": [], 211 | "source": [ 212 | "NUM_STEPS = 5000\n", 213 | "\n", 214 | "x = tf.placeholder(tf.float32, shape=[None, 784])\n", 215 | "x_image = tf.reshape(x, [-1, 28, 28, 1])\n", 216 | "y_ = tf.placeholder(tf.float32, shape=[None, 10])\n", 217 | "keep_prob = tf.placeholder(tf.float32)\n", 218 | "\n", 219 | "sess = tf.Session()\n", 220 | "\n", 221 | "cnn = simple_cnn(x_image,keep_prob, sess)\n", 222 | "\n", 223 | "cross_entropy = tf.reduce_mean(\n", 224 | " tf.nn.softmax_cross_entropy_with_logits(\n", 225 | " logits=cnn.y_conv, labels= y_))\n", 226 | "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n", 227 | "\n", 228 | "correct_prediction = tf.equal(tf.argmax(cnn.y_conv, 1), \n", 229 | " tf.argmax(y_, 1))\n", 230 | "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n", 231 | "sess.run(tf.global_variables_initializer())\n", 232 | "X = data.test.images.reshape(10, 1000, 784)\n", 233 | "Y = data.test.labels.reshape(10, 1000, 10)\n", 234 | "\n", 235 | "\n", 236 | "\n", 237 | "\n", 238 | "for _ in range(NUM_STEPS):\n", 239 | " batch_xs, batch_ys = data.train.next_batch(MINIBATCH_SIZE)\n", 240 | " sess.run(train_step, feed_dict={x:batch_xs, y_:batch_ys ,keep_prob:1.0})\n", 241 | "\n", 242 | "test_accuracy = np.mean([sess.run(accuracy, \n", 243 | " feed_dict={x:X[i], y_:Y[i],keep_prob:1.0}) \n", 244 | " for i in range(10)]) \n", 245 | "\n", 246 | "\n", 247 | "\n", 248 | "path = 'tmp//'\n", 249 | "weights = sess.run(cnn.parameters)\n", 250 | "np.savez(os.path.join(path, 'cnn_weight_storage'), weights)\n", 251 | "\n", 252 | "sess.close()\n", 253 | "\n", 254 | "print(\"test accuracy: {}\".format(test_accuracy))" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "### Loading weights" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "metadata": { 268 | "collapsed": false 269 | }, 270 | "outputs": [], 271 | "source": [ 272 | "\n", 273 | "x = tf.placeholder(tf.float32, shape=[None, 784])\n", 274 | "x_image = tf.reshape(x, [-1, 28, 28, 1])\n", 275 | "y_ = tf.placeholder(tf.float32, shape=[None, 10])\n", 276 | "keep_prob = tf.placeholder(tf.float32)\n", 277 | "\n", 278 | "sess = tf.Session()\n", 279 | "\n", 280 | "weights = np.load(path +'cnn_weight_storage.npz')\n", 281 | "weights = weights.items()[0][1]\n", 282 | "cnn = simple_cnn(x_image,keep_prob,weights, sess)\n", 283 | "\n", 284 | "cross_entropy = tf.reduce_mean(\n", 285 | " tf.nn.softmax_cross_entropy_with_logits(\n", 286 | " logits=cnn.y_conv, labels= y_))\n", 287 | "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n", 288 | "\n", 289 | "correct_prediction = tf.equal(tf.argmax(cnn.y_conv, 1), \n", 290 | " tf.argmax(y_, 1))\n", 291 | "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n", 292 | "\n", 293 | "X = data.test.images.reshape(10, 1000, 784)\n", 294 | "Y = data.test.labels.reshape(10, 1000, 10)\n", 295 | "test_accuracy = np.mean([sess.run(accuracy, \n", 296 | " feed_dict={x:X[i], y_:Y[i],keep_prob:1.0}) \n", 297 | " for i in range(10)])\n", 298 | "\n", 299 | "sess.close()\n", 300 | "\n", 301 | "print(\"test accuracy: {}\".format(test_accuracy))" 302 | ] 303 | }, 304 | { 305 | "cell_type": "markdown", 306 | "metadata": {}, 307 | "source": [ 308 | "### Saver class - save and restore weights only" 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": null, 314 | "metadata": { 315 | "collapsed": false 316 | }, 317 | "outputs": [], 318 | "source": [ 319 | "from tensorflow.examples.tutorials.mnist import input_data\n", 320 | "DATA_DIR = '/tmp/data'\n", 321 | "data = input_data.read_data_sets(DATA_DIR, one_hot=True)\n", 322 | "\n", 323 | "NUM_STEPS = 1000\n", 324 | "MINIBATCH_SIZE = 100\n", 325 | "\n", 326 | "DIR = 'saved_model/'\n", 327 | "\n", 328 | "x = tf.placeholder(tf.float32, [None, 784],name='x')\n", 329 | "W = tf.Variable(tf.zeros([784, 10]),name='W')\n", 330 | "y_true = tf.placeholder(tf.float32, [None, 10])\n", 331 | "y_pred = tf.matmul(x, W)\n", 332 | "cross_entropy = tf.reduce_mean(\n", 333 | " tf.nn.softmax_cross_entropy_with_logits(logits=y_pred, labels=y_true))\n", 334 | "gd_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n", 335 | "correct_mask = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\n", 336 | "accuracy = tf.reduce_mean(tf.cast(correct_mask, tf.float32))\n", 337 | "\n", 338 | "saver = tf.train.Saver(max_to_keep=7, \n", 339 | " keep_checkpoint_every_n_hours=1)\n", 340 | "\n", 341 | "with tf.Session() as sess:\n", 342 | " sess.run(tf.global_variables_initializer())\n", 343 | " for step in range(1,NUM_STEPS+1):\n", 344 | " batch_xs, batch_ys = data.train.next_batch(MINIBATCH_SIZE)\n", 345 | " sess.run(gd_step, feed_dict={x: batch_xs, y_true: batch_ys})\n", 346 | " \n", 347 | " if step % 50 == 0:\n", 348 | " saver.save(sess, os.path.join(DIR, \"model_ckpt\"), \n", 349 | " global_step=step)\n", 350 | " \n", 351 | " ans = sess.run(accuracy, feed_dict={x: data.test.images, \n", 352 | " y_true: data.test.labels})\n", 353 | "\n", 354 | "print(\"Accuracy: {:.4}%\".format(ans*100))" 355 | ] 356 | }, 357 | { 358 | "cell_type": "code", 359 | "execution_count": null, 360 | "metadata": { 361 | "collapsed": false 362 | }, 363 | "outputs": [], 364 | "source": [ 365 | "tf.reset_default_graph() \n", 366 | "x = tf.placeholder(tf.float32, [None, 784],name='x')\n", 367 | "W = tf.Variable(tf.zeros([784, 10]),name='W')\n", 368 | "y_true = tf.placeholder(tf.float32, [None, 10])\n", 369 | "y_pred = tf.matmul(x, W)\n", 370 | "cross_entropy = tf.reduce_mean(\n", 371 | " tf.nn.softmax_cross_entropy_with_logits(logits=y_pred,labels=y_true))\n", 372 | "gd_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n", 373 | "correct_mask = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\n", 374 | "accuracy = tf.reduce_mean(tf.cast(correct_mask, tf.float32))\n", 375 | "\n", 376 | "saver = tf.train.Saver()\n", 377 | "\n", 378 | "with tf.Session() as sess:\n", 379 | " sess.run(tf.global_variables_initializer())\n", 380 | " saver.restore(sess, os.path.join(DIR,\"model_ckpt-1000\"))\n", 381 | " ans = sess.run(accuracy, feed_dict={x: data.test.images, \n", 382 | " y_true: data.test.labels})\n", 383 | "\n", 384 | "print(\"Accuracy: {:.4}%\".format(ans*100))" 385 | ] 386 | }, 387 | { 388 | "cell_type": "markdown", 389 | "metadata": {}, 390 | "source": [ 391 | "### Save and restore weights + graph + collections" 392 | ] 393 | }, 394 | { 395 | "cell_type": "code", 396 | "execution_count": null, 397 | "metadata": { 398 | "collapsed": false 399 | }, 400 | "outputs": [], 401 | "source": [ 402 | "from tensorflow.examples.tutorials.mnist import input_data\n", 403 | "DATA_DIR = '/tmp/data'\n", 404 | "data = input_data.read_data_sets(DATA_DIR, one_hot=True)\n", 405 | "\n", 406 | "NUM_STEPS = 1000\n", 407 | "MINIBATCH_SIZE = 100\n", 408 | "\n", 409 | "DIR = 'saved_model/'\n", 410 | "\n", 411 | "x = tf.placeholder(tf.float32, [None, 784],name='x')\n", 412 | "W = tf.Variable(tf.zeros([784, 10]),name='W')\n", 413 | "y_true = tf.placeholder(tf.float32, [None, 10])\n", 414 | "y_pred = tf.matmul(x, W)\n", 415 | "cross_entropy = tf.reduce_mean(\n", 416 | " tf.nn.softmax_cross_entropy_with_logits(logits=y_pred, labels=y_true))\n", 417 | "gd_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n", 418 | "correct_mask = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\n", 419 | "accuracy = tf.reduce_mean(tf.cast(correct_mask, tf.float32))\n", 420 | "\n", 421 | "saver = tf.train.Saver(max_to_keep=7, \n", 422 | " keep_checkpoint_every_n_hours=1)\n", 423 | "\n", 424 | "train_var = [x,y_true,accuracy]\n", 425 | "tf.add_to_collection('train_var', train_var[0])\n", 426 | "tf.add_to_collection('train_var', train_var[1])\n", 427 | "tf.add_to_collection('train_var', train_var[2])\n", 428 | "\n", 429 | "with tf.Session() as sess:\n", 430 | " sess.run(tf.global_variables_initializer())\n", 431 | " for step in range(1,NUM_STEPS+1):\n", 432 | " batch_xs, batch_ys = data.train.next_batch(MINIBATCH_SIZE)\n", 433 | " sess.run(gd_step, feed_dict={x: batch_xs, y_true: batch_ys})\n", 434 | " \n", 435 | " if step % 50 == 0:\n", 436 | " saver.export_meta_graph(os.path.join(DIR,\"model_ckpt.meta\"),collection_list=['train_var'])\n", 437 | " saver.save(sess, os.path.join(DIR, \"model_ckpt\"), \n", 438 | " global_step=step)\n", 439 | " \n", 440 | " ans = sess.run(accuracy, feed_dict={x: data.test.images, \n", 441 | " y_true: data.test.labels})\n", 442 | "\n", 443 | "print(\"Accuracy: {:.4}%\".format(ans*100))" 444 | ] 445 | }, 446 | { 447 | "cell_type": "code", 448 | "execution_count": null, 449 | "metadata": { 450 | "collapsed": false 451 | }, 452 | "outputs": [], 453 | "source": [ 454 | "tf.reset_default_graph() \n", 455 | "\n", 456 | "with tf.Session() as sess:\n", 457 | " sess.run(tf.global_variables_initializer())\n", 458 | "\n", 459 | " saver = tf.train.import_meta_graph(os.path.join(DIR,\"model_ckpt.meta\"))\n", 460 | " saver.restore(sess, os.path.join(DIR,\"model_ckpt-1000\"))\n", 461 | " x = tf.get_collection('train_var')[0]\n", 462 | " y_true = tf.get_collection('train_var')[1]\n", 463 | " accuracy = tf.get_collection('train_var')[2]\n", 464 | "\n", 465 | " ans = sess.run(accuracy, feed_dict={x: data.test.images, \n", 466 | " y_true: data.test.labels})\n", 467 | "print(\"Accuracy: {:.4}%\".format(ans*100))" 468 | ] 469 | }, 470 | { 471 | "cell_type": "markdown", 472 | "metadata": { 473 | "collapsed": true 474 | }, 475 | "source": [ 476 | "### TensorFlow Serving" 477 | ] 478 | }, 479 | { 480 | "cell_type": "code", 481 | "execution_count": null, 482 | "metadata": { 483 | "collapsed": true 484 | }, 485 | "outputs": [], 486 | "source": [ 487 | "import os\n", 488 | "import sys\n", 489 | "import tensorflow as tf\n", 490 | "from tensorflow.python.saved_model import builder \n", 491 | " as saved_model_builder\n", 492 | "from tensorflow.python.saved_model import signature_constants\n", 493 | "from tensorflow.python.saved_model import signature_def_utils\n", 494 | "from tensorflow.python.saved_model import tag_constants\n", 495 | "from tensorflow.python.saved_model import utils\n", 496 | "from tensorflow.python.util import compat\n", 497 | "from tensorflow_serving.example import mnist_input_data\n", 498 | "\n", 499 | "tf.app.flags.DEFINE_integer('training_iteration', 10,\n", 500 | " 'number of training iterations.')\n", 501 | "tf.app.flags.DEFINE_integer(\n", 502 | " 'model_version', 1, 'version number of the model.')\n", 503 | "tf.app.flags.DEFINE_string('work_dir', '/tmp', 'Working directory.')\n", 504 | "FLAGS = tf.app.flags.FLAGS\n", 505 | "\n", 506 | "\n", 507 | "def weight_variable(shape):\n", 508 | " initial = tf.truncated_normal(shape, stddev=0.1)\n", 509 | " return tf.Variable(initial,dtype='float')\n", 510 | "\n", 511 | "def bias_variable(shape):\n", 512 | " initial = tf.constant(0.1, shape=shape)\n", 513 | " return tf.Variable(initial,dtype='float')\n", 514 | "\n", 515 | "def conv2d(x, W):\n", 516 | " return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n", 517 | "\n", 518 | "def max_pool_2x2(x):\n", 519 | " return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n", 520 | " strides=[1, 2, 2, 1], padding='SAME')\n", 521 | "\n", 522 | "def main(_):\n", 523 | " if len(sys.argv) < 2 or sys.argv[-1].startswith('-'):\n", 524 | " print('Usage: mnist_export.py [--training_iteration=x] '\n", 525 | " '[--model_version=y] export_dir')\n", 526 | " sys.exit(-1)\n", 527 | " if FLAGS.training_iteration <= 0:\n", 528 | " print('Please specify a positive \n", 529 | " value for training iteration.')\n", 530 | " sys.exit(-1)\n", 531 | " if FLAGS.model_version <= 0:\n", 532 | " print ('Please specify a positive \n", 533 | " value for version number.')\n", 534 | " sys.exit(-1)\n", 535 | " \n", 536 | "\n", 537 | " print('Training...')\n", 538 | " mnist = mnist_input_data.read_data_sets(\n", 539 | " FLAGS.work_dir, one_hot=True)\n", 540 | " sess = tf.InteractiveSession()\n", 541 | " serialized_tf_example = tf.placeholder(\n", 542 | " tf.string, name='tf_example')\n", 543 | " feature_configs = {'x': tf.FixedLenFeature(shape=[784],\\ \n", 544 | " dtype=tf.float32),}\n", 545 | " tf_example = tf.parse_example(serialized_tf_example, \n", 546 | " feature_configs)\n", 547 | " \n", 548 | " \n", 549 | " x = tf.identity(tf_example['x'], name='x') \n", 550 | " y_ = tf.placeholder('float', shape=[None, 10])\n", 551 | "\n", 552 | " W_conv1 = weight_variable([5, 5, 1, 32])\n", 553 | " b_conv1 = bias_variable([32])\n", 554 | " x_image = tf.reshape(x, [-1,28,28,1])\n", 555 | "\n", 556 | " h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n", 557 | " h_pool1 = max_pool_2x2(h_conv1)\n", 558 | "\n", 559 | " W_conv2 = weight_variable([5, 5, 32, 64])\n", 560 | " b_conv2 = bias_variable([64])\n", 561 | "\n", 562 | " h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n", 563 | " h_pool2 = max_pool_2x2(h_conv2)\n", 564 | "\n", 565 | "\n", 566 | " W_fc1 = weight_variable([7 * 7 * 64, 1024])\n", 567 | " b_fc1 = bias_variable([1024])\n", 568 | "\n", 569 | " h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n", 570 | " h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n", 571 | "\n", 572 | "\n", 573 | " keep_prob = tf.placeholder(tf.float32)\n", 574 | " h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n", 575 | "\n", 576 | " W_fc2 = weight_variable([1024, 10])\n", 577 | " b_fc2 = bias_variable([10])\n", 578 | "\n", 579 | " y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n", 580 | " \n", 581 | " \n", 582 | " y = tf.nn.softmax(y_conv, name='y')\n", 583 | " cross_entropy = -tf.reduce_sum(y_ * tf.log(y_conv))\n", 584 | " train_step = tf.train.AdamOptimizer(1e-4).\\\n", 585 | " minimize(cross_entropy)\n", 586 | " \n", 587 | " \n", 588 | " values, indices = tf.nn.top_k(y_conv, 10)\n", 589 | " prediction_classes = tf.contrib.lookup.index_to_string(\n", 590 | " tf.to_int64(indices), \n", 591 | " mapping=tf.constant([str(i) for i in xrange(10)]))\n", 592 | " \n", 593 | " sess.run(tf.global_variables_initializer())\n", 594 | "\n", 595 | " for _ in range(FLAGS.training_iteration):\n", 596 | " batch = mnist.train.next_batch(50)\n", 597 | " \n", 598 | " train_step.run(feed_dict={x: batch[0], \n", 599 | " y_: batch[1], keep_prob: 0.5})\n", 600 | " print(_)\n", 601 | " correct_prediction = tf.equal(tf.argmax(y_conv,1), \n", 602 | " tf.argmax(y_,1))\n", 603 | "\n", 604 | " \n", 605 | " accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))\n", 606 | " y_: mnist.test.labels})\n", 607 | " \n", 608 | " print('training accuracy %g' % accuracy.eval(feed_dict={\n", 609 | " x: mnist.test.images, \n", 610 | " y_: mnist.test.labels, keep_prob: 1.0}))\n", 611 | "\n", 612 | " print('training is finished!')\n", 613 | " \n", 614 | " export_path_base = sys.argv[-1]\n", 615 | " export_path = os.path.join(\n", 616 | " compat.as_bytes(export_path_base),\n", 617 | " compat.as_bytes(str(FLAGS.model_version)))\n", 618 | " print 'Exporting trained model to', export_path\n", 619 | " builder = saved_model_builder.SavedModelBuilder(export_path)\n", 620 | "\n", 621 | " classification_inputs = utils.build_tensor_info(\n", 622 | " serialized_tf_example)\n", 623 | " classification_outputs_classes = utils.build_tensor_info(\n", 624 | " prediction_classes)\n", 625 | " classification_outputs_scores = utils.build_tensor_info(values)\n", 626 | "\n", 627 | " classification_signature = signature_def_utils.\\\n", 628 | " build_signature_def(\n", 629 | " inputs={signature_constants.CLASSIFY_INPUTS:\\ \n", 630 | " classification_inputs},\n", 631 | " outputs={\n", 632 | " signature_constants.CLASSIFY_OUTPUT_CLASSES:\n", 633 | " classification_outputs_classes,\n", 634 | " signature_constants.CLASSIFY_OUTPUT_SCORES:\n", 635 | " classification_outputs_scores\n", 636 | " },\n", 637 | " method_name=signature_constants.CLASSIFY_METHOD_NAME)\n", 638 | "\n", 639 | " tensor_info_x = utils.build_tensor_info(x)\n", 640 | " tensor_info_y = utils.build_tensor_info(y_conv)\n", 641 | "\n", 642 | " prediction_signature = signature_def_utils.build_signature_def(\n", 643 | " inputs={'images': tensor_info_x},\n", 644 | " outputs={'scores': tensor_info_y},\n", 645 | " method_name=signature_constants.PREDICT_METHOD_NAME)\n", 646 | "\n", 647 | " legacy_init_op = tf.group(tf.initialize_all_tables(), \n", 648 | " name='legacy_init_op')\n", 649 | " builder.add_meta_graph_and_variables(\n", 650 | " sess, [tag_constants.SERVING],\n", 651 | " signature_def_map={\n", 652 | " 'predict_images':\n", 653 | " prediction_signature,\n", 654 | " signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:\n", 655 | " classification_signature,\n", 656 | " },\n", 657 | " legacy_init_op=legacy_init_op)\n", 658 | "\n", 659 | " builder.save()\n", 660 | "\n", 661 | " print('new model exported!')\n", 662 | "\n", 663 | "\n", 664 | "if __name__ == '__main__':\n", 665 | " tf.app.run()" 666 | ] 667 | } 668 | ], 669 | "metadata": { 670 | "kernelspec": { 671 | "display_name": "tf_1_1", 672 | "language": "python", 673 | "name": "tf_1_1" 674 | }, 675 | "language_info": { 676 | "codemirror_mode": { 677 | "name": "ipython", 678 | "version": 2 679 | }, 680 | "file_extension": ".py", 681 | "mimetype": "text/x-python", 682 | "name": "python", 683 | "nbconvert_exporter": "python", 684 | "pygments_lexer": "ipython2", 685 | "version": "2.7.12" 686 | } 687 | }, 688 | "nbformat": 4, 689 | "nbformat_minor": 1 690 | } 691 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Yehezkel Resheff 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Oreilly-Learning-TensorFlow 2 | 3 | ### Content (book chapters): 4 | 1. Introduction -- no code 5 | 2. Go with the Flow: Up and Running with TensorFlow 6 | 3. Understanding TensorFlow Basics 7 | 4. Convolutional Neural Networks 8 | 5. Text I: Working with Text and Sequences, and TensorBoard Visualization. 9 | 6. Text II: Word Vectors, Advanced RNN, and Embedding Visualization. 10 | 7. TensorFlow Abstractions and Simplications. 11 | 8. Queues, Threads, and Reading Data. 12 | 9. Distributed TensorFlow. 13 | 10. Exporting and Serving Models with TensorFlow. 14 | 15 | 16 | Check out the associated [repo](https://github.com/Hezi-Resheff/Oreilly-OSCON2017-Learning-TensorFlow) from our [OSCON2017](https://conferences.oreilly.com/oscon/oscon-tx) [training](https://conferences.oreilly.com/oscon/oscon-tx/public/schedule/detail/57856). 17 | 18 | --- 19 | If you like the book, please rate it on [Amazon](https://www.amazon.com/Learning-TensorFlow-Guide-Building-Systems/dp/1491978511/ref=sr_1_2?ie=UTF8&qid=1505770781&sr=8-2&keywords=resheff) :) 20 | --------------------------------------------------------------------------------