├── .gitignore ├── LICENCE ├── README.md ├── ch02_basics ├── .gitignore ├── Concept01_defining_tensors.ipynb ├── Concept02_evaluating_ops.ipynb ├── Concept03_interactive_session.ipynb ├── Concept04_session_logging.ipynb ├── Concept05_variables.ipynb ├── Concept06_saving_variables.ipynb ├── Concept07_loading_variables.ipynb ├── Concept08_TensorBoard.ipynb ├── Concept09_queue.ipynb ├── Concept10_queue_text.ipynb ├── README.md ├── gradient.py ├── interactive_session.py ├── loading_vars.py ├── log_example.py ├── logging_example.py ├── main.py ├── moving_avg.py ├── saving_vars.py ├── spikes.py └── types.py ├── ch03_regression ├── Concept01_linear_regression.ipynb ├── Concept02_poly_regression.ipynb ├── Concept03_regularization.ipynb ├── README.md └── data_reader.py ├── ch04_classification ├── Concept01_linear_regression_classification.ipynb ├── Concept02_logistic.ipynb ├── Concept03_logistic2d.ipynb ├── Concept04_softmax.ipynb ├── README.md ├── linear_1d.py ├── logistic_1d.py ├── logistic_2d.py └── softmax.py ├── ch05_clustering ├── Concept01_clustering.ipynb ├── Concept02_segmentation.ipynb ├── Concept03_som.ipynb ├── README.md ├── TalkingMachinesPodcast.wav ├── audio_clustering.py ├── audio_dataset │ ├── cough_1.wav │ ├── cough_1.wav.png │ ├── cough_2.wav │ ├── cough_2.wav.png │ ├── scream_1.wav │ ├── scream_1.wav.png │ ├── scream_2.wav │ ├── scream_2.wav.png │ ├── scream_3.wav │ └── scream_3.wav.png ├── audio_segmentation.py ├── som.py └── som_test.py ├── ch06_hmm ├── Concept01_forward.ipynb ├── Concept02_hmm.ipynb ├── README.md ├── __init__.py ├── forward.py └── hmm.py ├── ch07_autoencoder ├── .gitignore ├── Concept01_autoencoder.ipynb ├── Concept02_autoencoder_with_imgs.ipynb ├── Concept03_denoising.ipynb ├── README.md ├── autoencoder.py ├── autoencoder_batch.py ├── checkpoint ├── denoiser.py ├── denoising_autoencoder.py ├── export_parameters.py ├── main.py ├── main_imgs.py └── model.ckpt.meta ├── ch08_rl ├── .gitignore ├── Concept01_rl.ipynb ├── README.md ├── prices.png ├── rl.py └── stock_prices.npy ├── ch09_cnn ├── .gitignore ├── Concept01_cifar.ipynb ├── Concept02_convolution.ipynb ├── Concept03_cnn.ipynb ├── README.md ├── cifar_tools.py ├── cnn.py ├── cnn_viz.py ├── conv_visuals.py └── using_cifar.py ├── ch10_rnn ├── .gitignore ├── Concept01_timeseries_data.ipynb ├── Concept02_rnn.ipynb ├── Concept03_rnn_real_world.ipynb ├── README.md ├── data_loader.py ├── international-airline-passengers.csv ├── regression.py └── simple_regression.py ├── ch11_seq2seq ├── .ipynb_checkpoints │ ├── Concept01_multi_rnn-checkpoint.ipynb │ └── Concept02_embedding_lookup-checkpoint.ipynb ├── Concept01_multi_rnn.ipynb ├── Concept02_embedding_lookup.ipynb ├── Concept03_seq2seq.ipynb └── data │ ├── process_input.py │ ├── words_input.txt │ └── words_output.txt └── ch12_rank ├── Concept01_ranknet.ipynb ├── Concept02_embedding.ipynb ├── Concept03_image_ranking.ipynb ├── imagenet_classes.py └── vgg16.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.pyc 3 | *~ 4 | *.ckpt -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Nishant Shukla 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [Machine Learning with TensorFlow](http://www.tensorflowbook.com/) 2 | 3 | [This](https://github.com/BinRoot/TensorFlow-Book) is the official code repository for [Machine Learning with TensorFlow](http://www.tensorflowbook.com/). 4 | 5 | Get started with machine learning using TensorFlow, Google's latest and greatest machine learning library. 6 | 7 | # Summary 8 | 9 | ## [Chapter 2](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch02_basics) - TensorFlow Basics 10 | 11 | - **Concept 1**: Defining tensors 12 | - **Concept 2**: Evaluating ops 13 | - **Concept 3**: Interactive session 14 | - **Concept 4**: Session loggings 15 | - **Concept 5**: Variables 16 | - **Concept 6**: Saving variables 17 | - **Concept 7**: Loading variables 18 | - **Concept 8**: TensorBoard 19 | 20 | ## [Chapter 3](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch03_regression) - Regression 21 | 22 | - **Concept 1**: Linear regression 23 | - **Concept 2**: Polynomial regression 24 | - **Concept 3**: Regularization 25 | 26 | ## [Chapter 4](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch04_classification) - Classification 27 | 28 | - **Concept 1**: Linear regression for classification 29 | - **Concept 2**: Logistic regression 30 | - **Concept 3**: 2D Logistic regression 31 | - **Concept 4**: Softmax classification 32 | 33 | ## [Chapter 5](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch05_clustering) - Clustering 34 | 35 | - **Concept 1**: Clustering 36 | - **Concept 2**: Segmentation 37 | - **Concept 3**: Self-organizing map 38 | 39 | ## [Chapter 6](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch06_hmm) - Hidden markov models 40 | 41 | - **Concept 1**: Forward algorithm 42 | - **Concept 2**: Viterbi decode 43 | 44 | ## [Chapter 7](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch07_autoencoder) - Autoencoders 45 | 46 | - **Concept 1**: Autoencoder 47 | - **Concept 2**: Applying an autoencoder to images 48 | - **Concept 3**: Denoising autoencoder 49 | 50 | ## [Chapter 8](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch08_rl) - Reinforcement learning 51 | 52 | - **Concept 1**: Reinforcement learning 53 | 54 | ## [Chapter 9](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch09_cnn) - Convolutional Neural Networks 55 | 56 | - **Concept 1**: Using CIFAR-10 dataset 57 | - **Concept 2**: Convolutions 58 | - **Concept 3**: Convolutional neural network 59 | 60 | ## [Chapter 10](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch10_rnn) - Recurrent Neural Network 61 | 62 | - **Concept 1**: Loading timeseries data 63 | - **Concept 2**: Recurrent neural networks 64 | - **Concept 3**: Applying RNN to real-world data for timeseries prediction 65 | 66 | ## [Chapter 11](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch11_seq2seq) - Seq2Seq Model 67 | 68 | - **Concept 1**: Multi-cell RNN 69 | - **Concept 2**: Embedding lookup 70 | - **Concept 3**: Seq2seq model 71 | 72 | ## [Chapter 12](https://github.com/BinRoot/TensorFlow-Book/tree/master/ch12_rank) - Ranking 73 | 74 | - **Concept 1**: RankNet 75 | - **Concept 2**: Image embedding 76 | - **Concept 3**: Image ranking 77 | -------------------------------------------------------------------------------- /ch02_basics/.gitignore: -------------------------------------------------------------------------------- 1 | *.ckpt.* 2 | checkpoint 3 | logs 4 | big.txt 5 | .ipynb* -------------------------------------------------------------------------------- /ch02_basics/Concept01_defining_tensors.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `02`: Concept `01`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Defining tensors" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Import TensorFlow and Numpy:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "import numpy as np" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "collapsed": true, 51 | "deletable": true, 52 | "editable": true 53 | }, 54 | "source": [ 55 | "Now, define a 2x2 matrix in different ways:" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 2, 61 | "metadata": { 62 | "collapsed": false, 63 | "deletable": true, 64 | "editable": true 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "m1 = [[1.0, 2.0], \n", 69 | " [3.0, 4.0]]\n", 70 | "\n", 71 | "m2 = np.array([[1.0, 2.0], \n", 72 | " [3.0, 4.0]], dtype=np.float32)\n", 73 | "\n", 74 | "m3 = tf.constant([[1.0, 2.0], \n", 75 | " [3.0, 4.0]])" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": { 81 | "deletable": true, 82 | "editable": true 83 | }, 84 | "source": [ 85 | "Let's see what happens when we print them:" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": 3, 91 | "metadata": { 92 | "collapsed": false, 93 | "deletable": true, 94 | "editable": true 95 | }, 96 | "outputs": [ 97 | { 98 | "name": "stdout", 99 | "output_type": "stream", 100 | "text": [ 101 | "\n", 102 | "\n", 103 | "\n" 104 | ] 105 | } 106 | ], 107 | "source": [ 108 | "print(type(m1))\n", 109 | "print(type(m2))\n", 110 | "print(type(m3))" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": { 116 | "deletable": true, 117 | "editable": true 118 | }, 119 | "source": [ 120 | "So, that's what we're dealing with. Interesting. \n", 121 | "\n", 122 | "By the way, there's a function called `convert_to_tensor(...)` that does exactly what you might expect. \n", 123 | "\n", 124 | "Let's use it to create tensor objects out of various types:" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 4, 130 | "metadata": { 131 | "collapsed": true, 132 | "deletable": true, 133 | "editable": true 134 | }, 135 | "outputs": [], 136 | "source": [ 137 | "t1 = tf.convert_to_tensor(m1, dtype=tf.float32)\n", 138 | "t2 = tf.convert_to_tensor(m2, dtype=tf.float32)\n", 139 | "t3 = tf.convert_to_tensor(m3, dtype=tf.float32)" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": { 145 | "deletable": true, 146 | "editable": true 147 | }, 148 | "source": [ 149 | "Ok, ok! Time for the reveal:" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 5, 155 | "metadata": { 156 | "collapsed": false, 157 | "deletable": true, 158 | "editable": true 159 | }, 160 | "outputs": [ 161 | { 162 | "name": "stdout", 163 | "output_type": "stream", 164 | "text": [ 165 | "\n", 166 | "\n", 167 | "\n" 168 | ] 169 | } 170 | ], 171 | "source": [ 172 | "print(type(t1))\n", 173 | "print(type(t2))\n", 174 | "print(type(t3))" 175 | ] 176 | } 177 | ], 178 | "metadata": { 179 | "kernelspec": { 180 | "display_name": "Python 3", 181 | "language": "python", 182 | "name": "python3" 183 | }, 184 | "language_info": { 185 | "codemirror_mode": { 186 | "name": "ipython", 187 | "version": 3 188 | }, 189 | "file_extension": ".py", 190 | "mimetype": "text/x-python", 191 | "name": "python", 192 | "nbconvert_exporter": "python", 193 | "pygments_lexer": "ipython3", 194 | "version": "3.5.2" 195 | } 196 | }, 197 | "nbformat": 4, 198 | "nbformat_minor": 0 199 | } 200 | -------------------------------------------------------------------------------- /ch02_basics/Concept02_evaluating_ops.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `02`: Concept `02`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Evaluating ops" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Import TensorFlow:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": { 49 | "collapsed": true, 50 | "deletable": true, 51 | "editable": true 52 | }, 53 | "source": [ 54 | "Start with a 1x2 matrix:" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "metadata": { 61 | "collapsed": false, 62 | "deletable": true, 63 | "editable": true 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "x = tf.constant([[1, 2]])" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": { 73 | "deletable": true, 74 | "editable": true 75 | }, 76 | "source": [ 77 | "Let's negate it. Define the negation op to be run on the matrix:" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 4, 83 | "metadata": { 84 | "collapsed": false, 85 | "deletable": true, 86 | "editable": true 87 | }, 88 | "outputs": [], 89 | "source": [ 90 | "neg_x = tf.negative(x)" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": { 96 | "deletable": true, 97 | "editable": true 98 | }, 99 | "source": [ 100 | "It's nothing special if you print it out. In fact, it doesn't even perform the negation computation. Check out what happens when you simply print it:" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 16, 106 | "metadata": { 107 | "collapsed": false, 108 | "deletable": true, 109 | "editable": true 110 | }, 111 | "outputs": [ 112 | { 113 | "name": "stdout", 114 | "output_type": "stream", 115 | "text": [ 116 | "Tensor(\"Neg_3:0\", shape=(1, 2), dtype=int32)\n" 117 | ] 118 | } 119 | ], 120 | "source": [ 121 | "print(neg_x)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": { 127 | "deletable": true, 128 | "editable": true 129 | }, 130 | "source": [ 131 | "You need to summon a session so you can launch the negation op:" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 17, 137 | "metadata": { 138 | "collapsed": false, 139 | "deletable": true, 140 | "editable": true 141 | }, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "[[-1 -2]]\n" 148 | ] 149 | } 150 | ], 151 | "source": [ 152 | "with tf.Session() as sess:\n", 153 | " result = sess.run(neg_x)\n", 154 | " print(result)" 155 | ] 156 | } 157 | ], 158 | "metadata": { 159 | "kernelspec": { 160 | "display_name": "Python 3", 161 | "language": "python", 162 | "name": "python3" 163 | }, 164 | "language_info": { 165 | "codemirror_mode": { 166 | "name": "ipython", 167 | "version": 3 168 | }, 169 | "file_extension": ".py", 170 | "mimetype": "text/x-python", 171 | "name": "python", 172 | "nbconvert_exporter": "python", 173 | "pygments_lexer": "ipython3", 174 | "version": "3.5.2" 175 | } 176 | }, 177 | "nbformat": 4, 178 | "nbformat_minor": 0 179 | } 180 | -------------------------------------------------------------------------------- /ch02_basics/Concept03_interactive_session.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `02`: Concept `03`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Interactive session" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Interactive sessions are another way to use a session. Go ahead and define one:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "sess = tf.InteractiveSession()" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "collapsed": true, 51 | "deletable": true, 52 | "editable": true 53 | }, 54 | "source": [ 55 | "We have a matrix we want to invert:" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 2, 61 | "metadata": { 62 | "collapsed": false, 63 | "deletable": true, 64 | "editable": true 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "x = tf.constant([[1., 2.]])\n", 69 | "neg_op = tf.negative(x)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": { 75 | "deletable": true, 76 | "editable": true 77 | }, 78 | "source": [ 79 | "Since we're using an interactive session, we can just call the `eval()` method on the op." 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 3, 85 | "metadata": { 86 | "collapsed": false, 87 | "deletable": true, 88 | "editable": true 89 | }, 90 | "outputs": [ 91 | { 92 | "name": "stdout", 93 | "output_type": "stream", 94 | "text": [ 95 | "[[-1. -2.]]\n" 96 | ] 97 | } 98 | ], 99 | "source": [ 100 | "result = neg_op.eval()\n", 101 | "\n", 102 | "print(result)\n" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "metadata": { 108 | "deletable": true, 109 | "editable": true 110 | }, 111 | "source": [ 112 | "That code's a little cleaner when using Jupyter notebooks (like this one).\n", 113 | "\n", 114 | "Don't forget to close the session:" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 4, 120 | "metadata": { 121 | "collapsed": false, 122 | "deletable": true, 123 | "editable": true 124 | }, 125 | "outputs": [], 126 | "source": [ 127 | "sess.close()" 128 | ] 129 | } 130 | ], 131 | "metadata": { 132 | "kernelspec": { 133 | "display_name": "Python 3", 134 | "language": "python", 135 | "name": "python3" 136 | }, 137 | "language_info": { 138 | "codemirror_mode": { 139 | "name": "ipython", 140 | "version": 3 141 | }, 142 | "file_extension": ".py", 143 | "mimetype": "text/x-python", 144 | "name": "python", 145 | "nbconvert_exporter": "python", 146 | "pygments_lexer": "ipython3", 147 | "version": "3.5.2" 148 | } 149 | }, 150 | "nbformat": 4, 151 | "nbformat_minor": 0 152 | } 153 | -------------------------------------------------------------------------------- /ch02_basics/Concept04_session_logging.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `02`: Concept `04`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Session logging" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Define an op on a tensor. Here's an example:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": false, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "\n", 45 | "x = tf.constant([[1, 2]])\n", 46 | "neg_op = tf.negative(x)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": { 52 | "collapsed": true, 53 | "deletable": true, 54 | "editable": true 55 | }, 56 | "source": [ 57 | "Now let's use a session with a special argument passed in." 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 3, 63 | "metadata": { 64 | "collapsed": false, 65 | "deletable": true, 66 | "editable": true 67 | }, 68 | "outputs": [ 69 | { 70 | "name": "stdout", 71 | "output_type": "stream", 72 | "text": [ 73 | "[[-1 -2]]\n" 74 | ] 75 | } 76 | ], 77 | "source": [ 78 | "with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:\n", 79 | " result = sess.run(neg_op)\n", 80 | " print(result)" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": { 86 | "deletable": true, 87 | "editable": true 88 | }, 89 | "source": [ 90 | "Try this from a terminal. Jupyter notebooks won't show the logging info." 91 | ] 92 | } 93 | ], 94 | "metadata": { 95 | "kernelspec": { 96 | "display_name": "Python 3", 97 | "language": "python", 98 | "name": "python3" 99 | }, 100 | "language_info": { 101 | "codemirror_mode": { 102 | "name": "ipython", 103 | "version": 3 104 | }, 105 | "file_extension": ".py", 106 | "mimetype": "text/x-python", 107 | "name": "python", 108 | "nbconvert_exporter": "python", 109 | "pygments_lexer": "ipython3", 110 | "version": "3.5.2" 111 | } 112 | }, 113 | "nbformat": 4, 114 | "nbformat_minor": 0 115 | } 116 | -------------------------------------------------------------------------------- /ch02_basics/Concept05_variables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `02`: Concept `05`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Using variables" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Here we go, here we go, here we go! Moving on from those simple examples, let's get a better understanding of variables. Start with a session:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": false, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "sess = tf.InteractiveSession()" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "deletable": true, 51 | "editable": true 52 | }, 53 | "source": [ 54 | "Below is a series of numbers. Don't worry what they mean. Just for fun, let's think of them as neural activations." 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "metadata": { 61 | "collapsed": true, 62 | "deletable": true, 63 | "editable": true 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "raw_data = [1., 2., 8., -1., 0., 5.5, 6., 13]" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": { 73 | "collapsed": true, 74 | "deletable": true, 75 | "editable": true 76 | }, 77 | "source": [ 78 | "Create a boolean variable called `spike` to detect a sudden increase in the values.\n", 79 | "\n", 80 | "All variables must be initialized. Go ahead and initialize the variable by calling `run()` on its `initializer`:" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 3, 86 | "metadata": { 87 | "collapsed": false, 88 | "deletable": true, 89 | "editable": true 90 | }, 91 | "outputs": [], 92 | "source": [ 93 | "spike = tf.Variable(False)\n", 94 | "spike.initializer.run()" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": { 100 | "deletable": true, 101 | "editable": true 102 | }, 103 | "source": [ 104 | "Loop through the data and update the spike variable when there is a significant increase:" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 4, 110 | "metadata": { 111 | "collapsed": false, 112 | "deletable": true, 113 | "editable": true 114 | }, 115 | "outputs": [ 116 | { 117 | "name": "stdout", 118 | "output_type": "stream", 119 | "text": [ 120 | "Spike False\n", 121 | "Spike True\n", 122 | "Spike False\n", 123 | "Spike False\n", 124 | "Spike True\n", 125 | "Spike False\n", 126 | "Spike True\n" 127 | ] 128 | } 129 | ], 130 | "source": [ 131 | "for i in range(1, len(raw_data)):\n", 132 | " if raw_data[i] - raw_data[i-1] > 5:\n", 133 | " updater = tf.assign(spike, tf.constant(True))\n", 134 | " updater.eval()\n", 135 | " else:\n", 136 | " tf.assign(spike, False).eval()\n", 137 | " print(\"Spike\", spike.eval())" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": { 143 | "deletable": true, 144 | "editable": true 145 | }, 146 | "source": [ 147 | "You forgot to close the session! Here, let me do it:" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 5, 153 | "metadata": { 154 | "collapsed": true, 155 | "deletable": true, 156 | "editable": true 157 | }, 158 | "outputs": [], 159 | "source": [ 160 | "sess.close()" 161 | ] 162 | } 163 | ], 164 | "metadata": { 165 | "kernelspec": { 166 | "display_name": "Python 3", 167 | "language": "python", 168 | "name": "python3" 169 | }, 170 | "language_info": { 171 | "codemirror_mode": { 172 | "name": "ipython", 173 | "version": 3 174 | }, 175 | "file_extension": ".py", 176 | "mimetype": "text/x-python", 177 | "name": "python", 178 | "nbconvert_exporter": "python", 179 | "pygments_lexer": "ipython3", 180 | "version": "3.5.2" 181 | } 182 | }, 183 | "nbformat": 4, 184 | "nbformat_minor": 0 185 | } 186 | -------------------------------------------------------------------------------- /ch02_basics/Concept06_saving_variables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `02`: Concept `06`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Saving variables" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Create an interactive session and initialize a variable:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "sess = tf.InteractiveSession()\n", 45 | "\n", 46 | "raw_data = [1., 2., 8., -1., 0., 5.5, 6., 13]\n", 47 | "spikes = tf.Variable([False] * len(raw_data), name='spikes')\n", 48 | "spikes.initializer.run()" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": { 54 | "deletable": true, 55 | "editable": true 56 | }, 57 | "source": [ 58 | "The saver op will enable saving and restoring:" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 2, 64 | "metadata": { 65 | "collapsed": true, 66 | "deletable": true, 67 | "editable": true 68 | }, 69 | "outputs": [], 70 | "source": [ 71 | "saver = tf.train.Saver()" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": { 77 | "deletable": true, 78 | "editable": true 79 | }, 80 | "source": [ 81 | "Loop through the data and update the spike variable when there is a significant increase:" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 3, 87 | "metadata": { 88 | "collapsed": false, 89 | "deletable": true, 90 | "editable": true 91 | }, 92 | "outputs": [], 93 | "source": [ 94 | "for i in range(1, len(raw_data)):\n", 95 | " if raw_data[i] - raw_data[i-1] > 5:\n", 96 | " spikes_val = spikes.eval()\n", 97 | " spikes_val[i] = True\n", 98 | " updater = tf.assign(spikes, spikes_val)\n", 99 | " updater.eval()" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": { 105 | "deletable": true, 106 | "editable": true 107 | }, 108 | "source": [ 109 | "Now, save your variable to disk!" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 4, 115 | "metadata": { 116 | "collapsed": false, 117 | "deletable": true, 118 | "editable": true 119 | }, 120 | "outputs": [ 121 | { 122 | "name": "stdout", 123 | "output_type": "stream", 124 | "text": [ 125 | "spikes data saved in file: spikes.ckpt\n" 126 | ] 127 | } 128 | ], 129 | "source": [ 130 | "save_path = saver.save(sess, \"./spikes.ckpt\")\n", 131 | "print(\"spikes data saved in file: %s\" % save_path)" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "metadata": { 137 | "deletable": true, 138 | "editable": true 139 | }, 140 | "source": [ 141 | "Adieu:" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 5, 147 | "metadata": { 148 | "collapsed": true, 149 | "deletable": true, 150 | "editable": true 151 | }, 152 | "outputs": [], 153 | "source": [ 154 | "sess.close()" 155 | ] 156 | } 157 | ], 158 | "metadata": { 159 | "kernelspec": { 160 | "display_name": "Python 3", 161 | "language": "python", 162 | "name": "python3" 163 | }, 164 | "language_info": { 165 | "codemirror_mode": { 166 | "name": "ipython", 167 | "version": 3 168 | }, 169 | "file_extension": ".py", 170 | "mimetype": "text/x-python", 171 | "name": "python", 172 | "nbconvert_exporter": "python", 173 | "pygments_lexer": "ipython3", 174 | "version": "3.5.2" 175 | } 176 | }, 177 | "nbformat": 4, 178 | "nbformat_minor": 0 179 | } 180 | -------------------------------------------------------------------------------- /ch02_basics/Concept07_loading_variables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `02`: Concept `07`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Loading variables" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Concept 06 was about saving variables. This one's about loading what you saved. Start by creating an interactive session:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "sess = tf.InteractiveSession()" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "deletable": true, 51 | "editable": true 52 | }, 53 | "source": [ 54 | "Create a boolean vector called `spikes` of the same dimensions as before:" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "metadata": { 61 | "collapsed": false, 62 | "deletable": true, 63 | "editable": true 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "spikes = tf.Variable([False]*8, name='spikes')" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": { 73 | "deletable": true, 74 | "editable": true 75 | }, 76 | "source": [ 77 | "Restored the variable data from disk, serve warm, and enjoy:" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 3, 83 | "metadata": { 84 | "collapsed": false, 85 | "deletable": true, 86 | "editable": true 87 | }, 88 | "outputs": [ 89 | { 90 | "name": "stdout", 91 | "output_type": "stream", 92 | "text": [ 93 | "file not found\n" 94 | ] 95 | } 96 | ], 97 | "source": [ 98 | "saver = tf.train.Saver()\n", 99 | "\n", 100 | "try:\n", 101 | " saver.restore(sess, 'spikes.ckpt')\n", 102 | " print(spikes.eval())\n", 103 | "except:\n", 104 | " print('file not found')\n" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": { 110 | "deletable": true, 111 | "editable": true 112 | }, 113 | "source": [ 114 | "Show's over, goodnight:" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 4, 120 | "metadata": { 121 | "collapsed": true, 122 | "deletable": true, 123 | "editable": true 124 | }, 125 | "outputs": [], 126 | "source": [ 127 | "sess.close()" 128 | ] 129 | } 130 | ], 131 | "metadata": { 132 | "kernelspec": { 133 | "display_name": "Python 3", 134 | "language": "python", 135 | "name": "python3" 136 | }, 137 | "language_info": { 138 | "codemirror_mode": { 139 | "name": "ipython", 140 | "version": 3 141 | }, 142 | "file_extension": ".py", 143 | "mimetype": "text/x-python", 144 | "name": "python", 145 | "nbconvert_exporter": "python", 146 | "pygments_lexer": "ipython3", 147 | "version": "3.5.2" 148 | } 149 | }, 150 | "nbformat": 4, 151 | "nbformat_minor": 0 152 | } 153 | -------------------------------------------------------------------------------- /ch02_basics/Concept08_TensorBoard.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ch `02`: Concept `08`" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Using TensorBoard" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "TensorBoard is a great way to visualize what's happening behind the code. \n", 22 | "\n", 23 | "In this example, we'll loop through some numbers to improve our guess of the average value. Then we can visualize the results on TensorBoard. \n", 24 | "\n", 25 | "Let's just set ourselves up with some data to work with:" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 1, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "import tensorflow as tf\n", 35 | "import numpy as np\n", 36 | "\n", 37 | "raw_data = np.random.normal(10, 1, 100)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "The moving average is defined as follows:" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 2, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "alpha = tf.constant(0.05)\n", 54 | "curr_value = tf.placeholder(tf.float32)\n", 55 | "prev_avg = tf.Variable(0.)\n", 56 | "\n", 57 | "update_avg = alpha * curr_value + (1 - alpha) * prev_avg" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "Here's what we care to visualize:" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 3, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "avg_hist = tf.summary.scalar(\"running_average\", update_avg)\n", 74 | "value_hist = tf.summary.scalar(\"incoming_values\", curr_value)\n", 75 | "\n", 76 | "merged = tf.summary.merge_all()\n", 77 | "writer = tf.summary.FileWriter(\"./logs\")" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": {}, 83 | "source": [ 84 | "Time to compute the moving averages. We'll also run the `merged` op to track how the values change:" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 4, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "name": "stdout", 94 | "output_type": "stream", 95 | "text": [ 96 | "9.247841477069203 0.4623921\n", 97 | "10.019298730125382 0.9402374\n", 98 | "11.971773672793464 1.4918143\n", 99 | "10.702923359431118 1.9523697\n", 100 | "11.667057068606786 2.4381042\n", 101 | "9.143228690197773 2.7733603\n", 102 | "9.457709656523708 3.1075776\n", 103 | "12.33999608545561 3.5691986\n", 104 | "9.543410229631846 3.8679092\n", 105 | "9.251442209932934 4.137086\n", 106 | "8.942198790212387 4.3773413\n", 107 | "11.019946553148321 4.709471\n", 108 | "11.430198193578404 5.0455074\n", 109 | "8.6213954795195 5.224302\n", 110 | "10.822108995258686 5.504192\n", 111 | "10.58310002901428 5.7581377\n", 112 | "10.20420365104725 5.9804406\n", 113 | "10.312154931419304 6.1970263\n", 114 | "10.545111153579882 6.4144306\n", 115 | "8.797765458370709 6.5335975\n", 116 | "8.56686695526782 6.6352606\n", 117 | "12.570525410195215 6.9320235\n", 118 | "11.543815331679966 7.162613\n", 119 | "10.320920832332627 7.320528\n", 120 | "10.423914230722215 7.4756975\n", 121 | "10.619258439210187 7.6328754\n", 122 | "9.101109809288653 7.7062874\n", 123 | "9.841278298991933 7.813037\n", 124 | "9.099955845561944 7.877383\n", 125 | "9.41973125623955 7.9545\n", 126 | "11.082836040691273 8.110917\n", 127 | "10.116690980009775 8.2112055\n", 128 | "9.402594289154155 8.270775\n", 129 | "10.925993106488145 8.403536\n", 130 | "10.243254438024696 8.495522\n", 131 | "9.477769687949733 8.544634\n", 132 | "9.351362392482848 8.58497\n", 133 | "9.242191906408548 8.617831\n", 134 | "12.123667719477677 8.793122\n", 135 | "10.076009517273803 8.857266\n", 136 | "9.74900667301667 8.901854\n", 137 | "10.830363231386094 8.998279\n", 138 | "8.861116004341559 8.991421\n", 139 | "10.007389057190906 9.042218\n", 140 | "10.769369554012615 9.128575\n", 141 | "12.971561516039255 9.3207245\n", 142 | "9.875042913748056 9.34844\n", 143 | "9.64616462712992 9.363326\n", 144 | "9.76634851758219 9.383477\n", 145 | "9.326634526001623 9.380634\n", 146 | "8.492294014699189 9.336217\n", 147 | "10.006073094467316 9.369709\n", 148 | "9.442892778881891 9.373368\n", 149 | "9.56787198816676 9.383093\n", 150 | "9.961494974707488 9.412013\n", 151 | "9.572285501643822 9.420026\n", 152 | "11.851354361154291 9.541592\n", 153 | "10.833573476171445 9.606191\n", 154 | "11.836376240592454 9.7177\n", 155 | "11.047626672901409 9.784197\n", 156 | "10.913818292308468 9.840677\n", 157 | "10.60857743486623 9.879072\n", 158 | "9.883074005285522 9.879272\n", 159 | "8.227633816367192 9.79669\n", 160 | "9.788906167639809 9.796301\n", 161 | "9.001469197788671 9.756559\n", 162 | "8.918205933440774 9.714642\n", 163 | "9.885320274459133 9.723175\n", 164 | "10.77521268535355 9.775776\n", 165 | "9.68349427673202 9.771162\n", 166 | "10.113753965038361 9.788292\n", 167 | "9.6597232190883 9.781863\n", 168 | "9.323572053812015 9.758949\n", 169 | "9.618532841629188 9.751928\n", 170 | "9.011944462852757 9.714929\n", 171 | "8.323719148832197 9.645369\n", 172 | "9.442883485401897 9.635244\n", 173 | "10.430287903497137 9.674997\n", 174 | "10.838671174170663 9.733181\n", 175 | "9.346134056876938 9.713829\n", 176 | "10.234079103904495 9.739841\n", 177 | "9.692786236742311 9.737489\n", 178 | "8.675916172925552 9.68441\n", 179 | "9.7006074487691 9.68522\n", 180 | "10.064675943184373 9.704192\n", 181 | "9.4021612098359 9.689091\n", 182 | "11.124410899430886 9.760857\n", 183 | "10.034575898474612 9.774543\n", 184 | "9.793431430576485 9.775487\n", 185 | "10.889420930759462 9.831183\n", 186 | "9.253518007206916 9.8023\n", 187 | "11.114827470151916 9.867927\n", 188 | "9.378996323459113 9.843479\n", 189 | "9.864306640072803 9.844521\n", 190 | "11.803316169037448 9.94246\n", 191 | "10.103049011008196 9.95049\n", 192 | "8.723187258083906 9.889125\n", 193 | "8.985505621881307 9.843944\n", 194 | "10.690261212066178 9.88626\n", 195 | "8.426969249944442 9.813295\n" 196 | ] 197 | } 198 | ], 199 | "source": [ 200 | "init = tf.global_variables_initializer()\n", 201 | "\n", 202 | "with tf.Session() as sess:\n", 203 | " sess.run(init)\n", 204 | " for i in range(len(raw_data)):\n", 205 | " summary_str, curr_avg = sess.run([merged, update_avg], feed_dict={curr_value: raw_data[i]})\n", 206 | " sess.run(tf.assign(prev_avg, curr_avg))\n", 207 | " print(raw_data[i], curr_avg)\n", 208 | " writer.add_summary(summary_str, i)" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "Check out the visualization by running TensorBoard from the terminal:\n", 216 | "\n", 217 | " $ tensorboard --logdir=path/to/logs" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": 5, 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "#made the logs be written successfully\n", 227 | "writer.close()" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": null, 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [] 236 | } 237 | ], 238 | "metadata": { 239 | "kernelspec": { 240 | "display_name": "Python 3", 241 | "language": "python", 242 | "name": "python3" 243 | }, 244 | "language_info": { 245 | "codemirror_mode": { 246 | "name": "ipython", 247 | "version": 3 248 | }, 249 | "file_extension": ".py", 250 | "mimetype": "text/x-python", 251 | "name": "python", 252 | "nbconvert_exporter": "python", 253 | "pygments_lexer": "ipython3", 254 | "version": "3.6.4" 255 | } 256 | }, 257 | "nbformat": 4, 258 | "nbformat_minor": 1 259 | } 260 | -------------------------------------------------------------------------------- /ch02_basics/Concept09_queue.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ch `02`: Concept `09`" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Using Queues" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "If you have a lot of training data, you probably don't want to load it all into memory at once. The QueueRunner in TensorFlow is a tool to efficiently employ a queue data-structure in a multi-threaded way." 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import tensorflow as tf\n", 33 | "import numpy as np" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "We will be running multiple threads, so let's figure out the number of CPUs:" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": { 47 | "collapsed": false 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "import multiprocessing\n", 52 | "NUM_THREADS = multiprocessing.cpu_count()" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": {}, 58 | "source": [ 59 | "Generate some fake data to work with:" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 3, 65 | "metadata": { 66 | "collapsed": true 67 | }, 68 | "outputs": [], 69 | "source": [ 70 | "xs = np.random.randn(100, 3)\n", 71 | "ys = np.random.randint(0, 2, size=100)" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": {}, 77 | "source": [ 78 | "Here's a couple concrete examples of our data:" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 4, 84 | "metadata": { 85 | "collapsed": false 86 | }, 87 | "outputs": [ 88 | { 89 | "name": "stdout", 90 | "output_type": "stream", 91 | "text": [ 92 | "Input [ 1.46034759 0.71462742 0.73288402] ---> Output 0\n", 93 | "Input [ 1.1537654 -0.09128405 0.08036941] ---> Output 1\n", 94 | "Input [-0.61164559 -0.19188485 0.06064167] ---> Output 0\n", 95 | "Input [ 0.1007337 0.34815357 0.24346031] ---> Output 0\n", 96 | "Input [-1.25581117 1.44738085 1.15035257] ---> Output 0\n" 97 | ] 98 | } 99 | ], 100 | "source": [ 101 | "xs_and_ys = zip(xs, ys)\n", 102 | "for _ in range(5):\n", 103 | " x, y = next(xs_and_ys)\n", 104 | " print('Input {} ---> Output {}'.format(x, y))" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "Define a queue:" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 5, 117 | "metadata": { 118 | "collapsed": true 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "queue = tf.FIFOQueue(capacity=1000, dtypes=[tf.float32, tf.int32])" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "Set up the enqueue and dequeue ops:" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 6, 135 | "metadata": { 136 | "collapsed": true 137 | }, 138 | "outputs": [], 139 | "source": [ 140 | "enqueue_op = queue.enqueue_many([xs, ys])\n", 141 | "x_op, y_op = queue.dequeue()" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "metadata": {}, 147 | "source": [ 148 | "Define a QueueRunner:" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 7, 154 | "metadata": { 155 | "collapsed": false 156 | }, 157 | "outputs": [], 158 | "source": [ 159 | "qr = tf.train.QueueRunner(queue, [enqueue_op] * 4)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "Now that all variables and ops have been defined, let's get started with a session:" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 8, 172 | "metadata": { 173 | "collapsed": true 174 | }, 175 | "outputs": [], 176 | "source": [ 177 | "sess = tf.InteractiveSession()" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "Create threads for the QueueRunner:" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 9, 190 | "metadata": { 191 | "collapsed": true 192 | }, 193 | "outputs": [], 194 | "source": [ 195 | "coord = tf.train.Coordinator()\n", 196 | "enqueue_threads = qr.create_threads(sess, coord=coord, start=True)" 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": {}, 202 | "source": [ 203 | "Test out dequeueing:" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 10, 209 | "metadata": { 210 | "collapsed": false 211 | }, 212 | "outputs": [ 213 | { 214 | "name": "stdout", 215 | "output_type": "stream", 216 | "text": [ 217 | "[ 1.46034753 0.71462744 0.73288405] 0\n", 218 | "[ 1.15376544 -0.09128405 0.08036941] 1\n", 219 | "[-0.61164558 -0.19188486 0.06064167] 0\n", 220 | "[ 0.1007337 0.34815356 0.24346031] 0\n", 221 | "[-1.25581121 1.4473809 1.1503526 ] 0\n", 222 | "[ 0.60369009 -0.87942719 -1.37121975] 1\n", 223 | "[ 1.30641925 1.55316997 1.01789773] 0\n", 224 | "[ 0.0575242 0.59463078 0.47600508] 1\n", 225 | "[-1.22782397 -0.86792755 1.37459588] 1\n", 226 | "[-0.27896652 0.51645088 1.36873603] 0\n", 227 | "[-0.34542757 0.79360306 0.32000065] 0\n", 228 | "[-0.46792462 -0.31817994 0.91739392] 0\n", 229 | "[ 0.24787657 0.83848852 1.16125166] 0\n", 230 | "[-0.46220389 -0.09412029 -0.9981451 ] 1\n", 231 | "[ 0.06739734 -1.08405316 -0.3582162 ] 1\n", 232 | "[-1.2644819 -0.27479929 1.15882337] 1\n", 233 | "[-0.68015367 -0.10199564 1.4274267 ] 0\n", 234 | "[-0.48884565 -0.39484504 0.1496018 ] 1\n", 235 | "[ 1.48414564 -0.43943462 -0.12646018] 0\n", 236 | "[ 0.49450573 0.42091215 -0.17693481] 0\n", 237 | "[ 0.02265234 0.99832052 0.26808155] 1\n", 238 | "[-0.94086462 1.67000341 0.92434174] 1\n", 239 | "[-0.50961769 -0.39044595 -0.5737586 ] 0\n", 240 | "[-0.95702702 0.61196166 -0.86487901] 1\n", 241 | "[-0.6125344 -0.30916786 -1.06602347] 1\n", 242 | "[-1.91383719 0.26860073 0.50380921] 1\n", 243 | "[-0.14638679 0.11614402 1.36613548] 1\n", 244 | "[-0.56817967 1.4221288 0.99365205] 0\n", 245 | "[-0.04597072 0.43875724 -0.4809106 ] 0\n", 246 | "[-0.2000681 -0.2384561 0.06599616] 0\n", 247 | "[ 0.5862993 0.85386461 0.82285357] 1\n", 248 | "[ 1.64371336 -0.46838599 0.22755136] 0\n", 249 | "[ 0.21683638 -0.96399426 1.78278649] 1\n", 250 | "[ 0.03778305 2.49208736 0.07467758] 0\n", 251 | "[-1.48958826 -0.11699235 0.98281074] 1\n", 252 | "[-0.27623582 -0.41658697 -0.89554274] 0\n", 253 | "[-1.64742625 1.83507264 -0.76936585] 0\n", 254 | "[-1.5386405 0.14272654 0.17047048] 1\n", 255 | "[ 0.63654041 1.75451732 -1.14198494] 0\n", 256 | "[-0.57061732 0.11121389 1.39394116] 1\n", 257 | "[ 1.94736981 -0.36588097 0.54801333] 1\n", 258 | "[-0.56976408 -1.36990237 -0.9922803 ] 1\n", 259 | "[-2.47653961 1.19603479 -0.3038739 ] 0\n", 260 | "[-0.76740891 -0.49611184 0.47167206] 0\n", 261 | "[ 1.62004089 0.13268068 0.28845155] 0\n", 262 | "[-0.91749012 -0.30151108 -0.08271972] 0\n", 263 | "[-0.21053326 -0.16114895 -0.52424961] 1\n", 264 | "[ 0.19968066 0.2387522 2.0314014 ] 0\n", 265 | "[-0.29072183 0.53720349 -0.38972732] 0\n", 266 | "[-0.85891634 -0.26684314 -1.91741192] 1\n", 267 | "[-2.07077003 1.97488022 -0.92741841] 0\n", 268 | "[ 2.37270904 2.19385314 -0.29643178] 0\n", 269 | "[-0.18054648 -0.1651988 1.70858753] 1\n", 270 | "[-0.27851281 -0.13095042 0.30613536] 1\n", 271 | "[-0.13653868 -0.14431253 1.3018136 ] 1\n", 272 | "[-1.79938364 0.26698261 -0.3283855 ] 0\n", 273 | "[-0.43491617 -0.8737886 -0.48871836] 1\n", 274 | "[-0.27275884 0.08004636 -0.34334385] 0\n", 275 | "[-0.06538768 -0.47280514 -1.82918119] 0\n", 276 | "[ 1.72329473 0.6359638 1.53474641] 0\n", 277 | "[ 0.88200653 0.87051851 0.17676826] 1\n", 278 | "[-2.22127795 -0.39812142 0.69118947] 0\n", 279 | "[-0.90146214 0.23153968 -1.07890677] 0\n", 280 | "[-0.66513097 -0.74897975 -1.9886812 ] 0\n", 281 | "[ 0.95217085 -0.1361241 -0.81558466] 1\n", 282 | "[ 0.97319698 0.10349847 1.78010297] 0\n", 283 | "[ 0.54321396 1.10134006 -1.03641176] 1\n", 284 | "[ 0.46445891 0.56387979 0.10383373] 0\n", 285 | "[ 0.22231635 -1.20880091 0.20125042] 1\n", 286 | "[ 0.56338882 -0.76195502 -0.33035895] 0\n", 287 | "[ 0.13885871 0.62347603 0.32560909] 0\n", 288 | "[-0.63413048 0.19185983 1.65251637] 1\n", 289 | "[ 0.81965917 -0.14427175 -0.9943186 ] 0\n", 290 | "[ 1.98786604 -1.38118052 -0.34296793] 0\n", 291 | "[-0.49028778 -0.30242845 0.81718981] 0\n", 292 | "[ 0.48434621 -1.3200016 -0.32307461] 0\n", 293 | "[-0.91041267 -0.34315997 0.71205115] 0\n", 294 | "[ 0.61457998 -0.85814965 0.6939835 ] 0\n", 295 | "[-0.40195578 -1.11846507 -0.19713871] 1\n", 296 | "[-0.47889531 -0.75685191 1.68955612] 1\n", 297 | "[ 1.51117146 -2.23529124 1.13895822] 0\n", 298 | "[-0.00831293 -0.50950557 0.08648733] 1\n", 299 | "[-0.47011089 1.04781067 -0.05893843] 1\n", 300 | "[-0.34855339 -0.5695411 -0.12196264] 1\n", 301 | "[-0.47251806 -0.49479187 0.27609721] 0\n", 302 | "[-2.04546118 -0.16185458 1.42348552] 0\n", 303 | "[-0.67136103 -0.16650072 0.3609505 ] 0\n", 304 | "[ 1.22566068 1.18665588 -1.87292075] 0\n", 305 | "[-0.80474126 -0.1114784 0.00531922] 1\n", 306 | "[ 0.62691861 -3.26328206 -0.39003551] 0\n", 307 | "[-0.77470082 -1.23692167 -1.55790484] 0\n", 308 | "[-0.49005547 -0.19645052 -0.21566501] 1\n", 309 | "[-0.44095206 -0.13273652 -0.59810853] 0\n", 310 | "[-0.9750855 -0.46043435 0.06064714] 1\n", 311 | "[-0.181191 -0.12452056 0.23064452] 1\n", 312 | "[-0.34818363 -1.13179028 1.20628965] 0\n", 313 | "[-1.58196092 -1.3506341 -2.05767131] 1\n", 314 | "[-1.66225421 -0.43541616 1.55258 ] 0\n", 315 | "[-0.12949325 -0.15456693 0.04389611] 0\n", 316 | "[ 0.24592777 0.11407969 -0.31221709] 1\n" 317 | ] 318 | } 319 | ], 320 | "source": [ 321 | "for _ in range(100):\n", 322 | " if coord.should_stop():\n", 323 | " break\n", 324 | " x, y = sess.run([x_op, y_op])\n", 325 | " print(x, y)\n", 326 | "coord.request_stop()\n", 327 | "coord.join(enqueue_threads)" 328 | ] 329 | } 330 | ], 331 | "metadata": { 332 | "kernelspec": { 333 | "display_name": "Python 3", 334 | "language": "python", 335 | "name": "python3" 336 | }, 337 | "language_info": { 338 | "codemirror_mode": { 339 | "name": "ipython", 340 | "version": 3 341 | }, 342 | "file_extension": ".py", 343 | "mimetype": "text/x-python", 344 | "name": "python", 345 | "nbconvert_exporter": "python", 346 | "pygments_lexer": "ipython3", 347 | "version": "3.5.2" 348 | } 349 | }, 350 | "nbformat": 4, 351 | "nbformat_minor": 2 352 | } 353 | -------------------------------------------------------------------------------- /ch02_basics/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 2 2 | 3 |

4 | 5 | Before implementing machine learning algorithms, let’s first familiarize ourselves with how to use TensorFlow. You’re going to get your hands dirty writing some simple code right away! This chapter will cover some essential advantages of TensorFlow to convince you it’s the machine learning library of choice. 6 | 7 | - **Concept 1**: Defining tensors 8 | - **Concept 2**: Evaluating ops 9 | - **Concept 3**: Interactive session 10 | - **Concept 4**: Session loggings 11 | - **Concept 5**: Variables 12 | - **Concept 6**: Saving variables 13 | - **Concept 7**: Loading variables 14 | - **Concept 8**: TensorBoard 15 | 16 | --- 17 | 18 | * Listing 2-4: `types.py` 19 | * Listing 5-6: `main.py` 20 | * Listing 7: `interactive_session.py` 21 | * Listing 8: `logging.py` 22 | * Listing 9: `spikes.py` 23 | * Listing 10: `saving_vars.py` 24 | * Listing 11: `loading_vars.py` 25 | * Listing 12-15: `moving_avg.py` -------------------------------------------------------------------------------- /ch02_basics/gradient.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | def my_loss_function(var, data): 4 | return tf.abs(tf.subtract(var, data)) 5 | 6 | def my_other_loss_function(var, data): 7 | return tf.square(tf.subtract(var, data)) 8 | 9 | data = tf.placeholder(tf.float32) 10 | var = tf.Variable(1.) 11 | loss = my_loss_function(var, data) 12 | var_grad = tf.gradients(loss, [var])[0] 13 | 14 | with tf.Session() as sess: 15 | sess.run(tf.global_variables_initializer()) 16 | var_grad_val = sess.run(var_grad, feed_dict={data: 4}) 17 | print(var_grad_val) 18 | -------------------------------------------------------------------------------- /ch02_basics/interactive_session.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | sess = tf.InteractiveSession() 3 | 4 | matrix = tf.constant([[1., 2.]]) 5 | negMatrix = tf.neg(matrix) 6 | 7 | result = negMatrix.eval() 8 | print(result) 9 | sess.close() 10 | -------------------------------------------------------------------------------- /ch02_basics/loading_vars.py: -------------------------------------------------------------------------------- 1 | # # Loading Variables in TensorFlow 2 | 3 | import tensorflow as tf 4 | sess = tf.InteractiveSession() 5 | 6 | 7 | # Create a boolean vector called `spike` to locate a sudden spike in data. 8 | # 9 | # Since all variables must be initialized, initialize the variable by calling `run()` on its `initializer`. 10 | 11 | spikes = tf.Variable([False]*8, name='spikes') 12 | saver = tf.train.Saver() 13 | 14 | saver.restore(sess, "spikes.ckpt") 15 | print(spikes.eval()) 16 | 17 | sess.close() 18 | -------------------------------------------------------------------------------- /ch02_basics/log_example.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | matrix = tf.constant([[1., 2.]]) 4 | negMatrix = tf.neg(matrix) 5 | 6 | with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess: 7 | result = sess.run(negMatrix) 8 | 9 | print(result) 10 | -------------------------------------------------------------------------------- /ch02_basics/logging_example.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | matrix = tf.constant([[1, 2]]) 4 | neg_matrix = tf.neg(matrix) 5 | 6 | with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess: 7 | result = sess.run(neg_matrix) 8 | 9 | print result 10 | -------------------------------------------------------------------------------- /ch02_basics/main.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | x = tf.constant([[1, 2]]) 5 | neg_x = tf.neg(x) 6 | 7 | print(neg_x) 8 | 9 | with tf.Session() as sess: 10 | result = sess.run(neg_x) 11 | print(result) 12 | -------------------------------------------------------------------------------- /ch02_basics/moving_avg.py: -------------------------------------------------------------------------------- 1 | ## Using TensorBoard 2 | 3 | # mkdir logs 4 | 5 | import tensorflow as tf 6 | import numpy as np 7 | 8 | raw_data = np.random.normal(10, 1, 100) 9 | 10 | alpha = tf.constant(0.05) 11 | curr_value = tf.placeholder(tf.float32) 12 | prev_avg = tf.Variable(0.) 13 | update_avg = alpha * curr_value + (1 - alpha) * prev_avg 14 | 15 | avg_hist = tf.summary.scalar("running_avg", update_avg) 16 | value_hist = tf.summary.scalar("incoming_values", curr_value) 17 | merged = tf.summary.merge_all() 18 | writer = tf.summary.FileWriter('./logs') 19 | init = tf.global_variables_initializer() 20 | 21 | with tf.Session() as sess: 22 | sess.run(init) 23 | for i in range(len(raw_data)): 24 | summary_str, curr_avg = sess.run([merged, update_avg], feed_dict={curr_value: raw_data[i]}) 25 | sess.run(tf.assign(prev_avg, curr_avg)) 26 | print(raw_data[i], curr_avg) 27 | writer.add_summary(summary_str, i) 28 | 29 | writer.close() 30 | -------------------------------------------------------------------------------- /ch02_basics/saving_vars.py: -------------------------------------------------------------------------------- 1 | # # Saving Variables in TensorFlow 2 | 3 | import tensorflow as tf 4 | sess = tf.InteractiveSession() 5 | 6 | 7 | # Create a boolean vector called `spike` to locate a sudden spike in data. 8 | # 9 | # Since all variables must be initialized, initialize the variable by calling `run()` on its `initializer`. 10 | 11 | raw_data = [1., 2., 8., -1., 0., 5.5, 6., 13] 12 | spikes = tf.Variable([False] * len(raw_data), name='spikes') 13 | spikes.initializer.run() 14 | 15 | 16 | # The saver op will enable saving and restoring 17 | 18 | saver = tf.train.Saver() 19 | 20 | 21 | # Loop through the data and update the spike variable when there is a significant increase 22 | 23 | for i in range(1, len(raw_data)): 24 | if raw_data[i] - raw_data[i-1] > 5: 25 | spikes_val = spikes.eval() 26 | spikes_val[i] = True 27 | updater = tf.assign(spikes, spikes_val) 28 | updater.eval() 29 | 30 | 31 | save_path = saver.save(sess, "spikes.ckpt") 32 | print("spikes data saved in file: %s" % save_path) 33 | 34 | 35 | sess.close() 36 | -------------------------------------------------------------------------------- /ch02_basics/spikes.py: -------------------------------------------------------------------------------- 1 | # # Using Variables in TensorFlow 2 | 3 | import tensorflow as tf 4 | sess = tf.InteractiveSession() 5 | 6 | 7 | # Create a boolean variable called `spike` to detect sudden a sudden increase in a series of numbers. 8 | # 9 | # Since all variables must be initialized, initialize the variable by calling `run()` on its `initializer`. 10 | 11 | raw_data = [1., 2., 8., -1., 0., 5.5, 6., 13] 12 | spike = tf.Variable(False) 13 | spike.initializer.run() 14 | 15 | 16 | # Loop through the data and update the spike variable when there is a significant increase 17 | 18 | for i in range(1, len(raw_data)): 19 | if raw_data[i] - raw_data[i-1] > 5: 20 | updater = tf.assign(spike, tf.constant(True)) 21 | updater.eval() 22 | else: 23 | tf.assign(spike, False).eval() 24 | print("Spike", spike.eval()) 25 | 26 | sess.close() 27 | 28 | 29 | -------------------------------------------------------------------------------- /ch02_basics/types.py: -------------------------------------------------------------------------------- 1 | # # Tensor Types 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | 7 | # Define a 2x2 matrix in 3 different ways 8 | 9 | m1 = [[1.0, 2.0], [3.0, 4.0]] 10 | m2 = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) 11 | m3 = tf.constant([[1.0, 2.0], [3.0, 4.0]]) 12 | 13 | 14 | print(type(m1)) 15 | print(type(m2)) 16 | print(type(m3)) 17 | 18 | 19 | # Create tensor objects out of various types 20 | 21 | t1 = tf.convert_to_tensor(m1, dtype=tf.float32) 22 | t2 = tf.convert_to_tensor(m2, dtype=tf.float32) 23 | t3 = tf.convert_to_tensor(m3, dtype=tf.float32) 24 | 25 | 26 | print(type(t1)) 27 | print(type(t2)) 28 | print(type(t3)) 29 | -------------------------------------------------------------------------------- /ch03_regression/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 3 2 | 3 |

4 | 5 | Remember science courses back in grade school? It might have been a while ago, or who knows - maybe you’re in grade school now starting your journey in machine learning early. Either way, whether you took biology, chemistry, or physics, a common technique to analyze data is to plot how changing one variable affects the other. 6 | 7 | Imagine plotting the correlation between rainfall frequency and agriculture production. You may observe that an increase in rainfall produces an increase in agriculture rate. Fitting a line to these data points enables you to make predictions about the agriculture rate under different rain conditions. If you discover the underlying function from a few data points, then that learned function empowers you to make predictions about the values of unseen data. 8 | 9 | Regression is a study of how to best fit a curve to summarize your data. It is one of the most powerful and well-studied types of supervised learning algorithms. In regression, we try to understand the data points by discovering the curve that might have generated them. In doing so, we seek an explanation for why the given data is scattered the way it is. The best fit curve gives us a model for explaining how the dataset might have been produced. 10 | 11 | This chapter will show you how to formulate a real world problem to use regression. As you’ll see, TensorFlow is just the right tool that endows us with some of the most powerful predictors. 12 | 13 | - **Concept 1**: Linear regression 14 | - **Concept 2**: Polynomial regression 15 | - **Concept 3**: Regularization 16 | 17 | --- 18 | 19 | * Listing 1-2: `simple_model.py` 20 | * Listing 3: `polynomial_model.py` 21 | * Listing 4-5: `regularization.py` 22 | * Listing 6: `data_reader.py` 23 | -------------------------------------------------------------------------------- /ch03_regression/data_reader.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import time 3 | 4 | # 5 | # crime14_freq = data_reader.read('crimes_2014.csv', 1, '%d-%b-%y %H:%M:%S', 2014) 6 | # freq = read('311.csv', 0, '%m/%d/%Y', 2014) 7 | 8 | def read(filename, date_idx, date_parse, year, bucket=7): 9 | 10 | days_in_year = 365 11 | 12 | # Create initial frequency map 13 | freq = {} 14 | for period in range(0, int(days_in_year/bucket)): 15 | freq[period] = 0 16 | 17 | # Read data and aggregate crimes per day 18 | with open(filename, 'rb') as csvfile: 19 | csvreader = csv.reader(csvfile) 20 | csvreader.next() 21 | for row in csvreader: 22 | if row[date_idx] == '': 23 | continue 24 | t = time.strptime(row[date_idx], date_parse) 25 | if t.tm_year == year and t.tm_yday < (days_in_year-1): 26 | freq[int(t.tm_yday / bucket)] += 1 27 | 28 | return freq 29 | 30 | if __name__ == '__main__': 31 | freq = read('311.csv', 0, '%m/%d/%Y', 2014) 32 | print freq -------------------------------------------------------------------------------- /ch04_classification/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 4 2 | 3 |

4 | 5 | Imagine an advertisement agency collecting information about user interactions to decide what type of ad to show. That’s not so uncommon. Google, Twitter, Facebook, and other big tech giants that rely on ads have creepy-good personal profiles of their users to help deliver personalized ads. A user who’s recently searched for gaming keyboards or graphics cards is probably more likely to click ads about the latest and greatest video games. 6 | 7 | It may be difficult to cater a specially crafted advertisement for each individual, so grouping users into categories is a common technique. For example, a user may be categorized as a “gamer” to receive relevant video game related ads. 8 | 9 | Machine learning has been the go-to tool to accomplish such as task. At the most fundamental level, machine learning practitioners want to build a tool to help them understand data. Being able to label data items into separate categories is an excellent way to characterize it for specific needs. 10 | 11 | The previous chapter dealt with regression, which was about fitting a curve to data. If you recall, the best-fit curve is a function that takes as input a data item and assigns it a number. Creating a machine learning model that instead assigns discrete labels to its inputs is called classification. It is a supervised learning algorithm for dealing with discrete output. (Each discrete value is called a class.) The input is typically a feature vector, and the output is a class. If there are only two class labels (for example, True/False, On/Off, Yes/No), then we call this learning algorithm a binary classifier. Otherwise, it’s called a multiclass classifier. 12 | 13 | - **Concept 1**: Linear regression for classification 14 | - **Concept 2**: Logistic regression 15 | - **Concept 3**: 2D Logistic regression 16 | - **Concept 4**: Softmax classification 17 | 18 | --- 19 | 20 | * Listing 1-3: `linear_1d.py` 21 | * Listing 4: `logistic_1d.py` 22 | * Listing 5: `logistic_2d.py` 23 | * Listing 6-10: `softmax.py` 24 | -------------------------------------------------------------------------------- /ch04_classification/linear_1d.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | x_label0 = np.random.normal(5, 1, 10) 6 | x_label1 = np.random.normal(2, 1, 10) 7 | xs = np.append(x_label0, x_label1) 8 | labels = [0.] * len(x_label0) + [1.] * len(x_label1) 9 | 10 | plt.scatter(xs, labels) 11 | 12 | learning_rate = 0.001 13 | training_epochs = 1000 14 | 15 | X = tf.placeholder("float") 16 | Y = tf.placeholder("float") 17 | 18 | def model(X, w): 19 | return tf.add(tf.mul(w[1], tf.pow(X, 1)), 20 | tf.mul(w[0], tf.pow(X, 0))) 21 | 22 | w = tf.Variable([0., 0.], name="parameters") 23 | y_model = model(X, w) 24 | cost = tf.reduce_sum(tf.square(Y-y_model)) 25 | 26 | train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) 27 | correct_prediction = tf.equal(Y, tf.to_float(tf.greater(y_model, 0.5))) 28 | accuracy = tf.reduce_mean(tf.to_float(correct_prediction)) 29 | 30 | sess = tf.Session() 31 | init = tf.initialize_all_variables() 32 | sess.run(init) 33 | 34 | for epoch in range(training_epochs): 35 | sess.run(train_op, feed_dict={X: xs, Y: labels}) 36 | current_cost = sess.run(cost, feed_dict={X: xs, Y: labels}) 37 | print(epoch, current_cost) 38 | 39 | w_val = sess.run(w) 40 | print('learned parameters', w_val) 41 | 42 | print('accuracy', sess.run(accuracy, feed_dict={X: xs, Y: labels})) 43 | 44 | sess.close() 45 | 46 | all_xs = np.linspace(0, 10, 100) 47 | plt.plot(all_xs, all_xs*w_val[1] + w_val[0]) 48 | plt.show() -------------------------------------------------------------------------------- /ch04_classification/logistic_1d.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | 5 | learning_rate = 0.01 6 | training_epochs = 1000 7 | 8 | def sigmoid(x): 9 | return 1. / (1. + np.exp(-x)) 10 | 11 | x1 = np.random.normal(-4, 2, 1000) 12 | x2 = np.random.normal(4, 2, 1000) 13 | xs = np.append(x1, x2) 14 | ys = np.asarray([0.] * len(x1) + [1.] * len(x2)) 15 | 16 | plt.scatter(xs, ys) 17 | 18 | X = tf.placeholder(tf.float32, shape=(None,), name="x") 19 | Y = tf.placeholder(tf.float32, shape=(None,), name="y") 20 | w = tf.Variable([0., 0.], name="parameter", trainable=True) 21 | y_model = tf.sigmoid(-(w[1] * X + w[0])) 22 | cost = tf.reduce_mean(-tf.log(y_model * Y + (1 - y_model) * (1 - Y))) 23 | 24 | train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) 25 | 26 | with tf.Session() as sess: 27 | sess.run(tf.initialize_all_variables()) 28 | prev_err = 0 29 | for epoch in range(training_epochs): 30 | err, _ = sess.run([cost, train_op], {X: xs, Y: ys}) 31 | print(epoch, err) 32 | if abs(prev_err - err) < 0.0001: 33 | break 34 | prev_err = err 35 | w_val = sess.run(w, {X: xs, Y: ys}) 36 | 37 | all_xs = np.linspace(-10, 10, 100) 38 | plt.plot(all_xs, sigmoid(all_xs * w_val[1] + w_val[0])) 39 | plt.show() 40 | -------------------------------------------------------------------------------- /ch04_classification/logistic_2d.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | 5 | learning_rate = 0.1 6 | training_epochs = 2000 7 | 8 | 9 | def sigmoid(x): 10 | return 1. / (1. + np.exp(-x)) 11 | 12 | x1_label1 = np.random.normal(3, 1, 1000) 13 | x2_label1 = np.random.normal(2, 1, 1000) 14 | x1_label2 = np.random.normal(7, 1, 1000) 15 | x2_label2 = np.random.normal(6, 1, 1000) 16 | x1s = np.append(x1_label1, x1_label2) 17 | x2s = np.append(x2_label1, x2_label2) 18 | ys = np.asarray([0.] * len(x1_label1) + [1.] * len(x1_label2)) 19 | 20 | X1 = tf.placeholder(tf.float32, shape=(None,), name="x1") 21 | X2 = tf.placeholder(tf.float32, shape=(None,), name="x2") 22 | Y = tf.placeholder(tf.float32, shape=(None,), name="y") 23 | w = tf.Variable([0., 0., 0.], name="w", trainable=True) 24 | 25 | y_model = tf.sigmoid(-(w[2] * X2 + w[1] * X1 + w[0])) 26 | cost = tf.reduce_mean(-tf.log(y_model) * Y -tf.log(1 - y_model) * (1 - Y)) 27 | train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) 28 | 29 | with tf.Session() as sess: 30 | sess.run(tf.initialize_all_variables()) 31 | prev_err = 0 32 | for epoch in range(training_epochs): 33 | err, _ = sess.run([cost, train_op], {X1: x1s, X2: x2s, Y: ys}) 34 | print(epoch, err) 35 | if abs(prev_err - err) < 0.0001: 36 | break 37 | prev_err = err 38 | 39 | w_val = sess.run(w) 40 | 41 | x1_boundary, x2_boundary = [], [] 42 | for x1_test in np.linspace(0, 10, 100): 43 | for x2_test in np.linspace(0, 10, 100): 44 | z = sigmoid(-x2_test*w_val[2] - x1_test*w_val[1] - w_val[0]) 45 | if abs(z - 0.5) < 0.01: 46 | x1_boundary.append(x1_test) 47 | x2_boundary.append(x2_test) 48 | 49 | plt.scatter(x1_boundary, x2_boundary, c='b', marker='o', s=20) 50 | plt.scatter(x1_label1, x2_label1, c='r', marker='x', s=20) 51 | plt.scatter(x1_label2, x2_label2, c='g', marker='1', s=20) 52 | 53 | plt.show() 54 | -------------------------------------------------------------------------------- /ch04_classification/softmax.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | 5 | learning_rate = 0.01 6 | training_epochs = 1000 7 | num_labels = 3 8 | batch_size = 100 9 | 10 | x1_label0 = np.random.normal(1, 1, (100, 1)) 11 | x2_label0 = np.random.normal(1, 1, (100, 1)) 12 | x1_label1 = np.random.normal(5, 1, (100, 1)) 13 | x2_label1 = np.random.normal(4, 1, (100, 1)) 14 | x1_label2 = np.random.normal(8, 1, (100, 1)) 15 | x2_label2 = np.random.normal(0, 1, (100, 1)) 16 | 17 | plt.scatter(x1_label0, x2_label0, c='r', marker='o', s=60) 18 | plt.scatter(x1_label1, x2_label1, c='g', marker='x', s=60) 19 | plt.scatter(x1_label2, x2_label2, c='b', marker='_', s=60) 20 | plt.show() 21 | 22 | xs_label0 = np.hstack((x1_label0, x2_label0)) 23 | xs_label1 = np.hstack((x1_label1, x2_label1)) 24 | xs_label2 = np.hstack((x1_label2, x2_label2)) 25 | 26 | xs = np.vstack((xs_label0, xs_label1, xs_label2)) 27 | labels = np.matrix([[1., 0., 0.]] * len(x1_label0) + [[0., 1., 0.]] * len(x1_label1) + [[0., 0., 1.]] * len(x1_label2)) 28 | 29 | arr = np.arange(xs.shape[0]) 30 | np.random.shuffle(arr) 31 | xs = xs[arr, :] 32 | labels = labels[arr, :] 33 | 34 | test_x1_label0 = np.random.normal(1, 1, (10, 1)) 35 | test_x2_label0 = np.random.normal(1, 1, (10, 1)) 36 | test_x1_label1 = np.random.normal(5, 1, (10, 1)) 37 | test_x2_label1 = np.random.normal(4, 1, (10, 1)) 38 | test_x1_label2 = np.random.normal(8, 1, (10, 1)) 39 | test_x2_label2 = np.random.normal(0, 1, (10, 1)) 40 | test_xs_label0 = np.hstack((test_x1_label0, test_x2_label0)) 41 | test_xs_label1 = np.hstack((test_x1_label1, test_x2_label1)) 42 | test_xs_label2 = np.hstack((test_x1_label2, test_x2_label2)) 43 | 44 | test_xs = np.vstack((test_xs_label0, test_xs_label1, test_xs_label2)) 45 | test_labels = np.matrix([[1., 0., 0.]] * 10 + [[0., 1., 0.]] * 10 + [[0., 0., 1.]] * 10) 46 | 47 | train_size, num_features = xs.shape 48 | 49 | X = tf.placeholder("float", shape=[None, num_features]) 50 | Y = tf.placeholder("float", shape=[None, num_labels]) 51 | 52 | W = tf.Variable(tf.zeros([num_features, num_labels])) 53 | b = tf.Variable(tf.zeros([num_labels])) 54 | y_model = tf.nn.softmax(tf.matmul(X, W) + b) 55 | 56 | cost = -tf.reduce_sum(Y * tf.log(y_model)) 57 | train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) 58 | 59 | correct_prediction = tf.equal(tf.argmax(y_model, 1), tf.argmax(Y, 1)) 60 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 61 | 62 | with tf.Session() as sess: 63 | tf.initialize_all_variables().run() 64 | 65 | for step in xrange(training_epochs * train_size // batch_size): 66 | offset = (step * batch_size) % train_size 67 | batch_xs = xs[offset:(offset + batch_size), :] 68 | batch_labels = labels[offset:(offset + batch_size)] 69 | err, _ = sess.run([cost, train_op], feed_dict={X: batch_xs, Y: batch_labels}) 70 | print (step, err) 71 | 72 | W_val = sess.run(W) 73 | print('w', W_val) 74 | b_val = sess.run(b) 75 | print('b', b_val) 76 | print "accuracy", accuracy.eval(feed_dict={X: test_xs, Y: test_labels}) 77 | -------------------------------------------------------------------------------- /ch05_clustering/Concept02_segmentation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `05`: Concept `02`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Segmentation" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Import libraries and define hyper-parameters:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "import numpy as np\n", 45 | "from bregman.suite import *\n", 46 | "\n", 47 | "k = 2\n", 48 | "segment_size = 50\n", 49 | "max_iterations = 100" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": { 55 | "deletable": true, 56 | "editable": true 57 | }, 58 | "source": [ 59 | "Define functions to get the chromogram and the dataset:" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 2, 65 | "metadata": { 66 | "collapsed": true, 67 | "deletable": true, 68 | "editable": true 69 | }, 70 | "outputs": [], 71 | "source": [ 72 | "chromo = tf.placeholder(tf.float32)\n", 73 | "max_freqs = tf.argmax(chromo, 0)\n", 74 | "\n", 75 | "def get_chromogram(audio_file):\n", 76 | " F = Chromagram(audio_file, nfft=16384, wfft=8192, nhop=2205)\n", 77 | " return F.X\n", 78 | "\n", 79 | "def get_dataset(sess, audio_file):\n", 80 | " chromo_data = get_chromogram(audio_file)\n", 81 | " print('chromo_data', np.shape(chromo_data))\n", 82 | " chromo_length = np.shape(chromo_data)[1]\n", 83 | " xs = []\n", 84 | " for i in range(chromo_length/segment_size):\n", 85 | " chromo_segment = chromo_data[:, i*segment_size:(i+1)*segment_size]\n", 86 | " x = extract_feature_vector(sess, chromo_segment)\n", 87 | " if len(xs) == 0:\n", 88 | " xs = x\n", 89 | " else:\n", 90 | " xs = np.vstack((xs, x))\n", 91 | " return xs" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": { 97 | "deletable": true, 98 | "editable": true 99 | }, 100 | "source": [ 101 | "As required for the k-means algorithm, specify the assignment and re-centering code:" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 3, 107 | "metadata": { 108 | "collapsed": true, 109 | "deletable": true, 110 | "editable": true 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "def initial_cluster_centroids(X, k):\n", 115 | " return X[0:k, :]\n", 116 | "\n", 117 | "\n", 118 | "def assign_cluster(X, centroids):\n", 119 | " expanded_vectors = tf.expand_dims(X, 0)\n", 120 | " expanded_centroids = tf.expand_dims(centroids, 1)\n", 121 | " distances = tf.reduce_sum(tf.square(tf.subtract(expanded_vectors, expanded_centroids)), 2)\n", 122 | " mins = tf.argmin(distances, 0)\n", 123 | " return mins\n", 124 | "\n", 125 | "\n", 126 | "def recompute_centroids(X, Y):\n", 127 | " sums = tf.unsorted_segment_sum(X, Y, k)\n", 128 | " counts = tf.unsorted_segment_sum(tf.ones_like(X), Y, k)\n", 129 | " return sums / counts\n" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": { 135 | "deletable": true, 136 | "editable": true 137 | }, 138 | "source": [ 139 | "Given a chromogram, extract a histogram of sound frequencies as our feature vector: " 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 4, 145 | "metadata": { 146 | "collapsed": true, 147 | "deletable": true, 148 | "editable": true 149 | }, 150 | "outputs": [], 151 | "source": [ 152 | "def extract_feature_vector(sess, chromo_data):\n", 153 | " num_features, num_samples = np.shape(chromo_data)\n", 154 | " freq_vals = sess.run(max_freqs, feed_dict={chromo: chromo_data})\n", 155 | " hist, bins = np.histogram(freq_vals, bins=range(num_features + 1))\n", 156 | " return hist.astype(float) / num_samples" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": { 162 | "deletable": true, 163 | "editable": true 164 | }, 165 | "source": [ 166 | "In a session, segment an audio file using k-means:" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 5, 172 | "metadata": { 173 | "collapsed": false, 174 | "deletable": true, 175 | "editable": true 176 | }, 177 | "outputs": [ 178 | { 179 | "name": "stderr", 180 | "output_type": "stream", 181 | "text": [ 182 | "/usr/local/lib/python2.7/dist-packages/bregman/features_base.py:353: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future\n", 183 | " mxnorm = P.empty(self._cqtN) # Normalization coefficients\n", 184 | "/usr/local/lib/python2.7/dist-packages/bregman/features_base.py:357: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future\n", 185 | " for i in P.arange(self._cqtN)])\n" 186 | ] 187 | }, 188 | { 189 | "name": "stdout", 190 | "output_type": "stream", 191 | "text": [ 192 | "('chromo_data', (12, 633))\n", 193 | "(12, 12)\n", 194 | "('iteration', 50)\n", 195 | "('iteration', 100)\n", 196 | "('0.0m 0.0s', 0)\n", 197 | "('0.0m 2.5s', 1)\n", 198 | "('0.0m 5.0s', 0)\n", 199 | "('0.0m 7.5s', 1)\n", 200 | "('0.0m 10.0s', 1)\n", 201 | "('0.0m 12.5s', 1)\n", 202 | "('0.0m 15.0s', 1)\n", 203 | "('0.0m 17.5s', 0)\n", 204 | "('0.0m 20.0s', 1)\n", 205 | "('0.0m 22.5s', 1)\n", 206 | "('0.0m 25.0s', 0)\n", 207 | "('0.0m 27.5s', 0)\n" 208 | ] 209 | } 210 | ], 211 | "source": [ 212 | "with tf.Session() as sess:\n", 213 | " X = get_dataset(sess, 'TalkingMachinesPodcast.wav')\n", 214 | " print(np.shape(X))\n", 215 | " centroids = initial_cluster_centroids(X, k)\n", 216 | " i, converged = 0, False\n", 217 | " # prev_Y = None\n", 218 | " while not converged and i < max_iterations:\n", 219 | " i += 1\n", 220 | " Y = assign_cluster(X, centroids)\n", 221 | " # if prev_Y == Y:\n", 222 | " # converged = True\n", 223 | " # break\n", 224 | " # prev_Y = Y\n", 225 | " centroids = sess.run(recompute_centroids(X, Y))\n", 226 | " if i % 50 == 0:\n", 227 | " print('iteration', i)\n", 228 | " segments = sess.run(Y)\n", 229 | " for i in range(len(segments)):\n", 230 | " seconds = (i * segment_size) / float(20)\n", 231 | " min, sec = divmod(seconds, 60)\n", 232 | " time_str = str(min) + 'm ' + str(sec) + 's'\n", 233 | " print(time_str, segments[i])" 234 | ] 235 | } 236 | ], 237 | "metadata": { 238 | "kernelspec": { 239 | "display_name": "Python 2", 240 | "language": "python", 241 | "name": "python2" 242 | }, 243 | "language_info": { 244 | "codemirror_mode": { 245 | "name": "ipython", 246 | "version": 2 247 | }, 248 | "file_extension": ".py", 249 | "mimetype": "text/x-python", 250 | "name": "python", 251 | "nbconvert_exporter": "python", 252 | "pygments_lexer": "ipython2", 253 | "version": "2.7.12" 254 | } 255 | }, 256 | "nbformat": 4, 257 | "nbformat_minor": 1 258 | } 259 | -------------------------------------------------------------------------------- /ch05_clustering/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 5 2 | 3 |

4 | 5 | Suppose there’s a collection of not-pirated-totally-legal mp3s on your hard drive. All your songs are crowded in one massive folder. But it might help to automatically group together similar songs and organize them into categories like “country,” “rap,” “rock,” and so on. This act of assigning an item to a group (such as an mp3 to a playlist) in an unsupervised fashion is called clustering. 6 | 7 | The previous chapter on classification assumes you’re given a training dataset of correctly labeled data. Unfortunately, we don’t always have that luxury when we collect data in the real-world. For example, suppose we would like to divide up a large amount of music into interesting playlists. How could we possibly group together songs if we don’t have direct access to their metadata? 8 | 9 | Spotify, SoundCloud, Google Music, Pandora, and many other music streaming services try to solve this problem to recommend similar songs to customers. Their approach includes a mixture of various machine learning techniques, but clustering is often at the heart of the solution. 10 | 11 | The overall idea of clustering is that two items in the same cluster are “closer” to each other than items that belong to separate clusters. That is the general definition, leaving the interpretation of “closeness” open. For example, perhaps cheetahs and leopards belong in the same cluster, whereas elephants belong to another when closeness is measured by how similar two species are in the hierarchy of biological classification (family, genus, and species). 12 | 13 | You can image there are many clustering algorithms out there. In this chapter we’ll focus on two types, namely k-means and self-organizing map. These approaches are completely unsupervised, meaning they fit a model without ground-truth examples. 14 | 15 | - **Concept 1**: Clustering 16 | - **Concept 2**: Segmentation 17 | - **Concept 3**: Self-organizing map 18 | 19 | --- 20 | 21 | * Listing 1-4: `audio_clustering.py` 22 | * Listing 5-6: `audio_segmentation.py` 23 | * Listing 7-12: `som.py` 24 | -------------------------------------------------------------------------------- /ch05_clustering/TalkingMachinesPodcast.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/TalkingMachinesPodcast.wav -------------------------------------------------------------------------------- /ch05_clustering/audio_clustering.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | from bregman.suite import * 4 | 5 | k = 2 6 | max_iterations = 100 7 | 8 | filenames = tf.train.match_filenames_once('./audio_dataset/*.wav') 9 | count_num_files = tf.size(filenames) 10 | filename_queue = tf.train.string_input_producer(filenames) 11 | reader = tf.WholeFileReader() 12 | filename, file_contents = reader.read(filename_queue) 13 | 14 | chromo = tf.placeholder(tf.float32) 15 | max_freqs = tf.argmax(chromo, 0) 16 | 17 | 18 | def get_next_chromogram(sess): 19 | audio_file = sess.run(filename) 20 | F = Chromagram(audio_file, nfft=16384, wfft=8192, nhop=2205) 21 | return F.X, audio_file 22 | 23 | 24 | def extract_feature_vector(sess, chromo_data): 25 | num_features, num_samples = np.shape(chromo_data) 26 | freq_vals = sess.run(max_freqs, feed_dict={chromo: chromo_data}) 27 | hist, bins = np.histogram(freq_vals, bins=range(num_features + 1)) 28 | normalized_hist = hist.astype(float) / num_samples 29 | return normalized_hist 30 | 31 | 32 | def get_dataset(sess): 33 | num_files = sess.run(count_num_files) 34 | coord = tf.train.Coordinator() 35 | threads = tf.train.start_queue_runners(coord=coord) 36 | xs = list() 37 | names = list() 38 | plt.figure() 39 | for _ in range(num_files): 40 | chromo_data, filename = get_next_chromogram(sess) 41 | 42 | plt.subplot(1, 2, 1) 43 | plt.imshow(chromo_data, cmap='Greys', interpolation='nearest') 44 | plt.title('Visualization of Sound Spectrum') 45 | 46 | plt.subplot(1, 2, 2) 47 | freq_vals = sess.run(max_freqs, feed_dict={chromo: chromo_data}) 48 | plt.hist(freq_vals) 49 | plt.title('Histogram of Notes') 50 | plt.xlabel('Musical Note') 51 | plt.ylabel('Count') 52 | plt.savefig('{}.png'.format(filename)) 53 | plt.clf() 54 | 55 | plt.clf() 56 | names.append(filename) 57 | x = extract_feature_vector(sess, chromo_data) 58 | xs.append(x) 59 | xs = np.asmatrix(xs) 60 | return xs, names 61 | 62 | 63 | def initial_cluster_centroids(X, k): 64 | return X[0:k, :] 65 | 66 | 67 | def assign_cluster(X, centroids): 68 | expanded_vectors = tf.expand_dims(X, 0) 69 | expanded_centroids = tf.expand_dims(centroids, 1) 70 | distances = tf.reduce_sum(tf.square(tf.sub(expanded_vectors, expanded_centroids)), 2) 71 | mins = tf.argmin(distances, 0) 72 | return mins 73 | 74 | 75 | def recompute_centroids(X, Y): 76 | sums = tf.unsorted_segment_sum(X, Y, k) 77 | counts = tf.unsorted_segment_sum(tf.ones_like(X), Y, k) 78 | return sums / counts 79 | 80 | 81 | with tf.Session() as sess: 82 | sess.run(tf.initialize_all_variables()) 83 | X, names = get_dataset(sess) 84 | centroids = initial_cluster_centroids(X, k) 85 | i, converged = 0, False 86 | while not converged and i < max_iterations: 87 | i += 1 88 | Y = assign_cluster(X, centroids) 89 | centroids = sess.run(recompute_centroids(X, Y)) 90 | print(zip(sess.run(Y), names)) 91 | -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/cough_1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/cough_1.wav -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/cough_1.wav.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/cough_1.wav.png -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/cough_2.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/cough_2.wav -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/cough_2.wav.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/cough_2.wav.png -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/scream_1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/scream_1.wav -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/scream_1.wav.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/scream_1.wav.png -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/scream_2.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/scream_2.wav -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/scream_2.wav.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/scream_2.wav.png -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/scream_3.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/scream_3.wav -------------------------------------------------------------------------------- /ch05_clustering/audio_dataset/scream_3.wav.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch05_clustering/audio_dataset/scream_3.wav.png -------------------------------------------------------------------------------- /ch05_clustering/audio_segmentation.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | from bregman.suite import * 4 | 5 | k = 4 6 | segment_size = 50 # out of 24,526 7 | max_iterations = 100 8 | 9 | 10 | chromo = tf.placeholder(tf.float32) 11 | max_freqs = tf.argmax(chromo, 0) 12 | 13 | def get_chromogram(audio_file): 14 | F = Chromagram(audio_file, nfft=16384, wfft=8192, nhop=2205) 15 | return F.X 16 | 17 | def get_dataset(sess, audio_file): 18 | chromo_data = get_chromogram(audio_file) 19 | print('chromo_data', np.shape(chromo_data)) 20 | chromo_length = np.shape(chromo_data)[1] 21 | xs = [] 22 | for i in range(chromo_length/segment_size): 23 | chromo_segment = chromo_data[:, i*segment_size:(i+1)*segment_size] 24 | x = extract_feature_vector(sess, chromo_segment) 25 | if len(xs) == 0: 26 | xs = x 27 | else: 28 | xs = np.vstack((xs, x)) 29 | return xs 30 | 31 | 32 | def initial_cluster_centroids(X, k): 33 | return X[0:k, :] 34 | 35 | 36 | # op 37 | def assign_cluster(X, centroids): 38 | expanded_vectors = tf.expand_dims(X, 0) 39 | expanded_centroids = tf.expand_dims(centroids, 1) 40 | distances = tf.reduce_sum(tf.square(tf.sub(expanded_vectors, expanded_centroids)), 2) 41 | mins = tf.argmin(distances, 0) 42 | return mins 43 | 44 | 45 | # op 46 | def recompute_centroids(X, Y): 47 | sums = tf.unsorted_segment_sum(X, Y, k) 48 | counts = tf.unsorted_segment_sum(tf.ones_like(X), Y, k) 49 | return sums / counts 50 | 51 | 52 | def extract_feature_vector(sess, chromo_data): 53 | num_features, num_samples = np.shape(chromo_data) 54 | freq_vals = sess.run(max_freqs, feed_dict={chromo: chromo_data}) 55 | hist, bins = np.histogram(freq_vals, bins=range(num_features + 1)) 56 | return hist.astype(float) / num_samples 57 | 58 | 59 | with tf.Session() as sess: 60 | X = get_dataset(sess, 'sysk.wav') 61 | print(np.shape(X)) 62 | centroids = initial_cluster_centroids(X, k) 63 | i, converged = 0, False 64 | # prev_Y = None 65 | while not converged and i < max_iterations: 66 | i += 1 67 | Y = assign_cluster(X, centroids) 68 | # if prev_Y == Y: 69 | # converged = True 70 | # break 71 | # prev_Y = Y 72 | centroids = sess.run(recompute_centroids(X, Y)) 73 | if i % 50 == 0: 74 | print('iteration', i) 75 | segments = sess.run(Y) 76 | for i in range(len(segments)): 77 | seconds = (i * segment_size) / float(10) 78 | min, sec = divmod(seconds, 60) 79 | time_str = str(min) + 'm ' + str(sec) + 's' 80 | print(time_str, segments[i]) 81 | 82 | -------------------------------------------------------------------------------- /ch05_clustering/som.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | class SOM: 5 | def __init__(self, width, height, dim): 6 | self.num_iters = 100 7 | self.width = width 8 | self.height = height 9 | self.dim = dim 10 | self.node_locs = self.get_locs() 11 | 12 | # Each node is a vector of dimension `dim` 13 | # For a 2D grid, there are `width * height` nodes 14 | nodes = tf.Variable(tf.random_normal([width*height, dim])) 15 | self.nodes = nodes 16 | 17 | # These two ops are inputs at each iteration 18 | x = tf.placeholder(tf.float32, [dim]) 19 | iter = tf.placeholder(tf.float32) 20 | 21 | self.x = x 22 | self.iter = iter 23 | 24 | # Find the node that matches closest to the input 25 | bmu_loc = self.get_bmu_loc(x) 26 | 27 | self.propagate_nodes = self.get_propagation(bmu_loc, x, iter) 28 | 29 | def get_propagation(self, bmu_loc, x, iter): 30 | num_nodes = self.width * self.height 31 | rate = 1.0 - tf.div(iter, self.num_iters) 32 | alpha = rate * 0.5 33 | sigma = rate * tf.to_float(tf.maximum(self.width, self.height)) / 2. 34 | expanded_bmu_loc = tf.expand_dims(tf.to_float(bmu_loc), 0) 35 | sqr_dists_from_bmu = tf.reduce_sum(tf.square(tf.sub(expanded_bmu_loc, self.node_locs)), 1) 36 | neigh_factor = tf.exp(-tf.div(sqr_dists_from_bmu, 2 * tf.square(sigma))) 37 | rate = tf.mul(alpha, neigh_factor) 38 | rate_factor = tf.pack([tf.tile(tf.slice(rate, [i], [1]), [self.dim]) for i in range(num_nodes)]) 39 | nodes_diff = tf.mul(rate_factor, tf.sub(tf.pack([x for i in range(num_nodes)]), self.nodes)) 40 | update_nodes = tf.add(self.nodes, nodes_diff) 41 | return tf.assign(self.nodes, update_nodes) 42 | 43 | def get_bmu_loc(self, x): 44 | expanded_x = tf.expand_dims(x, 0) 45 | sqr_diff = tf.square(tf.sub(expanded_x, self.nodes)) 46 | dists = tf.reduce_sum(sqr_diff, 1) 47 | bmu_idx = tf.argmin(dists, 0) 48 | bmu_loc = tf.pack([tf.mod(bmu_idx, self.width), tf.div(bmu_idx, self.width)]) 49 | return bmu_loc 50 | 51 | def get_locs(self): 52 | locs = [[x, y] 53 | for y in range(self.height) 54 | for x in range(self.width)] 55 | return tf.to_float(locs) 56 | 57 | def train(self, data): 58 | with tf.Session() as sess: 59 | sess.run(tf.initialize_all_variables()) 60 | for i in range(self.num_iters): 61 | for data_x in data: 62 | sess.run(self.propagate_nodes, feed_dict={self.x: data_x, self.iter: i}) 63 | centroid_grid = [[] for i in range(self.width)] 64 | self.nodes_val = list(sess.run(self.nodes)) 65 | self.locs_val = list(sess.run(self.node_locs)) 66 | for i, l in enumerate(self.locs_val): 67 | centroid_grid[int(l[0])].append(self.nodes_val[i]) 68 | self.centroid_grid = centroid_grid -------------------------------------------------------------------------------- /ch05_clustering/som_test.py: -------------------------------------------------------------------------------- 1 | #For plotting the images 2 | from matplotlib import pyplot as plt 3 | import numpy as np 4 | from som import SOM 5 | 6 | colors = np.array( 7 | [[0., 0., 1.], 8 | [0., 0., 0.95], 9 | [0., 0.05, 1.], 10 | [0., 1., 0.], 11 | [0., 0.95, 0.], 12 | [0., 1, 0.05], 13 | [1., 0., 0.], 14 | [1., 0.05, 0.], 15 | [1., 0., 0.05], 16 | [1., 1., 0.]]) 17 | 18 | som = SOM(4, 4, 3) 19 | som.train(colors) 20 | 21 | plt.imshow(som.centroid_grid) 22 | plt.show() 23 | -------------------------------------------------------------------------------- /ch06_hmm/Concept01_forward.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `06`: Concept `01`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Hidden Markov model forward algorithm" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Oof this code's a bit complicated if you don't already know how HMMs work. Please see the book chapter for step-by-step explanations. I'll try to improve the documentation, or feel free to send a pull request with your own documentation!\n", 31 | "\n", 32 | "First, let's import TensorFlow and NumPy:" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 1, 38 | "metadata": { 39 | "collapsed": true, 40 | "deletable": true, 41 | "editable": true 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "import numpy as np\n", 46 | "import tensorflow as tf" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": { 52 | "deletable": true, 53 | "editable": true 54 | }, 55 | "source": [ 56 | "Define the HMM model:" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 2, 62 | "metadata": { 63 | "collapsed": true, 64 | "deletable": true, 65 | "editable": true 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "class HMM(object):\n", 70 | " def __init__(self, initial_prob, trans_prob, obs_prob):\n", 71 | " self.N = np.size(initial_prob)\n", 72 | " self.initial_prob = initial_prob\n", 73 | " self.trans_prob = trans_prob\n", 74 | " self.emission = tf.constant(obs_prob)\n", 75 | "\n", 76 | " assert self.initial_prob.shape == (self.N, 1)\n", 77 | " assert self.trans_prob.shape == (self.N, self.N)\n", 78 | " assert obs_prob.shape[0] == self.N\n", 79 | "\n", 80 | " self.obs_idx = tf.placeholder(tf.int32)\n", 81 | " self.fwd = tf.placeholder(tf.float64)\n", 82 | "\n", 83 | " def get_emission(self, obs_idx):\n", 84 | " slice_location = [0, obs_idx]\n", 85 | " num_rows = tf.shape(self.emission)[0]\n", 86 | " slice_shape = [num_rows, 1]\n", 87 | " return tf.slice(self.emission, slice_location, slice_shape)\n", 88 | "\n", 89 | " def forward_init_op(self):\n", 90 | " obs_prob = self.get_emission(self.obs_idx)\n", 91 | " fwd = tf.multiply(self.initial_prob, obs_prob)\n", 92 | " return fwd\n", 93 | "\n", 94 | " def forward_op(self):\n", 95 | " transitions = tf.matmul(self.fwd, tf.transpose(self.get_emission(self.obs_idx)))\n", 96 | " weighted_transitions = transitions * self.trans_prob\n", 97 | " fwd = tf.reduce_sum(weighted_transitions, 0)\n", 98 | " return tf.reshape(fwd, tf.shape(self.fwd))" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": { 104 | "deletable": true, 105 | "editable": true 106 | }, 107 | "source": [ 108 | "Define the forward algorithm:" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 3, 114 | "metadata": { 115 | "collapsed": true, 116 | "deletable": true, 117 | "editable": true 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "def forward_algorithm(sess, hmm, observations):\n", 122 | " fwd = sess.run(hmm.forward_init_op(), feed_dict={hmm.obs_idx: observations[0]})\n", 123 | " for t in range(1, len(observations)):\n", 124 | " fwd = sess.run(hmm.forward_op(), feed_dict={hmm.obs_idx: observations[t], hmm.fwd: fwd})\n", 125 | " prob = sess.run(tf.reduce_sum(fwd))\n", 126 | " return prob" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": { 132 | "deletable": true, 133 | "editable": true 134 | }, 135 | "source": [ 136 | "Let's try it out:" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 4, 142 | "metadata": { 143 | "collapsed": false, 144 | "deletable": true, 145 | "editable": true 146 | }, 147 | "outputs": [ 148 | { 149 | "name": "stdout", 150 | "output_type": "stream", 151 | "text": [ 152 | "Probability of observing [0, 1, 1, 2, 1] is 0.004540300799999999\n" 153 | ] 154 | } 155 | ], 156 | "source": [ 157 | "if __name__ == '__main__':\n", 158 | " initial_prob = np.array([[0.6], [0.4]])\n", 159 | " trans_prob = np.array([[0.7, 0.3], [0.4, 0.6]])\n", 160 | " obs_prob = np.array([[0.1, 0.4, 0.5], [0.6, 0.3, 0.1]])\n", 161 | "\n", 162 | " hmm = HMM(initial_prob=initial_prob, trans_prob=trans_prob, obs_prob=obs_prob)\n", 163 | "\n", 164 | " observations = [0, 1, 1, 2, 1]\n", 165 | " with tf.Session() as sess:\n", 166 | " prob = forward_algorithm(sess, hmm, observations)\n", 167 | " print('Probability of observing {} is {}'.format(observations, prob))" 168 | ] 169 | } 170 | ], 171 | "metadata": { 172 | "kernelspec": { 173 | "display_name": "Python 3", 174 | "language": "python", 175 | "name": "python3" 176 | }, 177 | "language_info": { 178 | "codemirror_mode": { 179 | "name": "ipython", 180 | "version": 3 181 | }, 182 | "file_extension": ".py", 183 | "mimetype": "text/x-python", 184 | "name": "python", 185 | "nbconvert_exporter": "python", 186 | "pygments_lexer": "ipython3", 187 | "version": "3.5.2" 188 | } 189 | }, 190 | "nbformat": 4, 191 | "nbformat_minor": 1 192 | } 193 | -------------------------------------------------------------------------------- /ch06_hmm/Concept02_hmm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Ch `06`: Concept `02`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Viterbi parse of a Hidden Markov model" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "Import TensorFlow and Numpy" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import numpy as np\n", 44 | "import tensorflow as tf" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "deletable": true, 51 | "editable": true 52 | }, 53 | "source": [ 54 | "Create the same HMM model as before. This time, we'll include a couple additional functions." 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "metadata": { 61 | "collapsed": true, 62 | "deletable": true, 63 | "editable": true 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "# initial parameters can be learned on training data\n", 68 | "# theory reference https://web.stanford.edu/~jurafsky/slp3/8.pdf\n", 69 | "# code reference https://phvu.net/2013/12/06/sweet-implementation-of-viterbi-in-python/\n", 70 | "class HMM(object):\n", 71 | " def __init__(self, initial_prob, trans_prob, obs_prob):\n", 72 | " self.N = np.size(initial_prob)\n", 73 | " self.initial_prob = initial_prob\n", 74 | " self.trans_prob = trans_prob\n", 75 | " self.obs_prob = obs_prob\n", 76 | " self.emission = tf.constant(obs_prob)\n", 77 | " assert self.initial_prob.shape == (self.N, 1)\n", 78 | " assert self.trans_prob.shape == (self.N, self.N)\n", 79 | " assert self.obs_prob.shape[0] == self.N\n", 80 | " self.obs = tf.placeholder(tf.int32)\n", 81 | " self.fwd = tf.placeholder(tf.float64)\n", 82 | " self.viterbi = tf.placeholder(tf.float64)\n", 83 | "\n", 84 | " def get_emission(self, obs_idx):\n", 85 | " slice_location = [0, obs_idx]\n", 86 | " num_rows = tf.shape(self.emission)[0]\n", 87 | " slice_shape = [num_rows, 1]\n", 88 | " return tf.slice(self.emission, slice_location, slice_shape)\n", 89 | "\n", 90 | " def forward_init_op(self):\n", 91 | " obs_prob = self.get_emission(self.obs)\n", 92 | " fwd = tf.multiply(self.initial_prob, obs_prob)\n", 93 | " return fwd\n", 94 | "\n", 95 | " def forward_op(self):\n", 96 | " transitions = tf.matmul(self.fwd, tf.transpose(self.get_emission(self.obs)))\n", 97 | " weighted_transitions = transitions * self.trans_prob\n", 98 | " fwd = tf.reduce_sum(weighted_transitions, 0)\n", 99 | " return tf.reshape(fwd, tf.shape(self.fwd))\n", 100 | "\n", 101 | " def decode_op(self):\n", 102 | " transitions = tf.matmul(self.viterbi, tf.transpose(self.get_emission(self.obs)))\n", 103 | " weighted_transitions = transitions * self.trans_prob\n", 104 | " viterbi = tf.reduce_max(weighted_transitions, 0)\n", 105 | " return tf.reshape(viterbi, tf.shape(self.viterbi))\n", 106 | "\n", 107 | " def backpt_op(self):\n", 108 | " back_transitions = tf.matmul(self.viterbi, np.ones((1, self.N)))\n", 109 | " weighted_back_transitions = back_transitions * self.trans_prob\n", 110 | " return tf.argmax(weighted_back_transitions, 0)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": { 116 | "deletable": true, 117 | "editable": true 118 | }, 119 | "source": [ 120 | "Define the forward algorithm from Concept01." 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 3, 126 | "metadata": { 127 | "collapsed": true, 128 | "deletable": true, 129 | "editable": true 130 | }, 131 | "outputs": [], 132 | "source": [ 133 | "def forward_algorithm(sess, hmm, observations):\n", 134 | " fwd = sess.run(hmm.forward_init_op(), feed_dict={hmm.obs: observations[0]})\n", 135 | " for t in range(1, len(observations)):\n", 136 | " fwd = sess.run(hmm.forward_op(), feed_dict={hmm.obs: observations[t], hmm.fwd: fwd})\n", 137 | " prob = sess.run(tf.reduce_sum(fwd))\n", 138 | " return prob" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": { 144 | "deletable": true, 145 | "editable": true 146 | }, 147 | "source": [ 148 | "Now, let's compute the Viterbi likelihood of the observed sequence:" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 4, 154 | "metadata": { 155 | "collapsed": true, 156 | "deletable": true, 157 | "editable": true 158 | }, 159 | "outputs": [], 160 | "source": [ 161 | "def viterbi_decode(sess, hmm, observations):\n", 162 | " viterbi = sess.run(hmm.forward_init_op(), feed_dict={hmm.obs: observations[0]})\n", 163 | " backpts = np.ones((hmm.N, len(observations)), 'int32') * -1\n", 164 | " for t in range(1, len(observations)):\n", 165 | " viterbi, backpt = sess.run([hmm.decode_op(), hmm.backpt_op()],\n", 166 | " feed_dict={hmm.obs: observations[t],\n", 167 | " hmm.viterbi: viterbi})\n", 168 | " backpts[:, t] = backpt\n", 169 | " tokens = [viterbi[:, -1].argmax()]\n", 170 | " for i in range(len(observations) - 1, 0, -1):\n", 171 | " tokens.append(backpts[tokens[-1], i])\n", 172 | " return tokens[::-1]" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": { 178 | "deletable": true, 179 | "editable": true 180 | }, 181 | "source": [ 182 | "Let's try it out on some example data:" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 5, 188 | "metadata": { 189 | "collapsed": false, 190 | "deletable": true, 191 | "editable": true 192 | }, 193 | "outputs": [ 194 | { 195 | "name": "stdout", 196 | "output_type": "stream", 197 | "text": [ 198 | "Probability of observing [0, 1, 1, 2, 1] is 0.0046421488\n", 199 | "Most likely hidden states are [0, 0, 0, 1, 1]\n" 200 | ] 201 | } 202 | ], 203 | "source": [ 204 | "if __name__ == '__main__':\n", 205 | " states = ('Healthy', 'Fever')\n", 206 | "# observations = ('normal', 'cold', 'dizzy')\n", 207 | "# start_probability = {'Healthy': 0.6, 'Fever': 0.4}\n", 208 | "# transition_probability = {\n", 209 | "# 'Healthy': {'Healthy': 0.7, 'Fever': 0.3},\n", 210 | "# 'Fever': {'Healthy': 0.4, 'Fever': 0.6}\n", 211 | "# }\n", 212 | "# emission_probability = {\n", 213 | "# 'Healthy': {'normal': 0.5, 'cold': 0.4, 'dizzy': 0.1},\n", 214 | "# 'Fever': {'normal': 0.1, 'cold': 0.3, 'dizzy': 0.6}\n", 215 | "# }\n", 216 | " initial_prob = np.array([[0.6], [0.4]])\n", 217 | " trans_prob = np.array([[0.7, 0.3], [0.4, 0.6]])\n", 218 | " obs_prob = np.array([[0.5, 0.4, 0.1], [0.1, 0.3, 0.6]])\n", 219 | " hmm = HMM(initial_prob=initial_prob, trans_prob=trans_prob, obs_prob=obs_prob)\n", 220 | "\n", 221 | " observations = [0, 1, 1, 2, 1]\n", 222 | " with tf.Session() as sess:\n", 223 | " prob = forward_algorithm(sess, hmm, observations)\n", 224 | " print('Probability of observing {} is {}'.format(observations, prob))\n", 225 | "\n", 226 | " seq = viterbi_decode(sess, hmm, observations)\n", 227 | " print('Most likely hidden states are {}'.format(seq))" 228 | ] 229 | } 230 | ], 231 | "metadata": { 232 | "kernelspec": { 233 | "display_name": "Python 3", 234 | "language": "python", 235 | "name": "python3" 236 | }, 237 | "language_info": { 238 | "codemirror_mode": { 239 | "name": "ipython", 240 | "version": 3 241 | }, 242 | "file_extension": ".py", 243 | "mimetype": "text/x-python", 244 | "name": "python", 245 | "nbconvert_exporter": "python", 246 | "pygments_lexer": "ipython3", 247 | "version": "3.5.2" 248 | } 249 | }, 250 | "nbformat": 4, 251 | "nbformat_minor": 1 252 | } 253 | -------------------------------------------------------------------------------- /ch06_hmm/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 6 2 | 3 |

4 | 5 | If a rocket blows up, someone’s probably getting fired, so rocket scientists and engineers must be able to make confident decisions about all components and configurations. They do so by physical simulations and mathematical deduction from first principles. You, too, have solved science problems with pure logical thinking. Consider Boyle’s law: pressure and volume of a gas are inversely related under a fixed temperature. You can make insightful inferences from these simple laws that have been discovered about the world. Recently, machine learning has started to play the role of an important side-kick to deductive reasoning. 6 | 7 | “Rocket science” and “machine learning” aren’t phrases that usually appear together. But nowadays, modeling real-world sensor readings using intelligent data-driven algorithms is more approachable in the aerospace industry. Also, the use of machine learning techniques is flourishing in the healthcare and automotive industries. But why? 8 | 9 | Part of the reason for this influx can be attributed to better understanding of interpretable models, which are machine learning models where the learned parameters have clear interpretations. If a rocket blows up, for example, an interpretable model might help trace the root cause. 10 | 11 | This chapter is about exposing the hidden explanations behind observations. Consider a puppet-master pulling strings to make a puppet appear alive. Analyzing only the motions of the puppet might lead to over-complicated conclusions about how it’s possible for an inanimate object to move. Once you notice the attached strings, you’ll realize that a puppet-master is the best explanation for the life-like motions. 12 | 13 | - **Concept 1**: Forward algorithm 14 | - **Concept 2**: Viterbi decode 15 | 16 | -- 17 | 18 | * Listing 1-6: `forward.py` 19 | * Listing 7-11: `hmm.py` 20 | -------------------------------------------------------------------------------- /ch06_hmm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch06_hmm/__init__.py -------------------------------------------------------------------------------- /ch06_hmm/forward.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | class HMM(object): 6 | def __init__(self, initial_prob, trans_prob, obs_prob): 7 | self.N = np.size(initial_prob) 8 | self.initial_prob = initial_prob 9 | self.trans_prob = trans_prob 10 | self.emission = tf.constant(obs_prob) 11 | 12 | assert self.initial_prob.shape == (self.N, 1) 13 | assert self.trans_prob.shape == (self.N, self.N) 14 | assert obs_prob.shape[0] == self.N 15 | 16 | self.obs_idx = tf.placeholder(tf.int32) 17 | self.fwd = tf.placeholder(tf.float64) 18 | 19 | def get_emission(self, obs_idx): 20 | slice_location = [0, obs_idx] 21 | num_rows = tf.shape(self.emission)[0] 22 | slice_shape = [num_rows, 1] 23 | return tf.slice(self.emission, slice_location, slice_shape) 24 | 25 | def forward_init_op(self): 26 | obs_prob = self.get_emission(self.obs_idx) 27 | fwd = tf.mul(self.initial_prob, obs_prob) 28 | return fwd 29 | 30 | def forward_op(self): 31 | transitions = tf.matmul(self.fwd, tf.transpose(self.get_emission(self.obs_idx))) 32 | weighted_transitions = transitions * self.trans_prob 33 | fwd = tf.reduce_sum(weighted_transitions, 0) 34 | return tf.reshape(fwd, tf.shape(self.fwd)) 35 | 36 | 37 | def forward_algorithm(sess, hmm, observations): 38 | fwd = sess.run(hmm.forward_init_op(), feed_dict={hmm.obs_idx: observations[0]}) 39 | for t in range(1, len(observations)): 40 | fwd = sess.run(hmm.forward_op(), feed_dict={hmm.obs_idx: observations[t], hmm.fwd: fwd}) 41 | prob = sess.run(tf.reduce_sum(fwd)) 42 | return prob 43 | 44 | if __name__ == '__main__': 45 | initial_prob = np.array([[0.6], [0.4]]) 46 | trans_prob = np.array([[0.7, 0.3], [0.4, 0.6]]) 47 | obs_prob = np.array([[0.5, 0.4, 0.1], [0.1, 0.3, 0.6]]) 48 | 49 | hmm = HMM(initial_prob=initial_prob, trans_prob=trans_prob, obs_prob=obs_prob) 50 | 51 | observations = [0, 1, 1, 2, 1] 52 | with tf.Session() as sess: 53 | prob = forward_algorithm(sess, hmm, observations) 54 | print('Probability of observing {} is {}'.format(observations, prob)) 55 | -------------------------------------------------------------------------------- /ch06_hmm/hmm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | # initial parameters can be learned on training data 6 | # theory reference https://web.stanford.edu/~jurafsky/slp3/8.pdf 7 | # code reference https://phvu.net/2013/12/06/sweet-implementation-of-viterbi-in-python/ 8 | class HMM(object): 9 | def __init__(self, initial_prob, trans_prob, obs_prob): 10 | self.N = np.size(initial_prob) 11 | self.initial_prob = initial_prob 12 | self.trans_prob = trans_prob 13 | self.obs_prob = obs_prob 14 | self.emission = tf.constant(obs_prob) 15 | assert self.initial_prob.shape == (self.N, 1) 16 | assert self.trans_prob.shape == (self.N, self.N) 17 | assert self.obs_prob.shape[0] == self.N 18 | self.obs = tf.placeholder(tf.int32) 19 | self.fwd = tf.placeholder(tf.float64) 20 | self.viterbi = tf.placeholder(tf.float64) 21 | 22 | def get_emission(self, obs_idx): 23 | slice_location = [0, obs_idx] 24 | num_rows = tf.shape(self.emission)[0] 25 | slice_shape = [num_rows, 1] 26 | return tf.slice(self.emission, slice_location, slice_shape) 27 | 28 | def forward_init_op(self): 29 | obs_prob = self.get_emission(self.obs) 30 | fwd = tf.mul(self.initial_prob, obs_prob) 31 | return fwd 32 | 33 | def forward_op(self): 34 | transitions = tf.matmul(self.fwd, tf.transpose(self.get_emission(self.obs))) 35 | weighted_transitions = transitions * self.trans_prob 36 | fwd = tf.reduce_sum(weighted_transitions, 0) 37 | return tf.reshape(fwd, tf.shape(self.fwd)) 38 | 39 | def decode_op(self): 40 | transitions = tf.matmul(self.viterbi, tf.transpose(self.get_emission(self.obs))) 41 | weighted_transitions = transitions * self.trans_prob 42 | viterbi = tf.reduce_max(weighted_transitions, 0) 43 | return tf.reshape(viterbi, tf.shape(self.viterbi)) 44 | 45 | def backpt_op(self): 46 | back_transitions = tf.matmul(self.viterbi, np.ones((1, self.N))) 47 | weighted_back_transitions = back_transitions * self.trans_prob 48 | return tf.argmax(weighted_back_transitions, 0) 49 | 50 | 51 | def forward_algorithm(sess, hmm, observations): 52 | fwd = sess.run(hmm.forward_init_op(), feed_dict={hmm.obs: observations[0]}) 53 | for t in range(1, len(observations)): 54 | fwd = sess.run(hmm.forward_op(), feed_dict={hmm.obs: observations[t], hmm.fwd: fwd}) 55 | prob = sess.run(tf.reduce_sum(fwd)) 56 | return prob 57 | 58 | def viterbi_decode(sess, hmm, observations): 59 | viterbi = sess.run(hmm.forward_init_op(), feed_dict={hmm.obs: observations[0]}) 60 | backpts = np.ones((hmm.N, len(observations)), 'int32') * -1 61 | for t in range(1, len(observations)): 62 | viterbi, backpt = sess.run([hmm.decode_op(), hmm.backpt_op()], 63 | feed_dict={hmm.obs: observations[t], 64 | hmm.viterbi: viterbi}) 65 | backpts[:, t] = backpt 66 | tokens = [viterbi[:, -1].argmax()] 67 | for i in range(len(observations) - 1, 0, -1): 68 | tokens.append(backpts[tokens[-1], i]) 69 | return tokens[::-1] 70 | 71 | if __name__ == '__main__': 72 | states = ('Healthy', 'Fever') 73 | observations = ('normal', 'cold', 'dizzy') 74 | start_probability = {'Healthy': 0.6, 'Fever': 0.4} 75 | transition_probability = { 76 | 'Healthy': {'Healthy': 0.7, 'Fever': 0.3}, 77 | 'Fever': {'Healthy': 0.4, 'Fever': 0.6} 78 | } 79 | emission_probability = { 80 | 'Healthy': {'normal': 0.5, 'cold': 0.4, 'dizzy': 0.1}, 81 | 'Fever': {'normal': 0.1, 'cold': 0.3, 'dizzy': 0.6} 82 | } 83 | initial_prob = np.array([[0.6], [0.4]]) 84 | trans_prob = np.array([[0.7, 0.3], [0.4, 0.6]]) 85 | obs_prob = np.array([[0.5, 0.4, 0.1], [0.1, 0.3, 0.6]]) 86 | hmm = HMM(initial_prob=initial_prob, trans_prob=trans_prob, obs_prob=obs_prob) 87 | 88 | observations = [0, 1, 1, 2, 1] 89 | with tf.Session() as sess: 90 | prob = forward_algorithm(sess, hmm, observations) 91 | print('Probability of observing {} is {}'.format(observations, prob)) 92 | 93 | seq = viterbi_decode(sess, hmm, observations) 94 | print('Most likely hidden states are {}'.format(seq)) 95 | 96 | -------------------------------------------------------------------------------- /ch07_autoencoder/.gitignore: -------------------------------------------------------------------------------- 1 | cifar-10-batches-py -------------------------------------------------------------------------------- /ch07_autoencoder/Concept01_autoencoder.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "## Ch `07`: Concept `01`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Autoencoder" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "deletable": true, 27 | "editable": true 28 | }, 29 | "source": [ 30 | "All we'll need is TensorFlow and NumPy:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true, 38 | "deletable": true, 39 | "editable": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "import tensorflow as tf\n", 44 | "import numpy as np" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "deletable": true, 51 | "editable": true 52 | }, 53 | "source": [ 54 | "Instead of feeding all the training data to the training op, we will feed data in small batches:" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "metadata": { 61 | "collapsed": true, 62 | "deletable": true, 63 | "editable": true 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "def get_batch(X, size):\n", 68 | " a = np.random.choice(len(X), size, replace=False)\n", 69 | " return X[a]" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": { 75 | "deletable": true, 76 | "editable": true 77 | }, 78 | "source": [ 79 | "Define the autoencoder class:" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 3, 85 | "metadata": { 86 | "collapsed": true, 87 | "deletable": true, 88 | "editable": true 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "class Autoencoder:\n", 93 | " def __init__(self, input_dim, hidden_dim, epoch=500, batch_size=10, learning_rate=0.001):\n", 94 | " self.epoch = epoch\n", 95 | " self.batch_size = batch_size\n", 96 | " self.learning_rate = learning_rate\n", 97 | "\n", 98 | " # Define input placeholder\n", 99 | " x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim])\n", 100 | " \n", 101 | " # Define variables\n", 102 | " with tf.name_scope('encode'):\n", 103 | " weights = tf.Variable(tf.random_normal([input_dim, hidden_dim], dtype=tf.float32), name='weights')\n", 104 | " biases = tf.Variable(tf.zeros([hidden_dim]), name='biases')\n", 105 | " encoded = tf.nn.sigmoid(tf.matmul(x, weights) + biases)\n", 106 | " with tf.name_scope('decode'):\n", 107 | " weights = tf.Variable(tf.random_normal([hidden_dim, input_dim], dtype=tf.float32), name='weights')\n", 108 | " biases = tf.Variable(tf.zeros([input_dim]), name='biases')\n", 109 | " decoded = tf.matmul(encoded, weights) + biases\n", 110 | "\n", 111 | " self.x = x\n", 112 | " self.encoded = encoded\n", 113 | " self.decoded = decoded\n", 114 | "\n", 115 | " # Define cost function and training op\n", 116 | " self.loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(self.x, self.decoded))))\n", 117 | "\n", 118 | " self.all_loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(self.x, self.decoded)), 1))\n", 119 | " self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)\n", 120 | " \n", 121 | " # Define a saver op\n", 122 | " self.saver = tf.train.Saver()\n", 123 | "\n", 124 | " def train(self, data):\n", 125 | " with tf.Session() as sess:\n", 126 | " sess.run(tf.global_variables_initializer())\n", 127 | " for i in range(self.epoch):\n", 128 | " for j in range(500):\n", 129 | " batch_data = get_batch(data, self.batch_size)\n", 130 | " l, _ = sess.run([self.loss, self.train_op], feed_dict={self.x: batch_data})\n", 131 | " if i % 50 == 0:\n", 132 | " print('epoch {0}: loss = {1}'.format(i, l))\n", 133 | " self.saver.save(sess, './model.ckpt')\n", 134 | " self.saver.save(sess, './model.ckpt')\n", 135 | " \n", 136 | " def test(self, data):\n", 137 | " with tf.Session() as sess:\n", 138 | " self.saver.restore(sess, './model.ckpt')\n", 139 | " hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data})\n", 140 | " print('input', data)\n", 141 | " print('compressed', hidden)\n", 142 | " print('reconstructed', reconstructed)\n", 143 | " return reconstructed\n", 144 | "\n", 145 | " def get_params(self):\n", 146 | " with tf.Session() as sess:\n", 147 | " self.saver.restore(sess, './model.ckpt')\n", 148 | " weights, biases = sess.run([self.weights1, self.biases1])\n", 149 | " return weights, biases\n", 150 | "\n", 151 | " def classify(self, data, labels):\n", 152 | " with tf.Session() as sess:\n", 153 | " sess.run(tf.global_variables_initializer())\n", 154 | " self.saver.restore(sess, './model.ckpt')\n", 155 | " hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data})\n", 156 | " reconstructed = reconstructed[0]\n", 157 | " # loss = sess.run(self.all_loss, feed_dict={self.x: data})\n", 158 | " print('data', np.shape(data))\n", 159 | " print('reconstructed', np.shape(reconstructed))\n", 160 | " loss = np.sqrt(np.mean(np.square(data - reconstructed), axis=1))\n", 161 | " print('loss', np.shape(loss))\n", 162 | " horse_indices = np.where(labels == 7)[0]\n", 163 | " not_horse_indices = np.where(labels != 7)[0]\n", 164 | " horse_loss = np.mean(loss[horse_indices])\n", 165 | " not_horse_loss = np.mean(loss[not_horse_indices])\n", 166 | " print('horse', horse_loss)\n", 167 | " print('not horse', not_horse_loss)\n", 168 | " return hidden[7,:]\n", 169 | "\n", 170 | " def decode(self, encoding):\n", 171 | " with tf.Session() as sess:\n", 172 | " sess.run(tf.global_variables_initializer())\n", 173 | " self.saver.restore(sess, './model.ckpt')\n", 174 | " reconstructed = sess.run(self.decoded, feed_dict={self.encoded: encoding})\n", 175 | " img = np.reshape(reconstructed, (32, 32))\n", 176 | " return img\n" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": { 182 | "deletable": true, 183 | "editable": true 184 | }, 185 | "source": [ 186 | "The *Iris dataset* is often used as a simple training dataset to check whether a classification algorithm is working. The sklearn library comes with it, `pip install sklearn`." 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 4, 192 | "metadata": { 193 | "collapsed": false, 194 | "deletable": true, 195 | "editable": true 196 | }, 197 | "outputs": [ 198 | { 199 | "name": "stdout", 200 | "output_type": "stream", 201 | "text": [ 202 | "epoch 0: loss = 3.8637373447418213\n", 203 | "epoch 50: loss = 0.25829368829727173\n", 204 | "epoch 100: loss = 0.3230888843536377\n", 205 | "epoch 150: loss = 0.3295430839061737\n", 206 | "epoch 200: loss = 0.24636892974376678\n", 207 | "epoch 250: loss = 0.22375555336475372\n", 208 | "epoch 300: loss = 0.19688692688941956\n", 209 | "epoch 350: loss = 0.2520211935043335\n", 210 | "epoch 400: loss = 0.29669439792633057\n", 211 | "epoch 450: loss = 0.2794385552406311\n", 212 | "input [[8, 4, 6, 2]]\n", 213 | "compressed [[ 0.72223264]]\n", 214 | "reconstructed [[ 6.87640762 2.79334426 6.23228502 2.21386957]]\n" 215 | ] 216 | }, 217 | { 218 | "data": { 219 | "text/plain": [ 220 | "array([[ 6.87640762, 2.79334426, 6.23228502, 2.21386957]], dtype=float32)" 221 | ] 222 | }, 223 | "execution_count": 4, 224 | "metadata": {}, 225 | "output_type": "execute_result" 226 | } 227 | ], 228 | "source": [ 229 | "from sklearn import datasets\n", 230 | "\n", 231 | "hidden_dim = 1\n", 232 | "data = datasets.load_iris().data\n", 233 | "input_dim = len(data[0])\n", 234 | "ae = Autoencoder(input_dim, hidden_dim)\n", 235 | "ae.train(data)\n", 236 | "ae.test([[8, 4, 6, 2]])" 237 | ] 238 | } 239 | ], 240 | "metadata": { 241 | "kernelspec": { 242 | "display_name": "Python 3", 243 | "language": "python", 244 | "name": "python3" 245 | }, 246 | "language_info": { 247 | "codemirror_mode": { 248 | "name": "ipython", 249 | "version": 3 250 | }, 251 | "file_extension": ".py", 252 | "mimetype": "text/x-python", 253 | "name": "python", 254 | "nbconvert_exporter": "python", 255 | "pygments_lexer": "ipython3", 256 | "version": "3.5.2" 257 | } 258 | }, 259 | "nbformat": 4, 260 | "nbformat_minor": 1 261 | } 262 | -------------------------------------------------------------------------------- /ch07_autoencoder/Concept03_denoising.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "## Ch `07`: Concept `03`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "deletable": true, 17 | "editable": true 18 | }, 19 | "source": [ 20 | "## Denoising autoencoder" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "collapsed": true, 27 | "deletable": true, 28 | "editable": true 29 | }, 30 | "source": [ 31 | "A denoising autoencoder is pretty much the same architecture as a normal autoencoder. The input is noised up, and cost function tries to denoise it by minimizing the construction error from denoised input to clean output." 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 1, 37 | "metadata": { 38 | "collapsed": true, 39 | "deletable": true, 40 | "editable": true 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "import tensorflow as tf\n", 45 | "import numpy as np\n", 46 | "import time\n", 47 | "\n", 48 | "def get_batch(X, Xn, size):\n", 49 | " a = np.random.choice(len(X), size, replace=False)\n", 50 | " return X[a], Xn[a]\n", 51 | "\n", 52 | "class Denoiser:\n", 53 | "\n", 54 | " def __init__(self, input_dim, hidden_dim, epoch=10000, batch_size=50, learning_rate=0.001):\n", 55 | " self.epoch = epoch\n", 56 | " self.batch_size = batch_size\n", 57 | " self.learning_rate = learning_rate\n", 58 | "\n", 59 | " self.x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x')\n", 60 | " self.x_noised = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x_noised')\n", 61 | " with tf.name_scope('encode'):\n", 62 | " self.weights1 = tf.Variable(tf.random_normal([input_dim, hidden_dim], dtype=tf.float32), name='weights')\n", 63 | " self.biases1 = tf.Variable(tf.zeros([hidden_dim]), name='biases')\n", 64 | " self.encoded = tf.nn.sigmoid(tf.matmul(self.x_noised, self.weights1) + self.biases1, name='encoded')\n", 65 | " with tf.name_scope('decode'):\n", 66 | " weights = tf.Variable(tf.random_normal([hidden_dim, input_dim], dtype=tf.float32), name='weights')\n", 67 | " biases = tf.Variable(tf.zeros([input_dim]), name='biases')\n", 68 | " self.decoded = tf.matmul(self.encoded, weights) + biases\n", 69 | " self.loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(self.x, self.decoded))))\n", 70 | " self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)\n", 71 | " self.saver = tf.train.Saver()\n", 72 | "\n", 73 | " def add_noise(self, data):\n", 74 | " noise_type = 'mask-0.2'\n", 75 | " if noise_type == 'gaussian':\n", 76 | " n = np.random.normal(0, 0.1, np.shape(data))\n", 77 | " return data + n\n", 78 | " if 'mask' in noise_type:\n", 79 | " frac = float(noise_type.split('-')[1])\n", 80 | " temp = np.copy(data)\n", 81 | " for i in temp:\n", 82 | " n = np.random.choice(len(i), round(frac * len(i)), replace=False)\n", 83 | " i[n] = 0\n", 84 | " return temp\n", 85 | "\n", 86 | " def train(self, data):\n", 87 | " data_noised = self.add_noise(data)\n", 88 | " with open('log.csv', 'w') as writer:\n", 89 | " with tf.Session() as sess:\n", 90 | " sess.run(tf.global_variables_initializer())\n", 91 | " for i in range(self.epoch):\n", 92 | " for j in range(50):\n", 93 | " batch_data, batch_data_noised = get_batch(data, data_noised, self.batch_size)\n", 94 | " l, _ = sess.run([self.loss, self.train_op], feed_dict={self.x: batch_data, self.x_noised: batch_data_noised})\n", 95 | " if i % 10 == 0:\n", 96 | " print('epoch {0}: loss = {1}'.format(i, l))\n", 97 | " self.saver.save(sess, './model.ckpt')\n", 98 | " epoch_time = int(time.time())\n", 99 | " row_str = str(epoch_time) + ',' + str(i) + ',' + str(l) + '\\n'\n", 100 | " writer.write(row_str)\n", 101 | " writer.flush()\n", 102 | " self.saver.save(sess, './model.ckpt')\n", 103 | "\n", 104 | " def test(self, data):\n", 105 | " with tf.Session() as sess:\n", 106 | " self.saver.restore(sess, './model.ckpt')\n", 107 | " hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data})\n", 108 | " print('input', data)\n", 109 | " print('compressed', hidden)\n", 110 | " print('reconstructed', reconstructed)\n", 111 | " return reconstructed\n", 112 | "\n", 113 | " def get_params(self):\n", 114 | " with tf.Session() as sess:\n", 115 | " self.saver.restore(sess, './model.ckpt')\n", 116 | " weights, biases = sess.run([self.weights1, self.biases1])\n", 117 | " return weights, biases" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": { 124 | "collapsed": true, 125 | "deletable": true, 126 | "editable": true 127 | }, 128 | "outputs": [], 129 | "source": [] 130 | } 131 | ], 132 | "metadata": { 133 | "kernelspec": { 134 | "display_name": "Python 3", 135 | "language": "python", 136 | "name": "python3" 137 | }, 138 | "language_info": { 139 | "codemirror_mode": { 140 | "name": "ipython", 141 | "version": 3 142 | }, 143 | "file_extension": ".py", 144 | "mimetype": "text/x-python", 145 | "name": "python", 146 | "nbconvert_exporter": "python", 147 | "pygments_lexer": "ipython3", 148 | "version": "3.5.2" 149 | } 150 | }, 151 | "nbformat": 4, 152 | "nbformat_minor": 1 153 | } 154 | -------------------------------------------------------------------------------- /ch07_autoencoder/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 7 2 | 3 |

4 | 5 | 6 | Have you ever identified a song from a person just humming a melody? It might be easy for you, but I’m comically tone-deaf when it comes to music. Humming, by itself, is an approximation of its corresponding song. An even better approximation could be singing. Include some instrumentals, and sometimes a cover of a song sounds indistinguishable from the original. 7 | 8 | Instead of songs, in this chapter, we will approximate functions. Functions are a very general notion of relations between inputs and outputs. In machine learning, we typically want to find the function that relates inputs to outputs. Finding the best possible function is difficult, but approximating the function is much easier. 9 | 10 | Conveniently, artificial neural networks are a model in machine learning that can approximate any function. Given training data, we want to build a neural network model that best approximates the implicit function that might have generated the data. 11 | 12 | After introducing neural networks in section 8.1, we’ll learn how to use them to encode data into a smaller representation in section 8.2, using a network structure called an autoencoder. 13 | 14 | - **Concept 1**: Autoencoder 15 | - **Concept 2**: Applying an autoencoder to images 16 | - **Concept 3**: Denoising autoencoder -------------------------------------------------------------------------------- /ch07_autoencoder/autoencoder.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | def get_batch(X, size): 5 | a = np.random.choice(len(X), size, replace=False) 6 | return X[a] 7 | 8 | class Autoencoder: 9 | def __init__(self, input_dim, hidden_dim, epoch=1000, batch_size=50, learning_rate=0.001): 10 | self.epoch = epoch 11 | self.batch_size = batch_size 12 | self.learning_rate = learning_rate 13 | 14 | x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim]) 15 | with tf.name_scope('encode'): 16 | weights = tf.Variable(tf.random_normal([input_dim, hidden_dim], dtype=tf.float32), name='weights') 17 | biases = tf.Variable(tf.zeros([hidden_dim]), name='biases') 18 | encoded = tf.nn.sigmoid(tf.matmul(x, weights) + biases) 19 | with tf.name_scope('decode'): 20 | weights = tf.Variable(tf.random_normal([hidden_dim, input_dim], dtype=tf.float32), name='weights') 21 | biases = tf.Variable(tf.zeros([input_dim]), name='biases') 22 | decoded = tf.matmul(encoded, weights) + biases 23 | 24 | self.x = x 25 | self.encoded = encoded 26 | self.decoded = decoded 27 | 28 | self.loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(self.x, self.decoded)))) 29 | self.train_op = tf.train.RMSPropOptimizer(self.learning_rate).minimize(self.loss) 30 | 31 | self.saver = tf.train.Saver() 32 | 33 | def train(self, data): 34 | with tf.Session() as sess: 35 | sess.run(tf.global_variables_initializer()) 36 | for i in range(self.epoch): 37 | for j in range(np.shape(data)[0] // self.batch_size): 38 | batch_data = get_batch(data, self.batch_size) 39 | l, _ = sess.run([self.loss, self.train_op], feed_dict={self.x: batch_data}) 40 | if i % 10 == 0: 41 | print('epoch {0}: loss = {1}'.format(i, l)) 42 | self.saver.save(sess, './model.ckpt') 43 | self.saver.save(sess, './model.ckpt') 44 | 45 | def test(self, data): 46 | with tf.Session() as sess: 47 | self.saver.restore(sess, './model.ckpt') 48 | hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data}) 49 | print('input', data) 50 | print('compressed', hidden) 51 | print('reconstructed', reconstructed) 52 | return reconstructed 53 | 54 | def get_params(self): 55 | with tf.Session() as sess: 56 | self.saver.restore(sess, './model.ckpt') 57 | weights, biases = sess.run([self.weights1, self.biases1]) 58 | return weights, biases 59 | 60 | def classify(self, data, labels): 61 | with tf.Session() as sess: 62 | sess.run(tf.global_variables_initializer()) 63 | self.saver.restore(sess, './model.ckpt') 64 | hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data}) 65 | reconstructed = reconstructed[0] 66 | # loss = sess.run(self.all_loss, feed_dict={self.x: data}) 67 | print('data', np.shape(data)) 68 | print('reconstructed', np.shape(reconstructed)) 69 | loss = np.sqrt(np.mean(np.square(data - reconstructed), axis=1)) 70 | print('loss', np.shape(loss)) 71 | horse_indices = np.where(labels == 7)[0] 72 | not_horse_indices = np.where(labels != 7)[0] 73 | horse_loss = np.mean(loss[horse_indices]) 74 | not_horse_loss = np.mean(loss[not_horse_indices]) 75 | print('horse', horse_loss) 76 | print('not horse', not_horse_loss) 77 | return hidden 78 | 79 | def decode(self, encoding): 80 | with tf.Session() as sess: 81 | sess.run(tf.global_variables_initializer()) 82 | self.saver.restore(sess, './model.ckpt') 83 | reconstructed = sess.run(self.decoded, feed_dict={self.encoded: encoding}) 84 | img = np.reshape(reconstructed, (32, 32)) 85 | return img 86 | -------------------------------------------------------------------------------- /ch07_autoencoder/autoencoder_batch.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | def get_batch(X, size): 5 | a = np.random.choice(len(X), size, replace=False) 6 | return X[a] 7 | 8 | class Autoencoder: 9 | def __init__(self, input_dim, hidden_dim, epoch=1000, batch_size=10, learning_rate=0.001): 10 | self.epoch = epoch 11 | self.batch_size = batch_size 12 | self.learning_rate = learning_rate 13 | 14 | x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim]) 15 | with tf.name_scope('encode'): 16 | weights = tf.Variable(tf.random_normal([input_dim, hidden_dim], dtype=tf.float32), name='weights') 17 | biases = tf.Variable(tf.zeros([hidden_dim]), name='biases') 18 | encoded = tf.nn.sigmoid(tf.matmul(x, weights) + biases) 19 | with tf.name_scope('decode'): 20 | weights = tf.Variable(tf.random_normal([hidden_dim, input_dim], dtype=tf.float32), name='weights') 21 | biases = tf.Variable(tf.zeros([input_dim]), name='biases') 22 | decoded = tf.matmul(encoded, weights) + biases 23 | 24 | self.x = x 25 | self.encoded = encoded 26 | self.decoded = decoded 27 | 28 | self.loss = tf.sqrt(tf.reduce_mean(tf.square(tf.sub(self.x, self.decoded)))) 29 | 30 | self.all_loss = tf.sqrt(tf.reduce_mean(tf.square(tf.sub(self.x, self.decoded)), 1)) 31 | self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss) 32 | self.saver = tf.train.Saver() 33 | 34 | def train(self, data): 35 | with tf.Session() as sess: 36 | sess.run(tf.initialize_all_variables()) 37 | for i in range(self.epoch): 38 | for j in range(500): 39 | batch_data = get_batch(data, self.batch_size) 40 | l, _ = sess.run([self.loss, self.train_op], feed_dict={self.x: batch_data}) 41 | if i % 10 == 0: 42 | print('epoch {0}: loss = {1}'.format(i, l)) 43 | self.saver.save(sess, './model.ckpt') 44 | self.saver.save(sess, './model.ckpt') 45 | 46 | def test(self, data): 47 | with tf.Session() as sess: 48 | self.saver.restore(sess, './model.ckpt') 49 | hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data}) 50 | print('input', data) 51 | print('compressed', hidden) 52 | print('reconstructed', reconstructed) 53 | return reconstructed 54 | 55 | def get_params(self): 56 | with tf.Session() as sess: 57 | self.saver.restore(sess, './model.ckpt') 58 | weights, biases = sess.run([self.weights1, self.biases1]) 59 | return weights, biases 60 | 61 | -------------------------------------------------------------------------------- /ch07_autoencoder/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "model.ckpt" 2 | all_model_checkpoint_paths: "model.ckpt" 3 | -------------------------------------------------------------------------------- /ch07_autoencoder/denoiser.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import time 4 | 5 | def get_batch(X, Xn, size): 6 | a = np.random.choice(len(X), size, replace=False) 7 | return X[a], Xn[a] 8 | 9 | class Denoiser: 10 | 11 | def __init__(self, input_dim, hidden_dim, epoch=10000, batch_size=50, learning_rate=0.001): 12 | self.epoch = epoch 13 | self.batch_size = batch_size 14 | self.learning_rate = learning_rate 15 | 16 | self.x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x') 17 | self.x_noised = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='x_noised') 18 | with tf.name_scope('encode'): 19 | self.weights1 = tf.Variable(tf.random_normal([input_dim, hidden_dim], dtype=tf.float32), name='weights') 20 | self.biases1 = tf.Variable(tf.zeros([hidden_dim]), name='biases') 21 | self.encoded = tf.nn.sigmoid(tf.matmul(self.x_noised, self.weights1) + self.biases1, name='encoded') 22 | with tf.name_scope('decode'): 23 | weights = tf.Variable(tf.random_normal([hidden_dim, input_dim], dtype=tf.float32), name='weights') 24 | biases = tf.Variable(tf.zeros([input_dim]), name='biases') 25 | self.decoded = tf.matmul(self.encoded, weights) + biases 26 | self.loss = tf.sqrt(tf.reduce_mean(tf.square(tf.sub(self.x, self.decoded)))) 27 | self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss) 28 | self.saver = tf.train.Saver() 29 | 30 | def add_noise(self, data): 31 | noise_type = 'mask-0.2' 32 | if noise_type == 'gaussian': 33 | n = np.random.normal(0, 0.1, np.shape(data)) 34 | return data + n 35 | if 'mask' in noise_type: 36 | frac = float(noise_type.split('-')[1]) 37 | temp = np.copy(data) 38 | for i in temp: 39 | n = np.random.choice(len(i), round(frac * len(i)), replace=False) 40 | i[n] = 0 41 | return temp 42 | 43 | def train(self, data): 44 | data_noised = self.add_noise(data) 45 | with open('log.csv', 'w') as writer: 46 | with tf.Session() as sess: 47 | sess.run(tf.initialize_all_variables()) 48 | for i in range(self.epoch): 49 | for j in range(50): 50 | batch_data, batch_data_noised = get_batch(data, data_noised, self.batch_size) 51 | l, _ = sess.run([self.loss, self.train_op], feed_dict={self.x: batch_data, self.x_noised: batch_data_noised}) 52 | if i % 10 == 0: 53 | print('epoch {0}: loss = {1}'.format(i, l)) 54 | self.saver.save(sess, './model.ckpt') 55 | epoch_time = int(time.time()) 56 | row_str = str(epoch_time) + ',' + str(i) + ',' + str(l) + '\n' 57 | writer.write(row_str) 58 | writer.flush() 59 | self.saver.save(sess, './model.ckpt') 60 | 61 | def test(self, data): 62 | with tf.Session() as sess: 63 | self.saver.restore(sess, './model.ckpt') 64 | hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data}) 65 | print('input', data) 66 | print('compressed', hidden) 67 | print('reconstructed', reconstructed) 68 | return reconstructed 69 | 70 | def get_params(self): 71 | with tf.Session() as sess: 72 | self.saver.restore(sess, './model.ckpt') 73 | weights, biases = sess.run([self.weights1, self.biases1]) 74 | return weights, biases 75 | -------------------------------------------------------------------------------- /ch07_autoencoder/denoising_autoencoder.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | def get_batch(X, size): 5 | a = np.random.choice(len(X), size, replace=False) 6 | return X[a] 7 | 8 | class Autoencoder: 9 | def __init__(self, input_dim, hidden_dim, epoch=1000, batch_size=50, learning_rate=0.001): 10 | self.epoch = epoch 11 | self.batch_size = batch_size 12 | self.learning_rate = learning_rate 13 | 14 | x = tf.placeholder(dtype=tf.float32, shape=[None, input_dim]) 15 | with tf.name_scope('encode'): 16 | weights = tf.Variable(tf.random_normal([input_dim, hidden_dim], dtype=tf.float32)) 17 | biases = tf.Variable(tf.zeros([hidden_dim])) 18 | encoded = tf.nn.sigmoid(tf.matmul(x, weights) + biases) 19 | with tf.name_scope('decode'): 20 | weights = tf.Variable(tf.random_normal([hidden_dim, input_dim], dtype=tf.float32)) 21 | biases = tf.Variable(tf.zeros([input_dim])) 22 | decoded = tf.matmul(encoded, weights) + biases 23 | 24 | self.x = x 25 | self.encoded = encoded 26 | self.decoded = decoded 27 | 28 | self.loss = tf.sqrt(tf.reduce_mean(tf.square(tf.sub(self.x, self.decoded)))) 29 | self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss) 30 | self.saver = tf.train.Saver() 31 | 32 | def train(self, data): 33 | with tf.Session() as sess: 34 | sess.run(tf.initialize_all_variables()) 35 | for i in range(self.epoch): 36 | for j in range(50): 37 | batch_data = get_batch(data, self.batch_size) 38 | l, _ = sess.run([self.loss, self.train_op], feed_dict={self.x: batch_data}) 39 | if i % 10 == 0: 40 | print('epoch {0}: loss = {1}'.format(i, l)) 41 | self.saver.save(sess, './model.ckpt') 42 | self.saver.save(sess, './model.ckpt') 43 | 44 | def test(self, data): 45 | with tf.Session() as sess: 46 | self.saver.restore(sess, './model.ckpt') 47 | hidden, reconstructed = sess.run([self.encoded, self.decoded], feed_dict={self.x: data}) 48 | print('input', data) 49 | print('compressed', hidden) 50 | print('reconstructed', reconstructed) 51 | return reconstructed 52 | 53 | def get_params(self): 54 | with tf.Session() as sess: 55 | self.saver.restore(sess, './model.ckpt') 56 | weights, biases = sess.run([self.weights1, self.biases1]) 57 | return weights, biases 58 | -------------------------------------------------------------------------------- /ch07_autoencoder/export_parameters.py: -------------------------------------------------------------------------------- 1 | from autoencoder import Autoencoder 2 | from scipy.misc import imread, imresize, imsave 3 | import numpy as np 4 | import h5py 5 | 6 | def zero_pad(num, pad): 7 | return format(num, '0' + str(pad)) 8 | 9 | data_dir = '../vids/' 10 | filename_prefix = 'raw_rgb_' 11 | 12 | hidden_dim = 1000 13 | 14 | filepath = data_dir + str(1) + '/' + filename_prefix + zero_pad(20, 5) + '.png' 15 | img = imresize(imread(filepath, True), 1. / 8.) 16 | 17 | img_data = img.flatten() 18 | 19 | ae = Autoencoder([img_data], hidden_dim) 20 | 21 | weights, biases = ae.get_params() 22 | 23 | print(np.shape(weights)) 24 | print(np.shape([biases])) 25 | 26 | h5f_W = h5py.File('encoder_W.h5', 'w') 27 | h5f_W.create_dataset('dataset_1', data=weights) 28 | h5f_W.close() 29 | 30 | h5f_b = h5py.File('encoder_b.h5', 'w') 31 | h5f_b.create_dataset('dataset_1', data=[biases]) 32 | h5f_b.close() 33 | -------------------------------------------------------------------------------- /ch07_autoencoder/main.py: -------------------------------------------------------------------------------- 1 | from autoencoder import Autoencoder 2 | from sklearn import datasets 3 | 4 | hidden_dim = 1 5 | data = datasets.load_iris().data 6 | input_dim = len(data[0]) 7 | ae = Autoencoder(input_dim, hidden_dim) 8 | ae.train(data) 9 | ae.test([[8, 4, 6, 2]]) 10 | -------------------------------------------------------------------------------- /ch07_autoencoder/main_imgs.py: -------------------------------------------------------------------------------- 1 | # https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz 2 | 3 | import cPickle 4 | import numpy as np 5 | from autoencoder import Autoencoder 6 | # 7 | # def grayscale(x): 8 | # gray = np.zeros(len(x)/3) 9 | # for i in range(len(x)/3): 10 | # gray[i] = (x[i] + x[2*i] + x[3*i]) / 3 11 | 12 | 13 | def grayscale(a): 14 | return a.reshape(a.shape[0], 3, 32, 32).mean(1).reshape(a.shape[0], -1) 15 | 16 | 17 | def unpickle(file): 18 | fo = open(file, 'rb') 19 | dict = cPickle.load(fo) 20 | fo.close() 21 | return dict 22 | 23 | names = unpickle('./cifar-10-batches-py/batches.meta')['label_names'] 24 | data, labels = [], [] 25 | for i in range(1, 6): 26 | filename = './cifar-10-batches-py/data_batch_' + str(i) 27 | batch_data = unpickle(filename) 28 | if len(data) > 0: 29 | data = np.vstack((data, batch_data['data'])) 30 | labels = np.vstack((labels, batch_data['labels'])) 31 | else: 32 | data = batch_data['data'] 33 | labels = batch_data['labels'] 34 | 35 | data = grayscale(data) 36 | 37 | x = np.matrix(data) 38 | y = np.array(labels) 39 | 40 | horse_indices = np.where(y == 7)[0] 41 | 42 | horse_x = x[horse_indices] 43 | 44 | print(np.shape(horse_x)) # (5000, 3072) 45 | 46 | input_dim = np.shape(horse_x)[1] 47 | hidden_dim = 100 48 | ae = Autoencoder(input_dim, hidden_dim) 49 | ae.train(horse_x) 50 | 51 | test_data = unpickle('./cifar-10-batches-py/test_batch') 52 | test_x = grayscale(test_data['data']) 53 | test_labels = np.array(test_data['labels']) 54 | encoding = ae.classify(test_x, test_labels) 55 | encoding = np.matrix(encoding) 56 | from matplotlib import pyplot as plt 57 | 58 | # encoding = np.matrix(np.random.choice([0, 1], size=(hidden_dim,))) 59 | 60 | original_img = np.reshape(test_x[7,:], (32,32)) 61 | plt.imshow(original_img, cmap='Greys_r') 62 | plt.show() 63 | 64 | print(np.size(encoding)) 65 | while(True): 66 | img = ae.decode(encoding) 67 | plt.imshow(img, cmap='Greys_r') 68 | plt.show() 69 | rand_idx = np.random.randint(np.size(encoding)) 70 | encoding[0,rand_idx] = np.random.randint(2) 71 | -------------------------------------------------------------------------------- /ch07_autoencoder/model.ckpt.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch07_autoencoder/model.ckpt.meta -------------------------------------------------------------------------------- /ch08_rl/.gitignore: -------------------------------------------------------------------------------- 1 | checkpoint -------------------------------------------------------------------------------- /ch08_rl/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 8 2 | 3 |

4 | 5 | Humans learn from past experiences (or, you know, at least they should). You didn’t get so charming by accident. Years of positive compliments as well as negative criticism have all helped shape who you are today. This chapter is all about designing a machine learning system driven by criticisms and rewards. 6 | 7 | Consider the following examples. You learn what makes people happy by interacting with friends, family, or even strangers, and you figure out how to ride a bike by trying out different muscle movements until it just clicks. When you perform actions, you’re sometimes rewarded immediately. For example, finding a restaurant nearby might yield instant gratification. Other times, the reward doesn’t appear right away, such as travelling a long distance to find an exceptional place to eat. Reinforcement learning is all about making the right actions given any state. 8 | 9 | - **Concept 1**: Reinforcement learning 10 | 11 | --- 12 | 13 | * Listing 1-10: `rl.py` -------------------------------------------------------------------------------- /ch08_rl/prices.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch08_rl/prices.png -------------------------------------------------------------------------------- /ch08_rl/rl.py: -------------------------------------------------------------------------------- 1 | from matplotlib import pyplot as plt 2 | import numpy as np 3 | import random 4 | import tensorflow as tf 5 | import random 6 | import pandas as pd 7 | pd.core.common.is_list_like = pd.api.types.is_list_like 8 | from pandas_datareader import data 9 | import datetime 10 | import requests_cache 11 | 12 | class DecisionPolicy: 13 | def select_action(self, current_state, step): 14 | pass 15 | 16 | def update_q(self, state, action, reward, next_state): 17 | pass 18 | 19 | 20 | class RandomDecisionPolicy(DecisionPolicy): 21 | def __init__(self, actions): 22 | self.actions = actions 23 | 24 | def select_action(self, current_state, step): 25 | action = self.actions[random.randint(0, len(self.actions) - 1)] 26 | return action 27 | 28 | 29 | class QLearningDecisionPolicy(DecisionPolicy): 30 | def __init__(self, actions, input_dim): 31 | self.epsilon = 0.9 32 | self.gamma = 0.001 33 | self.actions = actions 34 | output_dim = len(actions) 35 | h1_dim = 200 36 | 37 | self.x = tf.placeholder(tf.float32, [None, input_dim]) 38 | self.y = tf.placeholder(tf.float32, [output_dim]) 39 | W1 = tf.Variable(tf.random_normal([input_dim, h1_dim])) 40 | b1 = tf.Variable(tf.constant(0.1, shape=[h1_dim])) 41 | h1 = tf.nn.relu(tf.matmul(self.x, W1) + b1) 42 | W2 = tf.Variable(tf.random_normal([h1_dim, output_dim])) 43 | b2 = tf.Variable(tf.constant(0.1, shape=[output_dim])) 44 | self.q = tf.nn.relu(tf.matmul(h1, W2) + b2) 45 | 46 | loss = tf.square(self.y - self.q) 47 | self.train_op = tf.train.AdagradOptimizer(0.01).minimize(loss) 48 | self.sess = tf.Session() 49 | self.sess.run(tf.global_variables_initializer()) 50 | 51 | def select_action(self, current_state, step): 52 | threshold = min(self.epsilon, step / 1000.) 53 | if random.random() < threshold: 54 | # Exploit best option with probability epsilon 55 | action_q_vals = self.sess.run(self.q, feed_dict={self.x: current_state}) 56 | action_idx = np.argmax(action_q_vals) # TODO: replace w/ tensorflow's argmax 57 | action = self.actions[action_idx] 58 | else: 59 | # Explore random option with probability 1 - epsilon 60 | action = self.actions[random.randint(0, len(self.actions) - 1)] 61 | return action 62 | 63 | def update_q(self, state, action, reward, next_state): 64 | action_q_vals = self.sess.run(self.q, feed_dict={self.x: state}) 65 | next_action_q_vals = self.sess.run(self.q, feed_dict={self.x: next_state}) 66 | next_action_idx = np.argmax(next_action_q_vals) 67 | action_q_vals[0, next_action_idx] = reward + self.gamma * next_action_q_vals[0, next_action_idx] 68 | action_q_vals = np.squeeze(np.asarray(action_q_vals)) 69 | self.sess.run(self.train_op, feed_dict={self.x: state, self.y: action_q_vals}) 70 | 71 | 72 | def run_simulation(policy, initial_budget, initial_num_stocks, prices, hist, debug=False): 73 | budget = initial_budget 74 | num_stocks = initial_num_stocks 75 | share_value = 0 76 | transitions = list() 77 | for i in range(len(prices) - hist - 1): 78 | if i % 100 == 0: 79 | print('progress {:.2f}%'.format(float(100*i) / (len(prices) - hist - 1))) 80 | current_state = np.asmatrix(np.hstack((prices[i:i+hist], budget, num_stocks))) 81 | current_portfolio = budget + num_stocks * share_value 82 | action = policy.select_action(current_state, i) 83 | share_value = float(prices[i + hist + 1]) 84 | if action == 'Buy' and budget >= share_value: 85 | budget -= share_value 86 | num_stocks += 1 87 | elif action == 'Sell' and num_stocks > 0: 88 | budget += share_value 89 | num_stocks -= 1 90 | else: 91 | action = 'Hold' 92 | new_portfolio = budget + num_stocks * share_value 93 | reward = new_portfolio - current_portfolio 94 | next_state = np.asmatrix(np.hstack((prices[i+1:i+hist+1], budget, num_stocks))) 95 | transitions.append((current_state, action, reward, next_state)) 96 | policy.update_q(current_state, action, reward, next_state) 97 | 98 | portfolio = budget + num_stocks * share_value 99 | if debug: 100 | print('${}\t{} shares'.format(budget, num_stocks)) 101 | return portfolio 102 | 103 | 104 | def run_simulations(policy, budget, num_stocks, prices, hist): 105 | num_tries = 10 106 | final_portfolios = list() 107 | for i in range(num_tries): 108 | final_portfolio = run_simulation(policy, budget, num_stocks, prices, hist) 109 | final_portfolios.append(final_portfolio) 110 | avg, std = np.mean(final_portfolios), np.std(final_portfolios) 111 | return avg, std 112 | 113 | 114 | def get_prices(share_symbol, start_date, end_date): 115 | expire_after = datetime.timedelta(days=3) 116 | session = requests_cache.CachedSession(cache_name='cache', backend='sqlite', expire_after=expire_after) 117 | stock_hist = data.DataReader(share_symbol, 'iex', start_date, end_date, session=session) 118 | open_prices = stock_hist['open'] 119 | return open_prices.values.tolist() 120 | 121 | def plot_prices(prices): 122 | plt.title('Opening stock prices') 123 | plt.xlabel('day') 124 | plt.ylabel('price ($)') 125 | plt.plot(prices) 126 | plt.savefig('prices.png') 127 | 128 | 129 | if __name__ == '__main__': 130 | prices = get_prices('MSFT', '2013-07-22', '2018-07-22') 131 | plot_prices(prices) 132 | actions = ['Buy', 'Sell', 'Hold'] 133 | hist = 200 134 | # policy = RandomDecisionPolicy(actions) 135 | policy = QLearningDecisionPolicy(actions, hist + 2) 136 | budget = 1000.0 137 | num_stocks = 0 138 | avg, std = run_simulations(policy, budget, num_stocks, prices, hist) 139 | print(avg, std) 140 | 141 | -------------------------------------------------------------------------------- /ch08_rl/stock_prices.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BinRoot/TensorFlow-Book/7e8db4fec4333a3ada4954ff7f264328e931a22f/ch08_rl/stock_prices.npy -------------------------------------------------------------------------------- /ch09_cnn/.gitignore: -------------------------------------------------------------------------------- 1 | cifar-10-batches-py 2 | *.png 3 | summaries -------------------------------------------------------------------------------- /ch09_cnn/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 9 2 | 3 |

4 | 5 | After a long day of work, when I go grocery shopping, my eyes feel bombarded with information. Sales, coupons, colors, toddlers, flashing lights, crowded aisles are just a few examples of all the signals my eyes pick up, whether or not I actively try to pay attention. The visual system absorbs an abundance of information. 6 | 7 | Ever heard the phrase “a picture is worth a thousand words?" At least that might be true for you or me, but can a machine find meaning within images as well? The photoreceptor cells in our retinas pick up wavelengths of light, but that information doesn’t seem to propagate up to our consciousness. Similarly, a camera picks up pixels, yet we want to squeeze out some form of higher-level knowledg instead. 8 | 9 | To achieve some intelligent meaning from raw sensory input with machine learning, we’ll design a neural network model. In the previous chapters, we’ve seen a few types of neural networks models such as fully-connected ones or autoencoders. New to this chapter, there’s another type of model called a convolutional neural network (CNN), which performs exceptionally well on images and other sensory data such as audio. 10 | 11 | - **Concept 1**: Using CIFAR-10 dataset 12 | - **Concept 2**: Convolutions 13 | - **Concept 3**: Convolutional neural network -------------------------------------------------------------------------------- /ch09_cnn/cifar_tools.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | 4 | 5 | def unpickle(file): 6 | fo = open(file, 'rb') 7 | dict = pickle.load(fo, encoding='latin1') 8 | fo.close() 9 | return dict 10 | 11 | 12 | def clean(data): 13 | imgs = data.reshape(data.shape[0], 3, 32, 32) 14 | grayscale_imgs = imgs.mean(1) 15 | cropped_imgs = grayscale_imgs[:, 4:28, 4:28] 16 | img_data = cropped_imgs.reshape(data.shape[0], -1) 17 | img_size = np.shape(img_data)[1] 18 | means = np.mean(img_data, axis=1) 19 | meansT = means.reshape(len(means), 1) 20 | stds = np.std(img_data, axis=1) 21 | stdsT = stds.reshape(len(stds), 1) 22 | adj_stds = np.maximum(stdsT, 1.0 / np.sqrt(img_size)) 23 | normalized = (img_data - meansT) / adj_stds 24 | return normalized 25 | 26 | 27 | def read_data(directory): 28 | names = unpickle('{}/batches.meta'.format(directory))['label_names'] 29 | print('names', names) 30 | 31 | data, labels = [], [] 32 | for i in range(1, 6): 33 | filename = '{}/data_batch_{}'.format(directory, i) 34 | batch_data = unpickle(filename) 35 | if len(data) > 0: 36 | data = np.vstack((data, batch_data['data'])) 37 | labels = np.hstack((labels, batch_data['labels'])) 38 | else: 39 | data = batch_data['data'] 40 | labels = batch_data['labels'] 41 | 42 | print(np.shape(data), np.shape(labels)) 43 | 44 | data = clean(data) 45 | data = data.astype(np.float32) 46 | return names, data, labels 47 | -------------------------------------------------------------------------------- /ch09_cnn/cnn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import cifar_tools 4 | import tensorflow as tf 5 | 6 | learning_rate = 0.001 7 | 8 | names, data, labels = \ 9 | cifar_tools.read_data('/home/binroot/res/cifar-10-batches-py') 10 | 11 | x = tf.placeholder(tf.float32, [None, 24 * 24]) 12 | y = tf.placeholder(tf.float32, [None, len(names)]) 13 | W1 = tf.Variable(tf.random_normal([5, 5, 1, 64])) 14 | b1 = tf.Variable(tf.random_normal([64])) 15 | W2 = tf.Variable(tf.random_normal([5, 5, 64, 64])) 16 | b2 = tf.Variable(tf.random_normal([64])) 17 | W3 = tf.Variable(tf.random_normal([6*6*64, 1024])) 18 | b3 = tf.Variable(tf.random_normal([1024])) 19 | W_out = tf.Variable(tf.random_normal([1024, len(names)])) 20 | b_out = tf.Variable(tf.random_normal([len(names)])) 21 | 22 | 23 | def conv_layer(x, W, b): 24 | conv = tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 25 | conv_with_b = tf.nn.bias_add(conv, b) 26 | conv_out = tf.nn.relu(conv_with_b) 27 | return conv_out 28 | 29 | 30 | def maxpool_layer(conv, k=2): 31 | return tf.nn.max_pool(conv, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME') 32 | 33 | 34 | def model(): 35 | x_reshaped = tf.reshape(x, shape=[-1, 24, 24, 1]) 36 | 37 | conv_out1 = conv_layer(x_reshaped, W1, b1) 38 | maxpool_out1 = maxpool_layer(conv_out1) 39 | norm1 = tf.nn.lrn(maxpool_out1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) 40 | conv_out2 = conv_layer(norm1, W2, b2) 41 | norm2 = tf.nn.lrn(conv_out2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) 42 | maxpool_out2 = maxpool_layer(norm2) 43 | 44 | maxpool_reshaped = tf.reshape(maxpool_out2, [-1, W3.get_shape().as_list()[0]]) 45 | local = tf.add(tf.matmul(maxpool_reshaped, W3), b3) 46 | local_out = tf.nn.relu(local) 47 | 48 | out = tf.add(tf.matmul(local_out, W_out), b_out) 49 | return out 50 | 51 | model_op = model() 52 | 53 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(model_op, y)) 54 | train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) 55 | 56 | correct_pred = tf.equal(tf.argmax(model_op, 1), tf.argmax(y, 1)) 57 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) 58 | 59 | with tf.Session() as sess: 60 | sess.run(tf.initialize_all_variables()) 61 | onehot_labels = tf.one_hot(labels, len(names), on_value=1., off_value=0., axis=-1) 62 | onehot_vals = sess.run(onehot_labels) 63 | batch_size = len(data) / 200 64 | print('batch size', batch_size) 65 | for j in range(0, 1000): 66 | print('EPOCH', j) 67 | for i in range(0, len(data), batch_size): 68 | batch_data = data[i:i+batch_size, :] 69 | batch_onehot_vals = onehot_vals[i:i+batch_size, :] 70 | _, accuracy_val = sess.run([train_op, accuracy], feed_dict={x: batch_data, y: batch_onehot_vals}) 71 | if i % 1000 == 0: 72 | print(i, accuracy_val) 73 | print('DONE WITH EPOCH') 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /ch09_cnn/cnn_viz.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import cifar_tools 4 | import tensorflow as tf 5 | 6 | learning_rate = 0.001 7 | 8 | names, data, labels = \ 9 | cifar_tools.read_data('/home/binroot/res/cifar-10-batches-py') 10 | 11 | x = tf.placeholder(tf.float32, [None, 24 * 24], name='input') 12 | y = tf.placeholder(tf.float32, [None, len(names)], name='prediction') 13 | W1 = tf.Variable(tf.random_normal([5, 5, 1, 64]), name='W1') 14 | b1 = tf.Variable(tf.random_normal([64]), name='b1') 15 | W2 = tf.Variable(tf.random_normal([5, 5, 64, 64]), name='W2') 16 | b2 = tf.Variable(tf.random_normal([64]), name='b2') 17 | W3 = tf.Variable(tf.random_normal([6*6*64, 1024]), name='W3') 18 | b3 = tf.Variable(tf.random_normal([1024]), name='b3') 19 | W_out = tf.Variable(tf.random_normal([1024, len(names)]), name='W_out') 20 | b_out = tf.Variable(tf.random_normal([len(names)]), name='b_out') 21 | 22 | 23 | W1_summary = tf.image_summary('W1_img', W1) 24 | 25 | def conv_layer(x, W, b): 26 | conv = tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 27 | conv_with_b = tf.nn.bias_add(conv, b) 28 | conv_out = tf.nn.relu(conv_with_b) 29 | return conv_out 30 | 31 | 32 | def maxpool_layer(conv, k=2): 33 | return tf.nn.max_pool(conv, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME') 34 | 35 | 36 | def model(): 37 | x_reshaped = tf.reshape(x, shape=[-1, 24, 24, 1]) 38 | 39 | conv_out1 = conv_layer(x_reshaped, W1, b1) 40 | maxpool_out1 = maxpool_layer(conv_out1) 41 | norm1 = tf.nn.lrn(maxpool_out1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) 42 | conv_out2 = conv_layer(norm1, W2, b2) 43 | norm2 = tf.nn.lrn(conv_out2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) 44 | maxpool_out2 = maxpool_layer(norm2) 45 | 46 | maxpool_reshaped = tf.reshape(maxpool_out2, [-1, W3.get_shape().as_list()[0]]) 47 | local = tf.add(tf.matmul(maxpool_reshaped, W3), b3) 48 | local_out = tf.nn.relu(local) 49 | 50 | out = tf.add(tf.matmul(local_out, W_out), b_out) 51 | return out 52 | 53 | model_op = model() 54 | 55 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(model_op, y)) 56 | tf.scalar_summary('cost', cost) 57 | train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) 58 | 59 | correct_pred = tf.equal(tf.argmax(model_op, 1), tf.argmax(y, 1)) 60 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) 61 | 62 | merged = tf.merge_all_summaries() 63 | 64 | with tf.Session() as sess: 65 | summary_writer = tf.train.SummaryWriter('summaries/train', sess.graph) 66 | sess.run(tf.initialize_all_variables()) 67 | onehot_labels = tf.one_hot(labels, len(names), on_value=1., off_value=0., axis=-1) 68 | onehot_vals = sess.run(onehot_labels) 69 | batch_size = len(data) / 200 70 | print('batch size', batch_size) 71 | for j in range(0, 1000): 72 | print('EPOCH', j) 73 | for i in range(0, len(data), batch_size): 74 | batch_data = data[i:i+batch_size, :] 75 | batch_onehot_vals = onehot_vals[i:i+batch_size, :] 76 | _, accuracy_val, summary = sess.run([train_op, accuracy, merged], feed_dict={x: batch_data, y: batch_onehot_vals}) 77 | summary_writer.add_summary(summary, i) 78 | if i % 1000 == 0: 79 | print(i, accuracy_val) 80 | print('DONE WITH EPOCH') 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /ch09_cnn/conv_visuals.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import cifar_tools 4 | import tensorflow as tf 5 | 6 | names, data, labels = \ 7 | cifar_tools.read_data('/home/binroot/res/cifar-10-batches-py') 8 | 9 | 10 | def show_conv_results(data, filename=None): 11 | plt.figure() 12 | rows, cols = 4, 8 13 | for i in range(np.shape(data)[3]): 14 | img = data[0, :, :, i] 15 | plt.subplot(rows, cols, i + 1) 16 | plt.imshow(img, cmap='Greys_r', interpolation='none') 17 | plt.axis('off') 18 | if filename: 19 | plt.savefig(filename) 20 | else: 21 | plt.show() 22 | 23 | 24 | def show_weights(W, filename=None): 25 | plt.figure() 26 | rows, cols = 4, 8 27 | for i in range(np.shape(W)[3]): 28 | img = W[:, :, 0, i] 29 | plt.subplot(rows, cols, i + 1) 30 | plt.imshow(img, cmap='Greys_r', interpolation='none') 31 | plt.axis('off') 32 | if filename: 33 | plt.savefig(filename) 34 | else: 35 | plt.show() 36 | 37 | raw_data = data[4, :] 38 | raw_img = np.reshape(raw_data, (24, 24)) 39 | plt.figure() 40 | plt.imshow(raw_img, cmap='Greys_r') 41 | plt.savefig('input_image.png') 42 | 43 | x = tf.reshape(raw_data, shape=[-1, 24, 24, 1]) 44 | W = tf.Variable(tf.random_normal([5, 5, 1, 32])) 45 | b = tf.Variable(tf.random_normal([32])) 46 | 47 | conv = tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 48 | conv_with_b = tf.nn.bias_add(conv, b) 49 | conv_out = tf.nn.relu(conv_with_b) 50 | 51 | k = 2 52 | maxpool = tf.nn.max_pool(conv_out, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME') 53 | 54 | with tf.Session() as sess: 55 | sess.run(tf.initialize_all_variables()) 56 | 57 | W_val = sess.run(W) 58 | show_weights(W_val, 'step0_weights.png') 59 | 60 | conv_val = sess.run(conv) 61 | show_conv_results(conv_val, 'step1_convs.png') 62 | print(np.shape(conv_val)) 63 | 64 | conv_out_val = sess.run(conv_out) 65 | show_conv_results(conv_out_val, 'step2_conv_outs.png') 66 | print(np.shape(conv_out_val)) 67 | 68 | maxpool_val = sess.run(maxpool) 69 | show_conv_results(maxpool_val, 'step3_maxpool.png') 70 | print(np.shape(maxpool_val)) 71 | 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /ch09_cnn/using_cifar.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import cifar_tools 4 | import random 5 | 6 | names, data, labels = \ 7 | cifar_tools.read_data('/home/binroot/res/cifar-10-batches-py') 8 | 9 | random.seed(1) 10 | 11 | 12 | def show_some_examples(names, data, labels): 13 | plt.figure() 14 | rows, cols = 4, 4 15 | random_idxs = random.sample(range(len(data)), rows * cols) 16 | for i in range(rows * cols): 17 | plt.subplot(rows, cols, i + 1) 18 | j = random_idxs[i] 19 | plt.title(names[labels[j]]) 20 | img = np.reshape(data[j, :], (24, 24)) 21 | plt.imshow(img, cmap='Greys_r') 22 | plt.axis('off') 23 | plt.tight_layout() 24 | plt.savefig('cifar_examples.png') 25 | 26 | show_some_examples(names, data, labels) -------------------------------------------------------------------------------- /ch10_rnn/.gitignore: -------------------------------------------------------------------------------- 1 | *.png 2 | *.csv 3 | checkpoint 4 | *.ckpt* -------------------------------------------------------------------------------- /ch10_rnn/Concept02_rnn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ch `10`: Concept `02`" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Recurrent Neural Network" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "Import the relevant libraries:" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 2, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import numpy as np\n", 31 | "import tensorflow as tf\n", 32 | "from tensorflow.contrib import rnn" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "Define the RNN model:" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 3, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "class SeriesPredictor:\n", 49 | "\n", 50 | " def __init__(self, input_dim, seq_size, hidden_dim=10):\n", 51 | " # Hyperparameters\n", 52 | " self.input_dim = input_dim\n", 53 | " self.seq_size = seq_size\n", 54 | " self.hidden_dim = hidden_dim\n", 55 | "\n", 56 | " # Weight variables and input placeholders\n", 57 | " self.W_out = tf.Variable(tf.random_normal([hidden_dim, 1]), name='W_out')\n", 58 | " self.b_out = tf.Variable(tf.random_normal([1]), name='b_out')\n", 59 | " self.x = tf.placeholder(tf.float32, [None, seq_size, input_dim])\n", 60 | " self.y = tf.placeholder(tf.float32, [None, seq_size])\n", 61 | "\n", 62 | " # Cost optimizer\n", 63 | " self.cost = tf.reduce_mean(tf.square(self.model() - self.y))\n", 64 | " self.train_op = tf.train.AdamOptimizer().minimize(self.cost)\n", 65 | "\n", 66 | " # Auxiliary ops\n", 67 | " self.saver = tf.train.Saver()\n", 68 | "\n", 69 | " def model(self):\n", 70 | " \"\"\"\n", 71 | " :param x: inputs of size [T, batch_size, input_size]\n", 72 | " :param W: matrix of fully-connected output layer weights\n", 73 | " :param b: vector of fully-connected output layer biases\n", 74 | " \"\"\"\n", 75 | " cell = rnn.BasicLSTMCell(self.hidden_dim, reuse=tf.get_variable_scope().reuse)\n", 76 | " outputs, states = tf.nn.dynamic_rnn(cell, self.x, dtype=tf.float32)\n", 77 | " num_examples = tf.shape(self.x)[0]\n", 78 | " W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1])\n", 79 | " out = tf.matmul(outputs, W_repeated) + self.b_out\n", 80 | " out = tf.squeeze(out)\n", 81 | " return out\n", 82 | "\n", 83 | " def train(self, train_x, train_y):\n", 84 | " with tf.Session() as sess:\n", 85 | " tf.get_variable_scope().reuse_variables()\n", 86 | " sess.run(tf.global_variables_initializer())\n", 87 | " for i in range(1000):\n", 88 | " _, mse = sess.run([self.train_op, self.cost], feed_dict={self.x: train_x, self.y: train_y})\n", 89 | " if i % 100 == 0:\n", 90 | " print(i, mse)\n", 91 | " save_path = self.saver.save(sess, 'model.ckpt')\n", 92 | " print('Model saved to {}'.format(save_path))\n", 93 | "\n", 94 | " def test(self, test_x):\n", 95 | " with tf.Session() as sess:\n", 96 | " tf.get_variable_scope().reuse_variables()\n", 97 | " self.saver.restore(sess, './model.ckpt')\n", 98 | " output = sess.run(self.model(), feed_dict={self.x: test_x})\n", 99 | " return output\n", 100 | "\n", 101 | "\n" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "Now, we'll train a series predictor. Let's say we have a sequence of numbers `[a, b, c, d]` that we want to transform into `[a, a+b, b+c, c+d]`. We'll give the RNN a couple examples in the training data. Let's see how well it learns this intended transformation:" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 4, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "name": "stdout", 118 | "output_type": "stream", 119 | "text": [ 120 | "0 103.46295\n", 121 | "100 63.418705\n", 122 | "200 23.072838\n", 123 | "300 11.47684\n", 124 | "400 7.195353\n", 125 | "500 4.4564924\n", 126 | "600 2.8910196\n", 127 | "700 1.948163\n", 128 | "800 1.3193887\n", 129 | "900 0.88628125\n", 130 | "Model saved to model.ckpt\n", 131 | "INFO:tensorflow:Restoring parameters from ./model.ckpt\n", 132 | "\n", 133 | "Lets run some tests!\n", 134 | "\n", 135 | "When the input is [[1], [2], [3], [4]]\n", 136 | "The ground truth output should be [[1], [3], [5], [7]]\n", 137 | "And the model thinks it is [0.86705637 2.7930977 5.307706 7.302184 ]\n", 138 | "\n", 139 | "When the input is [[4], [5], [6], [7]]\n", 140 | "The ground truth output should be [[4], [9], [11], [13]]\n", 141 | "And the model thinks it is [ 4.0726233 9.083956 11.937489 12.943668 ]\n", 142 | "\n" 143 | ] 144 | } 145 | ], 146 | "source": [ 147 | "if __name__ == '__main__':\n", 148 | " predictor = SeriesPredictor(input_dim=1, seq_size=4, hidden_dim=10)\n", 149 | " train_x = [[[1], [2], [5], [6]],\n", 150 | " [[5], [7], [7], [8]],\n", 151 | " [[3], [4], [5], [7]]]\n", 152 | " train_y = [[1, 3, 7, 11],\n", 153 | " [5, 12, 14, 15],\n", 154 | " [3, 7, 9, 12]]\n", 155 | " predictor.train(train_x, train_y)\n", 156 | "\n", 157 | " test_x = [[[1], [2], [3], [4]], # 1, 3, 5, 7\n", 158 | " [[4], [5], [6], [7]]] # 4, 9, 11, 13\n", 159 | " actual_y = [[[1], [3], [5], [7]],\n", 160 | " [[4], [9], [11], [13]]]\n", 161 | " pred_y = predictor.test(test_x)\n", 162 | " \n", 163 | " print(\"\\nLets run some tests!\\n\")\n", 164 | " \n", 165 | " for i, x in enumerate(test_x):\n", 166 | " print(\"When the input is {}\".format(x))\n", 167 | " print(\"The ground truth output should be {}\".format(actual_y[i]))\n", 168 | " print(\"And the model thinks it is {}\\n\".format(pred_y[i]))" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "metadata": {}, 175 | "outputs": [], 176 | "source": [] 177 | } 178 | ], 179 | "metadata": { 180 | "kernelspec": { 181 | "display_name": "Python 3", 182 | "language": "python", 183 | "name": "python3" 184 | }, 185 | "language_info": { 186 | "codemirror_mode": { 187 | "name": "ipython", 188 | "version": 3 189 | }, 190 | "file_extension": ".py", 191 | "mimetype": "text/x-python", 192 | "name": "python", 193 | "nbconvert_exporter": "python", 194 | "pygments_lexer": "ipython3", 195 | "version": "3.6.5" 196 | } 197 | }, 198 | "nbformat": 4, 199 | "nbformat_minor": 1 200 | } 201 | -------------------------------------------------------------------------------- /ch10_rnn/README.md: -------------------------------------------------------------------------------- 1 | # Chapter 10 2 | 3 |

4 | 5 | Back in school, I remember the sigh of relief when one of my midterm exams was made up of only true-or-false questions. I can’t be the only one that assumed half the answers would be “true” and the other half would be “false.” 6 | 7 | I figured out answers to a most of the questions, and left the rest to random guessing. Actually, I did something clever, a strategy that you might have employed as well. After counting my number of “true” answers, I realized a disproportionate amount of “false” answers were lacking. So, a majority of my guesses were “false” to balance the distribution. 8 | 9 | It worked. I sure felt sly in the moment. What exactly is this feeling of craftiness that makes us feel so confident in our decisions, and how can we give a neural network the same power? 10 | 11 | One answer to this question is to use context to answer questions. Contextual cues are important signals that can also improve the performance of machine learning algorithms. For example, imagine you want to examine an English sentence and tag the part of speech of each word. The naive approach is to individually classify each word as a “noun,”, “adjective,”, and so on, without acknowledging its neighboring words. Consider trying that technique on the words in this sentence. The word “trying” was used as a verb, but depending on the context, you can also use it as an adjective, making parts-of-speech tagging a very trying problem. 12 | 13 | A better approach would consider the context. To bestow neural networks with contextual cues, we’ll study an architecture called a recurrent neural network. Instead of natural language data, we’ll be dealing with continuous timeseries data, similar to stock-market prices, as covered in previous chapters. By the end of the chapter, you’ll be able to model the patterns in timeseries data to make predictions about future value 14 | 15 | - **Concept 1**: Loading timeseries data 16 | - **Concept 2**: Recurrent neural networks 17 | - **Concept 3**: Applying RNN to real-world data for timeseries prediction -------------------------------------------------------------------------------- /ch10_rnn/data_loader.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | 6 | def load_series(filename, series_idx=1): 7 | try: 8 | with open(filename) as csvfile: 9 | csvreader = csv.reader(csvfile) 10 | data = [float(row[series_idx]) for row in csvreader if len(row) > 0] 11 | normalized_data = (data - np.mean(data)) / np.std(data) 12 | return normalized_data 13 | except IOError: 14 | return None 15 | 16 | 17 | def split_data(data, percent_train=0.80): 18 | num_rows = len(data) 19 | train_data, test_data = [], [] 20 | for idx, row in enumerate(data): 21 | if idx < num_rows * percent_train: 22 | train_data.append(row) 23 | else: 24 | test_data.append(row) 25 | return train_data, test_data 26 | 27 | 28 | if __name__=='__main__': 29 | # https://datamarket.com/data/set/22u3/international-airline-passengers-monthly-totals-in-thousands-jan-49-dec-60#!ds=22u3&display=line 30 | timeseries = load_series('international-airline-passengers.csv') 31 | print(np.shape(timeseries)) 32 | 33 | plt.figure() 34 | plt.plot(timeseries) 35 | plt.show() 36 | 37 | -------------------------------------------------------------------------------- /ch10_rnn/international-airline-passengers.csv: -------------------------------------------------------------------------------- 1 | "1949-01",112 2 | "1949-02",118 3 | "1949-03",132 4 | "1949-04",129 5 | "1949-05",121 6 | "1949-06",135 7 | "1949-07",148 8 | "1949-08",148 9 | "1949-09",136 10 | "1949-10",119 11 | "1949-11",104 12 | "1949-12",118 13 | "1950-01",115 14 | "1950-02",126 15 | "1950-03",141 16 | "1950-04",135 17 | "1950-05",125 18 | "1950-06",149 19 | "1950-07",170 20 | "1950-08",170 21 | "1950-09",158 22 | "1950-10",133 23 | "1950-11",114 24 | "1950-12",140 25 | "1951-01",145 26 | "1951-02",150 27 | "1951-03",178 28 | "1951-04",163 29 | "1951-05",172 30 | "1951-06",178 31 | "1951-07",199 32 | "1951-08",199 33 | "1951-09",184 34 | "1951-10",162 35 | "1951-11",146 36 | "1951-12",166 37 | "1952-01",171 38 | "1952-02",180 39 | "1952-03",193 40 | "1952-04",181 41 | "1952-05",183 42 | "1952-06",218 43 | "1952-07",230 44 | "1952-08",242 45 | "1952-09",209 46 | "1952-10",191 47 | "1952-11",172 48 | "1952-12",194 49 | "1953-01",196 50 | "1953-02",196 51 | "1953-03",236 52 | "1953-04",235 53 | "1953-05",229 54 | "1953-06",243 55 | "1953-07",264 56 | "1953-08",272 57 | "1953-09",237 58 | "1953-10",211 59 | "1953-11",180 60 | "1953-12",201 61 | "1954-01",204 62 | "1954-02",188 63 | "1954-03",235 64 | "1954-04",227 65 | "1954-05",234 66 | "1954-06",264 67 | "1954-07",302 68 | "1954-08",293 69 | "1954-09",259 70 | "1954-10",229 71 | "1954-11",203 72 | "1954-12",229 73 | "1955-01",242 74 | "1955-02",233 75 | "1955-03",267 76 | "1955-04",269 77 | "1955-05",270 78 | "1955-06",315 79 | "1955-07",364 80 | "1955-08",347 81 | "1955-09",312 82 | "1955-10",274 83 | "1955-11",237 84 | "1955-12",278 85 | "1956-01",284 86 | "1956-02",277 87 | "1956-03",317 88 | "1956-04",313 89 | "1956-05",318 90 | "1956-06",374 91 | "1956-07",413 92 | "1956-08",405 93 | "1956-09",355 94 | "1956-10",306 95 | "1956-11",271 96 | "1956-12",306 97 | "1957-01",315 98 | "1957-02",301 99 | "1957-03",356 100 | "1957-04",348 101 | "1957-05",355 102 | "1957-06",422 103 | "1957-07",465 104 | "1957-08",467 105 | "1957-09",404 106 | "1957-10",347 107 | "1957-11",305 108 | "1957-12",336 109 | "1958-01",340 110 | "1958-02",318 111 | "1958-03",362 112 | "1958-04",348 113 | "1958-05",363 114 | "1958-06",435 115 | "1958-07",491 116 | "1958-08",505 117 | "1958-09",404 118 | "1958-10",359 119 | "1958-11",310 120 | "1958-12",337 121 | "1959-01",360 122 | "1959-02",342 123 | "1959-03",406 124 | "1959-04",396 125 | "1959-05",420 126 | "1959-06",472 127 | "1959-07",548 128 | "1959-08",559 129 | "1959-09",463 130 | "1959-10",407 131 | "1959-11",362 132 | "1959-12",405 133 | "1960-01",417 134 | "1960-02",391 135 | "1960-03",419 136 | "1960-04",461 137 | "1960-05",472 138 | "1960-06",535 139 | "1960-07",622 140 | "1960-08",606 141 | "1960-09",508 142 | "1960-10",461 143 | "1960-11",390 144 | "1960-12",432 145 | -------------------------------------------------------------------------------- /ch10_rnn/regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from tensorflow.python.ops import rnn, rnn_cell 4 | import data_loader 5 | import matplotlib.pyplot as plt 6 | 7 | class SeriesPredictor: 8 | 9 | def __init__(self, input_dim, seq_size, hidden_dim): 10 | # Hyperparameters 11 | self.input_dim = input_dim 12 | self.seq_size = seq_size 13 | self.hidden_dim = hidden_dim 14 | 15 | # Weight variables and input placeholders 16 | self.W_out = tf.Variable(tf.random_normal([hidden_dim, 1]), name='W_out') 17 | self.b_out = tf.Variable(tf.random_normal([1]), name='b_out') 18 | self.x = tf.placeholder(tf.float32, [None, seq_size, input_dim]) 19 | self.y = tf.placeholder(tf.float32, [None, seq_size]) 20 | 21 | # Cost optimizer 22 | self.cost = tf.reduce_mean(tf.square(self.model() - self.y)) 23 | self.train_op = tf.train.AdamOptimizer(learning_rate=0.003).minimize(self.cost) 24 | 25 | # Auxiliary ops 26 | self.saver = tf.train.Saver() 27 | 28 | def model(self): 29 | """ 30 | :param x: inputs of size [T, batch_size, input_size] 31 | :param W: matrix of fully-connected output layer weights 32 | :param b: vector of fully-connected output layer biases 33 | """ 34 | cell = rnn_cell.BasicLSTMCell(self.hidden_dim) 35 | outputs, states = rnn.dynamic_rnn(cell, self.x, dtype=tf.float32) 36 | num_examples = tf.shape(self.x)[0] 37 | W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1]) 38 | out = tf.batch_matmul(outputs, W_repeated) + self.b_out 39 | out = tf.squeeze(out) 40 | return out 41 | 42 | def train(self, train_x, train_y, test_x, test_y): 43 | with tf.Session() as sess: 44 | tf.get_variable_scope().reuse_variables() 45 | sess.run(tf.initialize_all_variables()) 46 | max_patience = 3 47 | patience = max_patience 48 | min_test_err = float('inf') 49 | step = 0 50 | while patience > 0: 51 | _, train_err = sess.run([self.train_op, self.cost], feed_dict={self.x: train_x, self.y: train_y}) 52 | if step % 100 == 0: 53 | test_err = sess.run(self.cost, feed_dict={self.x: test_x, self.y: test_y}) 54 | print('step: {}\t\ttrain err: {}\t\ttest err: {}'.format(step, train_err, test_err)) 55 | if test_err < min_test_err: 56 | min_test_err = test_err 57 | patience = max_patience 58 | else: 59 | patience -= 1 60 | step += 1 61 | save_path = self.saver.save(sess, 'model.ckpt') 62 | print('Model saved to {}'.format(save_path)) 63 | 64 | def test(self, sess, test_x): 65 | tf.get_variable_scope().reuse_variables() 66 | self.saver.restore(sess, 'model.ckpt') 67 | output = sess.run(self.model(), feed_dict={self.x: test_x}) 68 | return output 69 | 70 | 71 | def plot_results(train_x, predictions, actual, filename): 72 | plt.figure() 73 | num_train = len(train_x) 74 | plt.plot(list(range(num_train)), train_x, color='b', label='training data') 75 | plt.plot(list(range(num_train, num_train + len(predictions))), predictions, color='r', label='predicted') 76 | plt.plot(list(range(num_train, num_train + len(actual))), actual, color='g', label='test data') 77 | plt.legend() 78 | if filename is not None: 79 | plt.savefig(filename) 80 | else: 81 | plt.show() 82 | 83 | 84 | if __name__ == '__main__': 85 | seq_size = 5 86 | predictor = SeriesPredictor(input_dim=1, seq_size=seq_size, hidden_dim=5) 87 | data = data_loader.load_series('international-airline-passengers.csv') 88 | train_data, actual_vals = data_loader.split_data(data) 89 | 90 | train_x, train_y = [], [] 91 | for i in range(len(train_data) - seq_size - 1): 92 | train_x.append(np.expand_dims(train_data[i:i+seq_size], axis=1).tolist()) 93 | train_y.append(train_data[i+1:i+seq_size+1]) 94 | 95 | test_x, test_y = [], [] 96 | for i in range(len(actual_vals) - seq_size - 1): 97 | test_x.append(np.expand_dims(actual_vals[i:i+seq_size], axis=1).tolist()) 98 | test_y.append(actual_vals[i+1:i+seq_size+1]) 99 | 100 | predictor.train(train_x, train_y, test_x, test_y) 101 | 102 | with tf.Session() as sess: 103 | predicted_vals = predictor.test(sess, test_x)[:,0] 104 | print('predicted_vals', np.shape(predicted_vals)) 105 | plot_results(train_data, predicted_vals, actual_vals, 'predictions.png') 106 | 107 | prev_seq = train_x[-1] 108 | predicted_vals = [] 109 | for i in range(20): 110 | next_seq = predictor.test(sess, [prev_seq]) 111 | predicted_vals.append(next_seq[-1]) 112 | prev_seq = np.vstack((prev_seq[1:], next_seq[-1])) 113 | plot_results(train_data, predicted_vals, actual_vals, 'hallucinations.png') 114 | -------------------------------------------------------------------------------- /ch10_rnn/simple_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from tensorflow.python.ops import rnn, rnn_cell 4 | 5 | 6 | class SeriesPredictor: 7 | 8 | def __init__(self, input_dim, seq_size, hidden_dim=10): 9 | # Hyperparameters 10 | self.input_dim = input_dim 11 | self.seq_size = seq_size 12 | self.hidden_dim = hidden_dim 13 | 14 | # Weight variables and input placeholders 15 | self.W_out = tf.Variable(tf.random_normal([hidden_dim, 1]), name='W_out') 16 | self.b_out = tf.Variable(tf.random_normal([1]), name='b_out') 17 | self.x = tf.placeholder(tf.float32, [None, seq_size, input_dim]) 18 | self.y = tf.placeholder(tf.float32, [None, seq_size]) 19 | 20 | # Cost optimizer 21 | self.cost = tf.reduce_mean(tf.square(self.model() - self.y)) 22 | self.train_op = tf.train.AdamOptimizer().minimize(self.cost) 23 | 24 | # Auxiliary ops 25 | self.saver = tf.train.Saver() 26 | 27 | def model(self): 28 | """ 29 | :param x: inputs of size [T, batch_size, input_size] 30 | :param W: matrix of fully-connected output layer weights 31 | :param b: vector of fully-connected output layer biases 32 | """ 33 | cell = rnn_cell.BasicLSTMCell(self.hidden_dim, reuse=tf.get_variable_scope().reuse) 34 | outputs, states = rnn.dynamic_rnn(cell, self.x, dtype=tf.float32) 35 | num_examples = tf.shape(self.x)[0] 36 | W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1]) 37 | out = tf.batch_matmul(outputs, W_repeated) + self.b_out 38 | out = tf.squeeze(out) 39 | return out 40 | 41 | def train(self, train_x, train_y): 42 | with tf.Session() as sess: 43 | tf.get_variable_scope().reuse_variables() 44 | sess.run(tf.initialize_all_variables()) 45 | for i in range(1000): 46 | _, mse = sess.run([self.train_op, self.cost], feed_dict={self.x: train_x, self.y: train_y}) 47 | if i % 100 == 0: 48 | print(i, mse) 49 | save_path = self.saver.save(sess, 'model.ckpt') 50 | print('Model saved to {}'.format(save_path)) 51 | 52 | def test(self, test_x): 53 | with tf.Session() as sess: 54 | tf.get_variable_scope().reuse_variables() 55 | self.saver.restore(sess, 'model.ckpt') 56 | output = sess.run(self.model(), feed_dict={self.x: test_x}) 57 | print(output) 58 | 59 | 60 | if __name__ == '__main__': 61 | predictor = SeriesPredictor(input_dim=1, seq_size=4, hidden_dim=10) 62 | train_x = [[[1], [2], [5], [6]], 63 | [[5], [7], [7], [8]], 64 | [[3], [4], [5], [7]]] 65 | train_y = [[1, 3, 7, 11], 66 | [5, 12, 14, 15], 67 | [3, 7, 9, 12]] 68 | predictor.train(train_x, train_y) 69 | 70 | test_x = [[[1], [2], [3], [4]], # 1, 3, 5, 7 71 | [[4], [5], [6], [7]]] # 4, 9, 11, 13 72 | predictor.test(test_x) 73 | -------------------------------------------------------------------------------- /ch11_seq2seq/.ipynb_checkpoints/Concept01_multi_rnn-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 1 6 | } 7 | -------------------------------------------------------------------------------- /ch11_seq2seq/.ipynb_checkpoints/Concept02_embedding_lookup-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ch `11`: Concept `02`\n", 8 | "\n", 9 | "## Embedding Lookup" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "Import TensorFlow, and begin an interactive session" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": { 23 | "collapsed": true 24 | }, 25 | "outputs": [], 26 | "source": [ 27 | "import tensorflow as tf\n", 28 | "sess = tf.InteractiveSession()" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "Let's say we only have 4 words in our vocabulary: *\"the\"*, *\"fight\"*, *\"wind\"*, and *\"like\"*.\n", 36 | "\n", 37 | "Maybe each word is associated with numbers.\n", 38 | "\n", 39 | "| Word | Number | \n", 40 | "| ------ |:------:|\n", 41 | "| *'the'* | 17 |\n", 42 | "| *'fight'* | 22 |\n", 43 | "| *'wind'* | 35 | \n", 44 | "| *'like'* | 51 |" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 2, 50 | "metadata": { 51 | "collapsed": false 52 | }, 53 | "outputs": [], 54 | "source": [ 55 | "embeddings_0d = tf.constant([17,22,35,51])\n", 56 | "\n" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "Or maybe, they're associated with one-hot vectors.\n", 64 | "\n", 65 | "| Word | Vector | \n", 66 | "| ------ |:------:|\n", 67 | "| *'the '* | [1, 0, 0, 0] |\n", 68 | "| *'fight'* | [0, 1, 0, 0] |\n", 69 | "| *'wind'* | [0, 0, 1, 0] | \n", 70 | "| *'like'* | [0, 0, 0, 1] |" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 3, 76 | "metadata": { 77 | "collapsed": false 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "embeddings_4d = tf.constant([[1, 0, 0, 0],\n", 82 | " [0, 1, 0, 0],\n", 83 | " [0, 0, 1, 0],\n", 84 | " [0, 0, 0, 1]])" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "This may sound over the top, but you can have any tensor you want, not just numbers or vectors.\n", 92 | "\n", 93 | "| Word | Tensor | \n", 94 | "| ------ |:------:|\n", 95 | "| *'the '* | [[1, 0] , [0, 0]] |\n", 96 | "| *'fight'* | [[0, 1] , [0, 0]] |\n", 97 | "| *'wind'* | [[0, 0] , [1, 0]] | \n", 98 | "| *'like'* | [[0, 0] , [0, 1]] |" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 4, 104 | "metadata": { 105 | "collapsed": true 106 | }, 107 | "outputs": [], 108 | "source": [ 109 | "embeddings_2x2d = tf.constant([[[1, 0], [0, 0]],\n", 110 | " [[0, 1], [0, 0]],\n", 111 | " [[0, 0], [1, 0]],\n", 112 | " [[0, 0], [0, 1]]])" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "Let's say we want to find the embeddings for the sentence, \"fight the wind\"." 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 5, 125 | "metadata": { 126 | "collapsed": true 127 | }, 128 | "outputs": [], 129 | "source": [ 130 | "ids = tf.constant([1, 0, 2])" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "metadata": {}, 136 | "source": [ 137 | "We can use the `embedding_lookup` function provided by TensorFlow:" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 6, 143 | "metadata": { 144 | "collapsed": false 145 | }, 146 | "outputs": [ 147 | { 148 | "name": "stdout", 149 | "output_type": "stream", 150 | "text": [ 151 | "[22 17 35]\n" 152 | ] 153 | } 154 | ], 155 | "source": [ 156 | "lookup_0d = sess.run(tf.nn.embedding_lookup(embeddings_0d, ids))\n", 157 | "print(lookup_0d)" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 7, 163 | "metadata": { 164 | "collapsed": false 165 | }, 166 | "outputs": [ 167 | { 168 | "name": "stdout", 169 | "output_type": "stream", 170 | "text": [ 171 | "[[0 1 0 0]\n", 172 | " [1 0 0 0]\n", 173 | " [0 0 1 0]]\n" 174 | ] 175 | } 176 | ], 177 | "source": [ 178 | "lookup_4d = sess.run(tf.nn.embedding_lookup(embeddings_4d, ids))\n", 179 | "print(lookup_4d)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 8, 185 | "metadata": { 186 | "collapsed": false 187 | }, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "[[[0 1]\n", 194 | " [0 0]]\n", 195 | "\n", 196 | " [[1 0]\n", 197 | " [0 0]]\n", 198 | "\n", 199 | " [[0 0]\n", 200 | " [1 0]]]\n" 201 | ] 202 | } 203 | ], 204 | "source": [ 205 | "lookup_2x2d = sess.run(tf.nn.embedding_lookup(embeddings_2x2d, ids))\n", 206 | "print(lookup_2x2d)" 207 | ] 208 | } 209 | ], 210 | "metadata": { 211 | "kernelspec": { 212 | "display_name": "Python 2", 213 | "language": "python", 214 | "name": "python2" 215 | }, 216 | "language_info": { 217 | "codemirror_mode": { 218 | "name": "ipython", 219 | "version": 2 220 | }, 221 | "file_extension": ".py", 222 | "mimetype": "text/x-python", 223 | "name": "python", 224 | "nbconvert_exporter": "python", 225 | "pygments_lexer": "ipython2", 226 | "version": "2.7.12" 227 | } 228 | }, 229 | "nbformat": 4, 230 | "nbformat_minor": 1 231 | } 232 | -------------------------------------------------------------------------------- /ch11_seq2seq/Concept01_multi_rnn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ch `11`: Concept `01`\n", 8 | "\n", 9 | "## Multi RNN" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "All we need is TensorFlow:" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": { 23 | "collapsed": true 24 | }, 25 | "outputs": [], 26 | "source": [ 27 | "import tensorflow as tf" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "First, define the constants. \n", 35 | "\n", 36 | "Let's say we're dealing with 1-dimensional vectors, and a maximum sequence size of 3." 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 2, 42 | "metadata": { 43 | "collapsed": true 44 | }, 45 | "outputs": [], 46 | "source": [ 47 | "input_dim = 1\n", 48 | "seq_size = 3" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "Next up, define the placeholder(s). \n", 56 | "\n", 57 | "We only need one for this simple example: the input placeholder." 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 3, 63 | "metadata": { 64 | "collapsed": false 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "input_placeholder = tf.placeholder(dtype=tf.float32, shape=[None, seq_size, input_dim])" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "Now let's make a helper function to create LSTM cells" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 4, 81 | "metadata": { 82 | "collapsed": true 83 | }, 84 | "outputs": [], 85 | "source": [ 86 | "def make_cell(state_dim):\n", 87 | " return tf.contrib.rnn.LSTMCell(state_dim)" 88 | ] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": {}, 93 | "source": [ 94 | "Call the function and extract the cell outputs." 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 5, 100 | "metadata": { 101 | "collapsed": false 102 | }, 103 | "outputs": [], 104 | "source": [ 105 | "with tf.variable_scope(\"first_cell\") as scope:\n", 106 | " cell = make_cell(state_dim=10)\n", 107 | " outputs, states = tf.nn.dynamic_rnn(cell, input_placeholder, dtype=tf.float32)" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "You know what? We can just keep stacking cells on top of each other. In a new variable scope, you can pipe the output of the previous cell to the input of the new cell. Check it out:" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 6, 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "outputs": [], 124 | "source": [ 125 | "with tf.variable_scope(\"second_cell\") as scope:\n", 126 | " cell2 = make_cell(state_dim=10)\n", 127 | " outputs2, states2 = tf.nn.dynamic_rnn(cell2, outputs, dtype=tf.float32)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "What if we wanted 5 layers of RNNs? \n", 135 | "\n", 136 | "There's a useful shortcut that the TensorFlow library supplies, called `MultiRNNCell`. Here's a helper function to use it:" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 7, 142 | "metadata": { 143 | "collapsed": true 144 | }, 145 | "outputs": [], 146 | "source": [ 147 | "def make_multi_cell(state_dim, num_layers):\n", 148 | " cells = [make_cell(state_dim) for _ in range(num_layers)]\n", 149 | " return tf.contrib.rnn.MultiRNNCell(cells)" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "Here's the helper function in action:" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 8, 162 | "metadata": { 163 | "collapsed": false 164 | }, 165 | "outputs": [], 166 | "source": [ 167 | "multi_cell = make_multi_cell(state_dim=10, num_layers=5)\n", 168 | "outputs5, states5 = tf.nn.dynamic_rnn(multi_cell, input_placeholder, dtype=tf.float32)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "Before starting a session, let's prepare some simple input to the network." 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 9, 181 | "metadata": { 182 | "collapsed": false 183 | }, 184 | "outputs": [], 185 | "source": [ 186 | "input_seq = [[1], [2], [3]]\n", 187 | "\n" 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "metadata": {}, 193 | "source": [ 194 | "Start the session, and initialize variables." 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 10, 200 | "metadata": { 201 | "collapsed": true 202 | }, 203 | "outputs": [], 204 | "source": [ 205 | "init_op = tf.global_variables_initializer()\n", 206 | "\n", 207 | "sess = tf.InteractiveSession()\n", 208 | "sess.run(init_op)" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "We can run the outputs to verify that the code is sound." 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": 11, 221 | "metadata": { 222 | "collapsed": false 223 | }, 224 | "outputs": [ 225 | { 226 | "name": "stdout", 227 | "output_type": "stream", 228 | "text": [ 229 | "[[[ 0.00141981 0.0722062 -0.08216076 0.03216607 0.02329798 0.06957388\n", 230 | " -0.04552787 -0.05649291 0.05059107 0.01796713]\n", 231 | " [-0.00228921 0.16875982 -0.21222715 0.0772687 0.05970224 0.16551635\n", 232 | " -0.10631067 -0.13780777 0.12389956 0.05248111]\n", 233 | " [-0.01359726 0.24421771 -0.33965409 0.11412902 0.0964628 0.25151449\n", 234 | " -0.16440172 -0.22563797 0.18972857 0.09557904]]]\n", 235 | "[[[-0.00224876 0.01402885 0.00929528 -0.00392457 0.00333697 -0.00213898\n", 236 | " -0.0046619 -0.01061259 0.00368386 0.00040365]\n", 237 | " [-0.00939647 0.04281064 0.02773804 -0.01503811 0.01025065 -0.00612708\n", 238 | " -0.01655139 -0.03407493 0.01263932 0.00136939]\n", 239 | " [-0.02229579 0.07774397 0.0480789 -0.03287651 0.017735 -0.01063949\n", 240 | " -0.03610384 -0.06736942 0.02458673 0.00139557]]]\n", 241 | "[[[ 1.42336748e-05 -1.58296571e-05 -1.62987853e-05 1.59381907e-05\n", 242 | " 1.33105495e-05 -1.38451333e-05 -2.20941274e-05 2.54621627e-05\n", 243 | " 2.26147549e-05 -3.30040712e-05]\n", 244 | " [ 8.32868463e-05 -8.99614606e-05 -1.02287340e-04 8.68237403e-05\n", 245 | " 8.19651614e-05 -7.38111194e-05 -1.29947555e-04 1.52955734e-04\n", 246 | " 1.36927833e-04 -1.89826897e-04]\n", 247 | " [ 2.79068598e-04 -2.77705112e-04 -3.54834105e-04 2.59170338e-04\n", 248 | " 2.73422222e-04 -2.13255596e-04 -4.27101302e-04 5.09521109e-04\n", 249 | " 4.57556103e-04 -6.11643540e-04]]]\n" 250 | ] 251 | } 252 | ], 253 | "source": [ 254 | "outputs_val, outputs2_val, outputs5_val = sess.run([outputs, outputs2, outputs5], \n", 255 | " feed_dict={input_placeholder: [input_seq]})\n", 256 | "print(outputs_val)\n", 257 | "print(outputs2_val)\n", 258 | "print(outputs5_val)" 259 | ] 260 | } 261 | ], 262 | "metadata": { 263 | "kernelspec": { 264 | "display_name": "Python 3", 265 | "language": "python", 266 | "name": "python3" 267 | }, 268 | "language_info": { 269 | "codemirror_mode": { 270 | "name": "ipython", 271 | "version": 3 272 | }, 273 | "file_extension": ".py", 274 | "mimetype": "text/x-python", 275 | "name": "python", 276 | "nbconvert_exporter": "python", 277 | "pygments_lexer": "ipython3", 278 | "version": "3.5.2" 279 | } 280 | }, 281 | "nbformat": 4, 282 | "nbformat_minor": 1 283 | } 284 | -------------------------------------------------------------------------------- /ch11_seq2seq/Concept02_embedding_lookup.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ch `11`: Concept `02`\n", 8 | "\n", 9 | "## Embedding Lookup" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "Import TensorFlow, and begin an interactive session" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": { 23 | "collapsed": true 24 | }, 25 | "outputs": [], 26 | "source": [ 27 | "import tensorflow as tf\n", 28 | "sess = tf.InteractiveSession()" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "Let's say we only have 4 words in our vocabulary: *\"the\"*, *\"fight\"*, *\"wind\"*, and *\"like\"*.\n", 36 | "\n", 37 | "Maybe each word is associated with numbers.\n", 38 | "\n", 39 | "| Word | Number | \n", 40 | "| ------ |:------:|\n", 41 | "| *'the'* | 17 |\n", 42 | "| *'fight'* | 22 |\n", 43 | "| *'wind'* | 35 | \n", 44 | "| *'like'* | 51 |" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 2, 50 | "metadata": { 51 | "collapsed": false 52 | }, 53 | "outputs": [], 54 | "source": [ 55 | "embeddings_0d = tf.constant([17,22,35,51])\n", 56 | "\n" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "Or maybe, they're associated with one-hot vectors.\n", 64 | "\n", 65 | "| Word | Vector | \n", 66 | "| ------ |:------:|\n", 67 | "| *'the '* | [1, 0, 0, 0] |\n", 68 | "| *'fight'* | [0, 1, 0, 0] |\n", 69 | "| *'wind'* | [0, 0, 1, 0] | \n", 70 | "| *'like'* | [0, 0, 0, 1] |" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 3, 76 | "metadata": { 77 | "collapsed": false 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "embeddings_4d = tf.constant([[1, 0, 0, 0],\n", 82 | " [0, 1, 0, 0],\n", 83 | " [0, 0, 1, 0],\n", 84 | " [0, 0, 0, 1]])" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "This may sound over the top, but you can have any tensor you want, not just numbers or vectors.\n", 92 | "\n", 93 | "| Word | Tensor | \n", 94 | "| ------ |:------:|\n", 95 | "| *'the '* | [[1, 0] , [0, 0]] |\n", 96 | "| *'fight'* | [[0, 1] , [0, 0]] |\n", 97 | "| *'wind'* | [[0, 0] , [1, 0]] | \n", 98 | "| *'like'* | [[0, 0] , [0, 1]] |" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 4, 104 | "metadata": { 105 | "collapsed": true 106 | }, 107 | "outputs": [], 108 | "source": [ 109 | "embeddings_2x2d = tf.constant([[[1, 0], [0, 0]],\n", 110 | " [[0, 1], [0, 0]],\n", 111 | " [[0, 0], [1, 0]],\n", 112 | " [[0, 0], [0, 1]]])" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "Let's say we want to find the embeddings for the sentence, \"fight the wind\"." 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 5, 125 | "metadata": { 126 | "collapsed": true 127 | }, 128 | "outputs": [], 129 | "source": [ 130 | "ids = tf.constant([1, 0, 2])" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "metadata": {}, 136 | "source": [ 137 | "We can use the `embedding_lookup` function provided by TensorFlow:" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 6, 143 | "metadata": { 144 | "collapsed": false 145 | }, 146 | "outputs": [ 147 | { 148 | "name": "stdout", 149 | "output_type": "stream", 150 | "text": [ 151 | "[22 17 35]\n" 152 | ] 153 | } 154 | ], 155 | "source": [ 156 | "lookup_0d = sess.run(tf.nn.embedding_lookup(embeddings_0d, ids))\n", 157 | "print(lookup_0d)" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 7, 163 | "metadata": { 164 | "collapsed": false 165 | }, 166 | "outputs": [ 167 | { 168 | "name": "stdout", 169 | "output_type": "stream", 170 | "text": [ 171 | "[[0 1 0 0]\n", 172 | " [1 0 0 0]\n", 173 | " [0 0 1 0]]\n" 174 | ] 175 | } 176 | ], 177 | "source": [ 178 | "lookup_4d = sess.run(tf.nn.embedding_lookup(embeddings_4d, ids))\n", 179 | "print(lookup_4d)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 8, 185 | "metadata": { 186 | "collapsed": false 187 | }, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "[[[0 1]\n", 194 | " [0 0]]\n", 195 | "\n", 196 | " [[1 0]\n", 197 | " [0 0]]\n", 198 | "\n", 199 | " [[0 0]\n", 200 | " [1 0]]]\n" 201 | ] 202 | } 203 | ], 204 | "source": [ 205 | "lookup_2x2d = sess.run(tf.nn.embedding_lookup(embeddings_2x2d, ids))\n", 206 | "print(lookup_2x2d)" 207 | ] 208 | } 209 | ], 210 | "metadata": { 211 | "kernelspec": { 212 | "display_name": "Python 2", 213 | "language": "python", 214 | "name": "python2" 215 | }, 216 | "language_info": { 217 | "codemirror_mode": { 218 | "name": "ipython", 219 | "version": 2 220 | }, 221 | "file_extension": ".py", 222 | "mimetype": "text/x-python", 223 | "name": "python", 224 | "nbconvert_exporter": "python", 225 | "pygments_lexer": "ipython2", 226 | "version": "2.7.12" 227 | } 228 | }, 229 | "nbformat": 4, 230 | "nbformat_minor": 1 231 | } 232 | -------------------------------------------------------------------------------- /ch11_seq2seq/data/process_input.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | NUM_CHARS = 30 4 | 5 | with open('words_input.txt', 'r', encoding='ISO-8859-1') as fi_i, open('words_output.txt', 'r', encoding='ISO-8859-1') as fi_o, open('input_sentences.txt', 'w') as fo_i, open('output_sentences.txt', 'w') as fo_o: 6 | fi_i_fixed = fi_i.read().encode('ascii', 'ignore').decode('UTF-8').lower() 7 | fi_i_alpha = re.sub('[^a-z\n]+', ' ', fi_i_fixed) 8 | fi_o_fixed = fi_o.read().encode('ascii', 'ignore').decode('UTF-8').lower() 9 | fi_o_alpha = re.sub('[^a-z\n]+', ' ', fi_o_fixed) 10 | 11 | for line in fi_i_alpha.split('\n'): 12 | fo_i.write(line[:NUM_CHARS] + '\n') 13 | for line in fi_o_alpha.split('\n'): 14 | fo_o.write(line[:NUM_CHARS] + '\n') 15 | --------------------------------------------------------------------------------