├── .gitignore ├── LICENSE ├── README.md ├── input_data ├── cat1.jpg ├── cat2.jpg └── imagenet.classname.txt ├── tf.version.1 ├── 01.tf.basic │ ├── 01.hello.tf.ipynb │ ├── 02.tf.eager.ipynb │ ├── 03.tensorboard.ipynb │ ├── 04.tf.dimension.ipynb │ ├── 05.tf.Variable.ipynb │ ├── 06.tf.placeholder.ipynb │ ├── 07.tf.train.Saver.ipynb │ ├── 08.tf.cond.ipynb │ ├── 09.tf.control_dependencies.ipynb │ └── 10.tf.data.ipynb ├── 02.regression │ ├── 01.1.linear.regression.ipynb │ ├── 01.2.linear.regression.with.minibatch.ipynb │ ├── 02.1.linear.regression.3rd.order.ipynb │ ├── 02.2.linear.regression.3rd.order.with.minibatch.ipynb │ ├── 03.1.mnist.softmax.ipynb │ ├── 03.2.mnist.softmax.with.tf.data.ipynb │ └── 04.mnist.summary.ipynb ├── 03.cnn │ ├── 01.1.mnist.deep.with.estimator.ipynb │ ├── 01.2.mnist.deep.with.tf.data.ipynb │ ├── 02.mnist.deep.slim.ipynb │ ├── 03.mnist.slim.options.ipynb │ ├── 04.mnist.slim.arg.scope.ipynb │ ├── 05.cnn.cifar10.ipynb │ ├── 06.cnn.cifar10.regularization.ipynb │ ├── 07.cnn.cifar10.data.augmentation.ipynb │ ├── 08.vgg19.slim.ipynb │ ├── 09.inception_v3.slim.ipynb │ ├── 10.tfrecords.ipynb │ └── 11.transfer.learning.with.inception_v3.ipynb ├── 04.rnn │ ├── 01.ready.for.sequence.data.ipynb │ ├── 02.rnn.basic.ipynb │ ├── 03.01.sequence.classification.RNN.ipynb │ ├── 03.02.sequence.classification.GRU.ipynb │ ├── 03.03.sequence.classification.LSTM.ipynb │ ├── 03.04.sequence.classification.biRNN.ipynb │ ├── 03.05.sequence.classification.biLSTM.ipynb │ ├── 03.06.sequence.classification.Multi.RNN.ipynb │ ├── 03.07.sequence.classification.Multi.LSTM.ipynb │ ├── 03.08.sequence.classification.Multi.RNN.dropout.ipynb │ ├── 03.09.sequence.classification.Multi.LSTM.dropout.ipynb │ ├── 03.10.sequence.classification.Multi.biRNN.dropout.ipynb │ ├── 03.11.sequence.classification.Multi.biLSTM.dropout.ipynb │ ├── 04.01.seq2seq.classification.RNN.ipynb │ ├── 04.02.seq2seq.classification.GRU.ipynb │ ├── 04.03.seq2seq.classification.LSTM.ipynb │ ├── 04.04.seq2seq.classification.biRNN.ipynb │ ├── 04.05.seq2seq.classification.biLSTM.ipynb │ ├── 04.06.seq2seq.classification.Multi.RNN.ipynb │ ├── 04.07.seq2seq.classification.Multi.LSTM.ipynb │ ├── 04.08.seq2seq.classification.Multi.RNN.dropout.ipynb │ └── 04.09.seq2seq.classification.Multi.LSTM.dropout.ipynb ├── 05.generative_models │ └── 01.dcgan.ipynb ├── 06.images │ └── 01.image_segmentation.ipynb └── 07.sequences │ ├── 01.sentiment.classification.ipynb │ ├── download.py │ └── imdb.py └── tf.version.2 ├── 01.tf.basic ├── 01.tf.keras.ipynb ├── 02.guideline.eager.ipynb └── 03.tf.data.ipynb ├── 02.regression ├── 01.mnist.tf.data.eager.ipynb └── 02.mnist.model.fit.ipynb ├── 03.cnn ├── 01.mnist.LeNet5.ipynb ├── 02.mnist.LeNet5.functional.API.ipynb ├── 03.mnist.LeNet5.with.regularization.ipynb ├── 04.vgg16.ipynb └── 05.transfer.learning.ipynb ├── 04.rnn ├── 00.rnn.custom.layers.implementation.ipynb ├── 01.rnn.basic.ipynb ├── 02.simple.sequence.classification.ipynb ├── 03.sentiment.classfication.imdb.ipynb ├── 04.text.generation.ipynb └── 05.machine.translation.attention.ipynb └── 05.generative_models └── 01.dcgan.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # MAC OS 2 | .DS_Store 3 | */.DS_Store 4 | */*/.DS_Store 5 | 6 | # ipynb_checkpoints 7 | tf.version.*/*/.ipynb_checkpoints 8 | 9 | # __pycache__ 10 | tf.version.*/*/__pycache__ 11 | 12 | # graph or train folder 13 | tf.version.*/*/graphs 14 | tf.version.*/*/train 15 | tf.version.*/*/training_checkpoints 16 | 17 | # result files (jpg, png, gif) 18 | tf.version.*/*/*.jpg 19 | tf.version.*/*/*.png 20 | tf.version.*/*/*.gif 21 | 22 | # checkpoints folder 23 | checkpoints 24 | 25 | # data folder 26 | data 27 | datasets 28 | 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow Tutorials 2 | * Final update: 2019. 08. 20. 3 | * All right reserved @ Il Gu Yi 2018 4 | 5 | ## Getting Started 6 | 7 | ### Prerequisites 8 | * `TensorFlow` above 1.13 9 | * [github.com/tensorflow/models](https://github.com/tensorflow/models) for tensorflow version 1 codes 10 | * `inception_v3` and `vgg_19` pretrained models for tensorflow version 1 codes 11 | * Python 3.6 or 3.7 12 | * `numpy`, `matplotlib`, `PIL` 13 | * Jupyter notebook 14 | * OS X and Linux (Not validated on Windows but probably it would work) 15 | 16 | 17 | ## Contents (TF version 2.0 style) 18 | 19 | ### TensorFlow Basic Syntax 20 | * `tf.keras` and `eager.execution` 21 | - [01.tf.keras.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/01.tf.basic/01.tf.keras.ipynb) 22 | - [02.guideline.eager.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/01.tf.basic/02.guideline.eager.ipynb) 23 | - [03.tf.data.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/01.tf.basic/03.tf.data.ipynb) 24 | 25 | 26 | ### Regression 27 | * Logistic Regression 28 | - [01.mnist.tf.data.eager.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/02.regression/01.mnist.tf.data.eager.ipynb) 29 | - [02.mnist.model.fit.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/02.regression/02.mnist.model.fit.ipynb) 30 | 31 | 32 | ### Convolutional Neural Networks 33 | * Simple CNN model (LeNet5) and regularization & batch norm 34 | - [01.mnist.LeNet5.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/03.cnn/01.mnist.LeNet5.ipynb) 35 | - [02.mnist.LeNet5.functional.API.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/03.cnn/02.mnist.LeNet5.functional.API.ipynb) 36 | - [03.mnist.LeNet5.with.regularization.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/03.cnn/03.mnist.LeNet5.with.regularization.ipynb) 37 | * Load vgg16 and transfer learning with vgg16 38 | - [04.vgg16.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/03.cnn/04.vgg16.ipynb) 39 | - [05.transfer.learning.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/03.cnn/05.transfer.learning.ipynb) 40 | 41 | 42 | ### Recurrent Neural Networks 43 | * Many to one problem (Sentiment classification) 44 | - [01.rnn.basic.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/04.rnn/01.rnn.basic.ipynb) 45 | - [02.simple.sequence.classification.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/04.rnn/02.simple.sequence.classification.ipynb) 46 | - [03.sentiment.classfication.imdb.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/04.rnn/03.sentiment.classfication.imdb.ipynb) 47 | - [04.text.generation.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/04.rnn/04.text.generation.ipynb) 48 | 49 | 50 | ### Generative Models 51 | * Generative Adversarial Networks 52 | - [01.dcgan.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.2/05.generative_models/01.dcgan.ipynb) 53 | 54 | 55 | 56 | 57 | ## Contents (TF version 1.x style) 58 | 59 | ### TensorFlow Basic Syntax 60 | * Overview and Operations 61 | - [01.hello.tf.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/01.hello.tf.ipynb) 62 | - [02.tf.eager.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/02.tf.eager.ipynb) 63 | - [03.tensorboard.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/03.tensorboard.ipynb) 64 | - [04.tf.dimension.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/04.tf.dimension.ipynb) 65 | - [05.tf.Variable.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/05.tf.Variable.ipynb) 66 | - [06.tf.placeholder.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/06.tf.placeholder.ipynb) 67 | * Managing and `tf.data` 68 | - [07.tf.train.Saver.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/07.tf.train.Saver.ipynb) 69 | - [08.tf.cond.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/08.tf.cond.ipynb) 70 | - [09.tf.control_dependencies.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/09.tf.control_dependencies.ipynb) 71 | - [10.tf.data.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/01.tf.basic/10.tf.data.ipynb) 72 | 73 | 74 | ### Regression 75 | * Linear Regression 76 | - [01.1.linear.regression.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/02.regression/01.1.linear.regression.ipynb) 77 | - [01.2.linear.regression.with.minibatch.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/02.regression/01.2.linear.regression.with.minibatch.ipynb) 78 | - [02.1.linear.regression.3rd.order.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/02.regression/02.1.linear.regression.3rd.order.ipynb) 79 | - [02.2.linear.regression.3rd.order.with.minibatch.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/02.regression/02.2.linear.regression.3rd.order.with.minibatch.ipynb) 80 | * Logistic Regression 81 | - [03.1.mnist.softmax.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/02.regression/03.1.mnist.softmax.ipynb) 82 | - [03.2.mnist.softmax.with.tf.data.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/02.regression/03.2.mnist.softmax.with.tf.data.ipynb) 83 | - [04.mnist.summary.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/02.regression/04.mnist.summary.ipynb) 84 | 85 | 86 | ### Convolutional Neural Networks 87 | * Simple CNN model (LeNet5) and regularization & batch norm 88 | - [01.1.mnist.deep.with.estimator.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/01.1.mnist.deep.with.estimator.ipynb) 89 | - [01.2.mnist.deep.with.tf.data.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/01.2.mnist.deep.with.tf.data.ipynb) 90 | - [02.mnist.deep.slim.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/02.mnist.deep.slim.ipynb) 91 | - [03.mnist.slim.options.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/03.mnist.slim.options.ipynb) 92 | - [04.mnist.slim.arg.scope.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/04.mnist.slim.arg.scope.ipynb) 93 | * Advanced CNN model (Cifar10) and data augmentation 94 | - [05.cnn.cifar10.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/05.cnn.cifar10.ipynb) 95 | - [06.cnn.cifar10.regularization.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/06.cnn.cifar10.regularization.ipynb) 96 | - [07.cnn.cifar10.data.augmentation.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/07.cnn.cifar10.data.augmentation.ipynb) 97 | * Pretrained CNN models 98 | - [08.vgg19.slim.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/08.vgg19.slim.ipynb) 99 | - [09.inception_v3.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/09.inception_v3.ipynb) 100 | * Transfer learning and `tfrecords` format 101 | - [10.tfrecords.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/10.tfrecords.ipynb) 102 | - [11.transfer.learning.with.inception_v3.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/03.cnn/11.transfer.learning.with.inception_v3.ipynb) 103 | 104 | 105 | ### Recurrent Neural Networks 106 | * Usage for sequence data 107 | - [01.ready.for.sequence.data.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/01.ready.for.sequence.data.ipynb) 108 | - [02.rnn.basic.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/02.rnn.basic.ipynb) 109 | * Sequence classification (many to one classification) 110 | - [03.01.sequence.classification.RNN.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.01.sequence.classification.RNN.ipynb) 111 | - [03.02.sequence.classification.GRU.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.02.sequence.classification.GRU.ipynb) 112 | - [03.03.sequence.classification.LSTM.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.03.sequence.classification.LSTM.ipynb) 113 | - [03.04.sequence.classification.biRNN.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.04.sequence.classification.biRNN.ipynb) 114 | - [03.05.sequence.classification.biLSTM.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.05.sequence.classification.biLSTM.ipynb) 115 | - [03.06.sequence.classification.Multi.RNN.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.06.sequence.classification.Multi.RNN.ipynb) 116 | - [03.07.sequence.classification.Multi.LSTM.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.07.sequence.classification.Multi.LSTM.ipynb) 117 | - [03.08.sequence.classification.Multi.RNN.dropout.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.08.sequence.classification.Multi.RNN.dropout.ipynb) 118 | - [03.09.sequence.classification.Multi.LSTM.dropout.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.09.sequence.classification.Multi.LSTM.dropout.ipynb) 119 | - [03.10.sequence.classification.Multi.biRNN.dropout.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.10.sequence.classification.Multi.biRNN.dropout.ipynb) 120 | - [03.11.sequence.classification.Multi.biLSTM.dropout.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/03.11.sequence.classification.Multi.biLSTM.dropout.ipynb) 121 | * Sequence to sequence classification (many to many classification) 122 | - [04.01.seq2seq.classification.RNN.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.01.seq2seq.classification.RNN.ipynb) 123 | - [04.02.seq2seq.classification.GRU.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.02.seq2seq.classification.GRU.ipynb) 124 | - [04.03.seq2seq.classification.LSTM.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.03.seq2seq.classification.LSTM.ipynb) 125 | - [04.04.seq2seq.classification.biRNN.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.04.seq2seq.classification.biRNN.ipynb) 126 | - [04.05.seq2seq.classification.biLSTM.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.05.seq2seq.classification.biLSTM.ipynb) 127 | - [04.06.seq2seq.classification.Multi.RNN.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.06.seq2seq.classification.Multi.RNN.ipynb) 128 | - [04.07.seq2seq.classification.Multi.LSTM.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.07.seq2seq.classification.Multi.LSTM.ipynb) 129 | - [04.08.seq2seq.classification.Multi.RNN.dropout.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.08.seq2seq.classification.Multi.RNN.dropout.ipynb) 130 | - [04.09.seq2seq.classification.Multi.LSTM.dropout.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/04.rnn/04.09.seq2seq.classification.Multi.LSTM.dropout.ipynb) 131 | 132 | 133 | ### Generative Models 134 | * Generative Adversarial Networks 135 | - [01.dcgan.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/05.generative_models/01.dcgan.ipynb) 136 | 137 | ### Images 138 | * Image Segmentation with U-Net 139 | - [01.image_segmentation.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/06.images/01.image_segmentation.ipynb) 140 | 141 | ### Sequences 142 | * Sentiment Classification 143 | - [01.sentiment.classification.ipynb](https://nbviewer.jupyter.org/github/ilguyi/tensorflow.tutorials/tree/master/tf.version.1/07.sequences/01.sentiment.classification.ipynb) 144 | 145 | 146 | ## Author 147 | Il Gu Yi 148 | 149 | ### Lecture Notes 150 | [link](https://www.notion.so/with-TF-60e03aa6dc8542339fe5de89ebee853b) 151 | -------------------------------------------------------------------------------- /input_data/cat1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ilguyi/tensorflow.tutorials/67a80aef55a0d240ba1a4fd37e9ff9774ff55eb0/input_data/cat1.jpg -------------------------------------------------------------------------------- /input_data/cat2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ilguyi/tensorflow.tutorials/67a80aef55a0d240ba1a4fd37e9ff9774ff55eb0/input_data/cat2.jpg -------------------------------------------------------------------------------- /tf.version.1/01.tf.basic/02.tf.eager.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# TensorFlow Eager Execution" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "name": "stdout", 17 | "output_type": "stream", 18 | "text": [ 19 | "TensorFlow version: 1.11.0\n", 20 | "Eager execution: True\n" 21 | ] 22 | } 23 | ], 24 | "source": [ 25 | "from __future__ import absolute_import\n", 26 | "from __future__ import division\n", 27 | "from __future__ import print_function\n", 28 | "\n", 29 | "import tensorflow as tf\n", 30 | "import tensorflow.contrib.eager as tfe\n", 31 | "\n", 32 | "tf.enable_eager_execution()\n", 33 | "\n", 34 | "print(\"TensorFlow version: {}\".format(tf.VERSION))\n", 35 | "print(\"Eager execution: {}\".format(tf.executing_eagerly()))" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": {}, 41 | "source": [ 42 | "## Execution Style\n", 43 | "\n", 44 | "* [Eager execution](https://www.tensorflow.org/get_started/eager)\n", 45 | " * 최근에 나온 방식\n", 46 | " * 변수값을 바로 확인할 수 있음\n", 47 | " * 마치 `numpy` 짜듯이 짤 수 있음\n", 48 | "* [Graph execution](https://www.tensorflow.org/get_started/get_started_for_beginners)\n", 49 | " * TensorFlow 초창기 구현 방법\n", 50 | " * 여전히 많이 이용하고 있음\n", 51 | " * Graph 선언부분과 실행(Session)하는 부분이 분리되어 있음" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "## Eager Execution examples" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 2, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "a = tf.add(3, 5)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 3, 73 | "metadata": {}, 74 | "outputs": [ 75 | { 76 | "name": "stdout", 77 | "output_type": "stream", 78 | "text": [ 79 | "tf.Tensor(8, shape=(), dtype=int32)\n" 80 | ] 81 | } 82 | ], 83 | "source": [ 84 | "print(a)" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "### 약간 더 복잡한 계산" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 4, 97 | "metadata": {}, 98 | "outputs": [ 99 | { 100 | "name": "stdout", 101 | "output_type": "stream", 102 | "text": [ 103 | "x: 2\n", 104 | "y: 3\n", 105 | "w: tf.Tensor(5, shape=(), dtype=int32)\n", 106 | "z: tf.Tensor(6, shape=(), dtype=int32)\n", 107 | "p: tf.Tensor(7776, shape=(), dtype=int32)\n" 108 | ] 109 | } 110 | ], 111 | "source": [ 112 | "x = 2\n", 113 | "y = 3\n", 114 | "w = tf.add(x, y)\n", 115 | "z = tf.multiply(x, y)\n", 116 | "p = tf.pow(z, w)\n", 117 | "print(\"x: \", x)\n", 118 | "print(\"y: \", y)\n", 119 | "print(\"w: \", w)\n", 120 | "print(\"z: \", z)\n", 121 | "print(\"p: \", p)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "### `Tensor` 변수와 일반 `python` 변수" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 5, 134 | "metadata": {}, 135 | "outputs": [ 136 | { 137 | "name": "stdout", 138 | "output_type": "stream", 139 | "text": [ 140 | "x: tf.Tensor(2, shape=(), dtype=int32)\n", 141 | "y: tf.Tensor(3, shape=(), dtype=int32)\n", 142 | "w: tf.Tensor(5, shape=(), dtype=int32)\n", 143 | "z: tf.Tensor(6, shape=(), dtype=int32)\n", 144 | "p: tf.Tensor(7776, shape=(), dtype=int32)\n" 145 | ] 146 | } 147 | ], 148 | "source": [ 149 | "x = tf.constant(2)\n", 150 | "y = tf.constant(3)\n", 151 | "w = x + y\n", 152 | "z = x * y\n", 153 | "p = tf.pow(z, w)\n", 154 | "print(\"x: \", x)\n", 155 | "print(\"y: \", y)\n", 156 | "print(\"w: \", w)\n", 157 | "print(\"z: \", z)\n", 158 | "print(\"p: \", p)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "### Eager Execution another examples" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 6, 171 | "metadata": {}, 172 | "outputs": [ 173 | { 174 | "name": "stdout", 175 | "output_type": "stream", 176 | "text": [ 177 | "hello, [[4.]]\n" 178 | ] 179 | } 180 | ], 181 | "source": [ 182 | "tf.executing_eagerly() # => True\n", 183 | "\n", 184 | "x = [[2.]]\n", 185 | "m = tf.matmul(x, x)\n", 186 | "print(\"hello, {}\".format(m)) # => \"hello, [[4.]]\"" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 7, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "name": "stdout", 196 | "output_type": "stream", 197 | "text": [ 198 | "tf.Tensor(\n", 199 | "[[1 2]\n", 200 | " [3 4]], shape=(2, 2), dtype=int32)\n" 201 | ] 202 | } 203 | ], 204 | "source": [ 205 | "a = tf.constant([[1, 2],\n", 206 | " [3, 4]])\n", 207 | "print(a)\n", 208 | "# => tf.Tensor([[1 2]\n", 209 | "# [3 4]], shape=(2, 2), dtype=int32)" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": 8, 215 | "metadata": {}, 216 | "outputs": [ 217 | { 218 | "name": "stdout", 219 | "output_type": "stream", 220 | "text": [ 221 | "tf.Tensor(\n", 222 | "[[2 3]\n", 223 | " [4 5]], shape=(2, 2), dtype=int32)\n" 224 | ] 225 | } 226 | ], 227 | "source": [ 228 | "# Broadcasting support\n", 229 | "b = tf.add(a, 1)\n", 230 | "print(b)\n", 231 | "# => tf.Tensor([[2 3]\n", 232 | "# [4 5]], shape=(2, 2), dtype=int32)" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": 9, 238 | "metadata": {}, 239 | "outputs": [ 240 | { 241 | "name": "stdout", 242 | "output_type": "stream", 243 | "text": [ 244 | "tf.Tensor(\n", 245 | "[[ 2 6]\n", 246 | " [12 20]], shape=(2, 2), dtype=int32)\n" 247 | ] 248 | } 249 | ], 250 | "source": [ 251 | "# Operator overloading is supported\n", 252 | "print(a * b)\n", 253 | "# => tf.Tensor([[ 2 6]\n", 254 | "# [12 20]], shape=(2, 2), dtype=int32)" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 10, 260 | "metadata": {}, 261 | "outputs": [ 262 | { 263 | "name": "stdout", 264 | "output_type": "stream", 265 | "text": [ 266 | "[[ 2 6]\n", 267 | " [12 20]]\n" 268 | ] 269 | } 270 | ], 271 | "source": [ 272 | "# Use NumPy functions\n", 273 | "import numpy as np\n", 274 | "\n", 275 | "c = np.multiply(a, b)\n", 276 | "print(c)\n", 277 | "# => [[ 2 6]\n", 278 | "# [12 20]]" 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": {}, 284 | "source": [ 285 | "### `tf.Tensor.numpy()`\n", 286 | "\n", 287 | "* returns the object's value as a NumPy ndarray." 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": 11, 293 | "metadata": {}, 294 | "outputs": [ 295 | { 296 | "name": "stdout", 297 | "output_type": "stream", 298 | "text": [ 299 | "[[1 2]\n", 300 | " [3 4]]\n" 301 | ] 302 | } 303 | ], 304 | "source": [ 305 | "# Obtain numpy value from a tensor:\n", 306 | "print(a.numpy())\n", 307 | "# => [[1 2]\n", 308 | "# [3 4]]" 309 | ] 310 | } 311 | ], 312 | "metadata": { 313 | "kernelspec": { 314 | "display_name": "Python 3", 315 | "language": "python", 316 | "name": "python3" 317 | }, 318 | "language_info": { 319 | "codemirror_mode": { 320 | "name": "ipython", 321 | "version": 3 322 | }, 323 | "file_extension": ".py", 324 | "mimetype": "text/x-python", 325 | "name": "python", 326 | "nbconvert_exporter": "python", 327 | "pygments_lexer": "ipython3", 328 | "version": "3.6.5" 329 | } 330 | }, 331 | "nbformat": 4, 332 | "nbformat_minor": 2 333 | } 334 | -------------------------------------------------------------------------------- /tf.version.1/01.tf.basic/03.tensorboard.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Tensorboard 사용법" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "name": "stdout", 17 | "output_type": "stream", 18 | "text": [ 19 | "TensorFlow version: 1.11.0\n" 20 | ] 21 | } 22 | ], 23 | "source": [ 24 | "from __future__ import absolute_import\n", 25 | "from __future__ import division\n", 26 | "from __future__ import print_function\n", 27 | "\n", 28 | "import tensorflow as tf\n", 29 | "\n", 30 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))\n", 31 | "\n", 32 | "print(\"TensorFlow version: {}\".format(tf.VERSION))" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "### `tf.summary.FileWriter`를 사용하여 그래프를 그림\n", 40 | "\n", 41 | "* [`tf.summary.FileWriter`](https://www.tensorflow.org/api_docs/python/tf/summary/FileWriter)\n", 42 | "* 그래프를 확인할 때는 아래의 명령어를 입력함\n", 43 | "\n", 44 | "```shell\n", 45 | "$ tensorboard --logdir path\n", 46 | "```\n", 47 | "\n", 48 | "여기서는\n", 49 | "```shell\n", 50 | "$ tensorboard --logdir graphs\n", 51 | "```\n", 52 | "\n", 53 | "* 그리고 웹브라우저를 열어 `localhost:6006` (주로 `127.0.0.1:6006`)을 입력함\n", 54 | "* 포트번호 `6006`은 goog(le) 을 뒤집은 것임." 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 2, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "name": "stdout", 64 | "output_type": "stream", 65 | "text": [ 66 | "5\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "a = tf.constant(2)\n", 72 | "b = tf.constant(3)\n", 73 | "x = tf.add(a, b)\n", 74 | "\n", 75 | "with tf.Session(config=sess_config) as sess:\n", 76 | " # add this line to use TensorBoard.\n", 77 | " writer = tf.summary.FileWriter(\"./graphs/03.tensorboard.1\", sess.graph)\n", 78 | " print(sess.run(x))\n", 79 | " writer.close() # close the writer when you’re done using it" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "### 같은 내용을 한번 더 실행해보자\n", 87 | "\n", 88 | "* 실행 후 `tensorboard`를 열어서 그래프 모양을 확인해보자" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 3, 94 | "metadata": {}, 95 | "outputs": [ 96 | { 97 | "name": "stdout", 98 | "output_type": "stream", 99 | "text": [ 100 | "5\n" 101 | ] 102 | } 103 | ], 104 | "source": [ 105 | "a = tf.constant(2)\n", 106 | "b = tf.constant(3)\n", 107 | "x = tf.add(a, b)\n", 108 | "\n", 109 | "with tf.Session(config=sess_config) as sess:\n", 110 | " # add this line to use TensorBoard.\n", 111 | " writer = tf.summary.FileWriter(\"./graphs/03.tensorboard.2\", sess.graph)\n", 112 | " print(sess.run(x))\n", 113 | " writer.close() # close the writer when you’re done using it" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "### 같은 내용을 한번 더 실행해보자\n", 121 | "\n", 122 | "* 이번엔 위에 만들었던 그래프들을 지우고 다시 그래프를 그려보자\n", 123 | "* 실행 후 `tensorboard`를 열어서 그래프 모양을 확인해보자" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 4, 129 | "metadata": {}, 130 | "outputs": [ 131 | { 132 | "name": "stdout", 133 | "output_type": "stream", 134 | "text": [ 135 | "5\n" 136 | ] 137 | } 138 | ], 139 | "source": [ 140 | "# Only necessary if you use IDLE or a jupyter notebook\n", 141 | "tf.reset_default_graph()\n", 142 | "\n", 143 | "a = tf.constant(2)\n", 144 | "b = tf.constant(3)\n", 145 | "x = tf.add(a, b)\n", 146 | "\n", 147 | "with tf.Session(config=sess_config) as sess:\n", 148 | " # add this line to use TensorBoard.\n", 149 | " writer = tf.summary.FileWriter(\"./graphs/03.tensorboard.3\", sess.graph)\n", 150 | " print(sess.run(x))\n", 151 | " writer.close() # close the writer when you’re done using it" 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "### Explicitly Name\n", 159 | "\n", 160 | "* 명시적으로 변수에 이름을 정하지 않으면 tensorflow 내부적으로 tensorflow 기본이름을 붙여준다.\n", 161 | " * `Const`, `Const_1`, `Const_2`, 이런식으로 같은 타입의 변수들은 자동으로 숫자가 붙는다." 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 5, 167 | "metadata": {}, 168 | "outputs": [ 169 | { 170 | "name": "stdout", 171 | "output_type": "stream", 172 | "text": [ 173 | "5\n" 174 | ] 175 | } 176 | ], 177 | "source": [ 178 | "# Only necessary if you use IDLE or a jupyter notebook\n", 179 | "tf.reset_default_graph()\n", 180 | "\n", 181 | "a = tf.constant(2, name='a')\n", 182 | "b = tf.constant(3, name='b')\n", 183 | "x = tf.add(a, b, name='add')\n", 184 | "with tf.Session() as sess:\n", 185 | " # add this line to use TensorBoard.\n", 186 | " writer = tf.summary.FileWriter(\"./graphs/03.tensorboard.4\", sess.graph)\n", 187 | " print(sess.run(x))\n", 188 | " writer.close() # close the writer when you’re done using it" 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": {}, 194 | "source": [ 195 | "## 직접 실습" 196 | ] 197 | }, 198 | { 199 | "cell_type": "markdown", 200 | "metadata": {}, 201 | "source": [ 202 | "### 본인이 직접 연산을 구현하고 Tensorboard로 확인 할 것" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": null, 208 | "metadata": {}, 209 | "outputs": [], 210 | "source": [ 211 | "# TODO" 212 | ] 213 | } 214 | ], 215 | "metadata": { 216 | "kernelspec": { 217 | "display_name": "Python 3", 218 | "language": "python", 219 | "name": "python3" 220 | }, 221 | "language_info": { 222 | "codemirror_mode": { 223 | "name": "ipython", 224 | "version": 3 225 | }, 226 | "file_extension": ".py", 227 | "mimetype": "text/x-python", 228 | "name": "python", 229 | "nbconvert_exporter": "python", 230 | "pygments_lexer": "ipython3", 231 | "version": "3.6.5" 232 | } 233 | }, 234 | "nbformat": 4, 235 | "nbformat_minor": 2 236 | } 237 | -------------------------------------------------------------------------------- /tf.version.1/01.tf.basic/07.tf.train.Saver.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# [`tf.train.Saver`](https://www.tensorflow.org/api_docs/python/tf/train/Saver)\n", 8 | "\n", 9 | "* 변수를 저장하거나 불러올때 쓴다" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "from __future__ import absolute_import\n", 19 | "from __future__ import division\n", 20 | "from __future__ import print_function\n", 21 | "\n", 22 | "import tensorflow as tf\n", 23 | "\n", 24 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "### 변수선언" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "a = tf.Variable(5)\n", 41 | "b = tf.Variable(4, name=\"my_variable\")\n", 42 | "x = tf.add(a, b, name=\"add\")\n", 43 | "\n", 44 | "# set the value of a to 3\n", 45 | "op = tf.assign(a, 3) " 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "### `saver`를 이용한 변수 값 저장" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": {}, 59 | "outputs": [ 60 | { 61 | "name": "stdout", 62 | "output_type": "stream", 63 | "text": [ 64 | "a: 3\n", 65 | "my_variable: 4\n", 66 | "Variable:0\n", 67 | "my_variable:0\n" 68 | ] 69 | } 70 | ], 71 | "source": [ 72 | "# create saver object\n", 73 | "saver = tf.train.Saver()\n", 74 | "\n", 75 | "with tf.Session(config=sess_config) as sess:\n", 76 | " sess.run(tf.global_variables_initializer())\n", 77 | "\n", 78 | " sess.run(op)\n", 79 | "\n", 80 | " print (\"a:\", sess.run(a))\n", 81 | " print (\"my_variable:\", sess.run(b))\n", 82 | "\n", 83 | " # use saver object to save variables\n", 84 | " # within the context of the current session \n", 85 | " saver.save(sess, \"graphs/07.tf.train_Saver/my_model.ckpt\")\n", 86 | " \n", 87 | " print(a.name)\n", 88 | " print(b.name)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "### `saver`를 이용하여 모델 restore 하기\n", 96 | "\n", 97 | "* `saver.save`는 그래프 자체를 저장하지 않는다.\n", 98 | "* 변수의 값만 저장할 뿐이다.\n", 99 | "* 따라서 `saver.restore`를 하기전에 그래프 구조를 만들어줘야 한다.\n", 100 | "* 위의 예제에서 save할 때는 `a`와 `b`를 저장하였으나 로드 할때는 `c`와 `d`를 만들어서 로드한다.\n", 101 | "* 중요한 것은 변수의 tensorflow로 지정한 이름이다.\n", 102 | " * python 변수 이름 형태인 `a`, `b`, `c`, `d`가 아니다.\n", 103 | "* `name=my_variable`형태로 저장된 변수의 값을 불러와서 새로운 `c`와 `d` 라는 변수에 넣었다.\n", 104 | " * 저장할 때 `a`와 `b`라는 변수 이름은 중요하지 않다." 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 4, 110 | "metadata": {}, 111 | "outputs": [ 112 | { 113 | "name": "stdout", 114 | "output_type": "stream", 115 | "text": [ 116 | "INFO:tensorflow:Restoring parameters from graphs/07.tf.train_Saver/my_model.ckpt\n", 117 | "c: 3\n", 118 | "my_variable: 4\n", 119 | "Variable:0\n", 120 | "my_variable:0\n" 121 | ] 122 | } 123 | ], 124 | "source": [ 125 | "# Only necessary if you use IDLE or a jupyter notebook\n", 126 | "tf.reset_default_graph()\n", 127 | "\n", 128 | "# make a dummy variable\n", 129 | "# the value is arbitrary, here just zero\n", 130 | "# but the shape must the the same as in the saved model\n", 131 | "c = tf.Variable(5)\n", 132 | "d = tf.Variable(0, name=\"my_variable\")\n", 133 | "y = tf.add(c, d, name='add')\n", 134 | "\n", 135 | "saver = tf.train.Saver()\n", 136 | "\n", 137 | "with tf.Session(config=sess_config) as sess:\n", 138 | "\n", 139 | " # use saver object to load variables from the saved model\n", 140 | " saver.restore(sess, \"graphs/07.tf.train_Saver/my_model.ckpt\")\n", 141 | "\n", 142 | " print (\"c:\", sess.run(c))\n", 143 | " print (\"my_variable:\", sess.run(d))\n", 144 | " \n", 145 | " print(c.name)\n", 146 | " print(d.name)" 147 | ] 148 | } 149 | ], 150 | "metadata": { 151 | "kernelspec": { 152 | "display_name": "Python 3", 153 | "language": "python", 154 | "name": "python3" 155 | }, 156 | "language_info": { 157 | "codemirror_mode": { 158 | "name": "ipython", 159 | "version": 3 160 | }, 161 | "file_extension": ".py", 162 | "mimetype": "text/x-python", 163 | "name": "python", 164 | "nbconvert_exporter": "python", 165 | "pygments_lexer": "ipython3", 166 | "version": "3.6.5" 167 | } 168 | }, 169 | "nbformat": 4, 170 | "nbformat_minor": 2 171 | } 172 | -------------------------------------------------------------------------------- /tf.version.1/01.tf.basic/08.tf.cond.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# [`tf.cond`](https://www.tensorflow.org/api_docs/python/tf/cond)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "from __future__ import absolute_import\n", 17 | "from __future__ import division\n", 18 | "from __future__ import print_function\n", 19 | "\n", 20 | "import numpy as np\n", 21 | "import tensorflow as tf\n", 22 | "\n", 23 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "## A frequent mistake" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 2, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "np.random.seed(219)\n", 40 | "\n", 41 | "def what_is_x():\n", 42 | " if np.random.rand() < 0.5:\n", 43 | " x = tf.constant(10)\n", 44 | " else:\n", 45 | " x = tf.constant(20)\n", 46 | " \n", 47 | " return x" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "### Nomral loading" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 3, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "name": "stdout", 64 | "output_type": "stream", 65 | "text": [ 66 | "10\n", 67 | "10\n", 68 | "10\n", 69 | "10\n", 70 | "10\n", 71 | "10\n", 72 | "10\n", 73 | "10\n", 74 | "10\n", 75 | "10\n" 76 | ] 77 | } 78 | ], 79 | "source": [ 80 | "tf.reset_default_graph()\n", 81 | "\n", 82 | "x = what_is_x()\n", 83 | "with tf.Session(config=sess_config) as sess:\n", 84 | " for i in range(10):\n", 85 | " print(sess.run(x))" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "### Lazy loading" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 4, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "name": "stdout", 102 | "output_type": "stream", 103 | "text": [ 104 | "10\n", 105 | "10\n", 106 | "10\n", 107 | "20\n", 108 | "20\n", 109 | "10\n", 110 | "20\n", 111 | "20\n", 112 | "20\n", 113 | "20\n" 114 | ] 115 | } 116 | ], 117 | "source": [ 118 | "tf.reset_default_graph()\n", 119 | "tf.set_random_seed(219)\n", 120 | "\n", 121 | "with tf.Session(config=sess_config) as sess:\n", 122 | " for i in range(10):\n", 123 | " print(sess.run(what_is_x()))" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "## How to solve" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 5, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "def what_is_x2():\n", 140 | " def f1(): return tf.constant(10)\n", 141 | " def f2(): return tf.constant(20)\n", 142 | " x = tf.cond(tf.less(tf.random_uniform([]), 0.5), f1, f2)\n", 143 | " \n", 144 | " return x" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 6, 150 | "metadata": {}, 151 | "outputs": [ 152 | { 153 | "name": "stdout", 154 | "output_type": "stream", 155 | "text": [ 156 | "20\n", 157 | "10\n", 158 | "20\n", 159 | "10\n", 160 | "20\n", 161 | "10\n", 162 | "10\n", 163 | "10\n", 164 | "20\n", 165 | "10\n" 166 | ] 167 | } 168 | ], 169 | "source": [ 170 | "tf.reset_default_graph()\n", 171 | "tf.set_random_seed(219)\n", 172 | "\n", 173 | "x = what_is_x2()\n", 174 | "with tf.Session(config=sess_config) as sess:\n", 175 | " for i in range(10):\n", 176 | " print(sess.run(x))" 177 | ] 178 | } 179 | ], 180 | "metadata": { 181 | "kernelspec": { 182 | "display_name": "Python 3", 183 | "language": "python", 184 | "name": "python3" 185 | }, 186 | "language_info": { 187 | "codemirror_mode": { 188 | "name": "ipython", 189 | "version": 3 190 | }, 191 | "file_extension": ".py", 192 | "mimetype": "text/x-python", 193 | "name": "python", 194 | "nbconvert_exporter": "python", 195 | "pygments_lexer": "ipython3", 196 | "version": "3.6.5" 197 | } 198 | }, 199 | "nbformat": 4, 200 | "nbformat_minor": 2 201 | } 202 | -------------------------------------------------------------------------------- /tf.version.1/01.tf.basic/09.tf.control_dependencies.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# [`tf.control_dependencies`](https://www.tensorflow.org/api_docs/python/tf/control_dependencies)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "from __future__ import absolute_import\n", 17 | "from __future__ import division\n", 18 | "from __future__ import print_function\n", 19 | "\n", 20 | "import numpy as np\n", 21 | "import tensorflow as tf\n", 22 | "\n", 23 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "## `tf.control_dependencies`\n", 31 | "\n", 32 | "* 뜻: 이거 하기 전에 저거 먼저 해라\n", 33 | "* [API 링크 1](https://www.tensorflow.org/api_docs/python/tf/control_dependencies)\n", 34 | "* [API 링크 2](https://www.tensorflow.org/api_docs/python/tf/Graph#control_dependencies)\n", 35 | "\n", 36 | "```python\n", 37 | "with tf.control_dependencies([a, b, c]):\n", 38 | " # `d` and `e` will only run after `a`, `b`, and `c` have executed.\n", 39 | " d = ...\n", 40 | " e = ...\n", 41 | "```\n", 42 | "\n", 43 | "* `a`, `b`, `c`를 먼저 하고 `d`, `e`를 해라\n", 44 | "* `batch_normalization`을 할 때 이용함" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "### Example" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 2, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "name": "stdout", 61 | "output_type": "stream", 62 | "text": [ 63 | "a: 5\n", 64 | "b: [4 8]\n" 65 | ] 66 | } 67 | ], 68 | "source": [ 69 | "tf.reset_default_graph()\n", 70 | "\n", 71 | "a = tf.get_variable('scalar', initializer=tf.constant(2))\n", 72 | "b = tf.get_variable('vector', initializer=tf.constant([2, 4]))\n", 73 | "a_op = a.assign_add(3)\n", 74 | "\n", 75 | "with tf.control_dependencies([a_op]):\n", 76 | " # `b` 하기 전에 `a` 먼저 해라\n", 77 | " b_op = b.assign(b * 2)\n", 78 | "\n", 79 | "with tf.Session(config=sess_config) as sess:\n", 80 | " writer = tf.summary.FileWriter(\"./graphs/09.tf.control_dependencies.1\", sess.graph)\n", 81 | " writer.close()\n", 82 | "\n", 83 | " sess.run(tf.global_variables_initializer())\n", 84 | " sess.run(b_op)\n", 85 | " \n", 86 | " a_out, b_out = sess.run([a, b])\n", 87 | " print('a: ', a_out)\n", 88 | " print('b: ', b_out)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "### Do not use `tf.control_dependencies`" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 3, 101 | "metadata": {}, 102 | "outputs": [ 103 | { 104 | "name": "stdout", 105 | "output_type": "stream", 106 | "text": [ 107 | "a: 2\n", 108 | "b: [4 8]\n" 109 | ] 110 | } 111 | ], 112 | "source": [ 113 | "tf.reset_default_graph()\n", 114 | "\n", 115 | "a = tf.get_variable('scalar', initializer=tf.constant(2))\n", 116 | "b = tf.get_variable('vector', initializer=tf.constant([2, 4]))\n", 117 | "a_op = a.assign_add(3)\n", 118 | "\n", 119 | "#with tf.control_dependencies([a_op]):\n", 120 | "# `a`와 상관없이 그냥 `b`만 계산하기\n", 121 | "b_op = b.assign(b * 2)\n", 122 | "\n", 123 | "with tf.Session(config=sess_config) as sess:\n", 124 | " writer = tf.summary.FileWriter(\"./graphs/09.tf.control_dependencies.2\", sess.graph)\n", 125 | " writer.close()\n", 126 | "\n", 127 | " sess.run(tf.global_variables_initializer())\n", 128 | " sess.run(b_op)\n", 129 | " a_out, b_out = sess.run([a, b])\n", 130 | " print('a: ', a_out)\n", 131 | " print('b: ', b_out)" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "metadata": {}, 137 | "source": [ 138 | "### Tensorboard 확인\n", 139 | "\n", 140 | "* 직접 tensorboard 열고 두 그래프를 비교" 141 | ] 142 | } 143 | ], 144 | "metadata": { 145 | "kernelspec": { 146 | "display_name": "Python 3", 147 | "language": "python", 148 | "name": "python3" 149 | }, 150 | "language_info": { 151 | "codemirror_mode": { 152 | "name": "ipython", 153 | "version": 3 154 | }, 155 | "file_extension": ".py", 156 | "mimetype": "text/x-python", 157 | "name": "python", 158 | "nbconvert_exporter": "python", 159 | "pygments_lexer": "ipython3", 160 | "version": "3.6.5" 161 | } 162 | }, 163 | "nbformat": 4, 164 | "nbformat_minor": 2 165 | } 166 | -------------------------------------------------------------------------------- /tf.version.1/04.rnn/01.ready.for.sequence.data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# RNN for text and sequences\n", 8 | "\n", 9 | "### Reference\n", 10 | "* Deep Learning with Python, F. Chollet\n", 11 | "* All figures are borrowed from above book\n", 12 | "\n", 13 | "### Preprocessing\n", 14 | "* Preprocessing text data into useful representations\n", 15 | "* Working with RNN\n", 16 | "\n", 17 | "### Applications of RNN\n", 18 | "\n", 19 | "* Document classification and timeseries classification, such as identifying the topic of an article or the author of a book\n", 20 | "* Timeseries comparisons, such as estimating how closely related two documents or two stock tickers are\n", 21 | "* Sequence-to-sequence learning, such as decoding an English sentence into French\n", 22 | "* Sentiment analysis, such as classifying the sentiment of tweets or movie reviews as positive or negative\n", 23 | "* Timeseries forecasting, such as predicting the future weather at a certain location, given recent weather data" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "### *Vetorizing* text\n", 31 | "\n", 32 | "* Examples\n", 33 | " * Segment text into words, and transform each word into a vector.\n", 34 | " * Segment text into characters, and transform each character into a vector.\n", 35 | "* **Tokenization**: segment text into words or charaters\n", 36 | "* **Embedding**: transform them into a vectors\n", 37 | " * *one-hot* encoding (sparse vector)\n", 38 | " * **embedding** (dense vector)\n", 39 | "\n", 40 | "Tokenization\n", 41 | "\"tokenization\"\n", 42 | "\n", 43 | "Word embedding\n", 44 | "\"word_embedding\"\n", 45 | "\n", 46 | "\n", 47 | "* Two ways of word embedding\n", 48 | " * Learned from data\n", 49 | " * import pre-trained word vector (like word2vec)\n", 50 | "\n", 51 | "```python\n", 52 | "# TensorFlow code (one-hot embedding)\n", 53 | "one_hot_matrix = tf.get_variable(name='one_hot_embedding',\n", 54 | " initializer=tf.eye(num_words, dtype=tf.float32),\n", 55 | " trainable=False) \n", 56 | "embeddings = tf.nn.embedding_lookup(params=one_hot_matrix, ids=train_seq)\n", 57 | "\n", 58 | "# TensorFlow code (dense vector embedding)\n", 59 | "embedding_matrix = tf.get_variable(name='embedding',\n", 60 | " shape=[num_words, embedding_size],\n", 61 | " initializer=tf.random_uniform_initializer(minval=-0.1,\n", 62 | " maxval=0.1))\n", 63 | "embeddings = tf.nn.embedding_lookup(params=embedding_matrix, ids=train_seq)\n", 64 | "\n", 65 | "```" 66 | ] 67 | } 68 | ], 69 | "metadata": { 70 | "kernelspec": { 71 | "display_name": "Python 3", 72 | "language": "python", 73 | "name": "python3" 74 | }, 75 | "language_info": { 76 | "codemirror_mode": { 77 | "name": "ipython", 78 | "version": 3 79 | }, 80 | "file_extension": ".py", 81 | "mimetype": "text/x-python", 82 | "name": "python", 83 | "nbconvert_exporter": "python", 84 | "pygments_lexer": "ipython3", 85 | "version": "3.6.5" 86 | } 87 | }, 88 | "nbformat": 4, 89 | "nbformat_minor": 2 90 | } 91 | -------------------------------------------------------------------------------- /tf.version.1/04.rnn/04.01.seq2seq.classification.RNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sequence to Sequence Classification by RNN\n", 8 | "\n", 9 | "- Creating the **data pipeline** with `tf.data`\n", 10 | "- Preprocessing word sequences (variable input sequence length) using `padding technique` by `user function (pad_seq)`\n", 11 | "- Using `tf.nn.embedding_lookup` for getting vector of tokens (eg. word, character)\n", 12 | "- Training **many to many classification** with `tf.contrib.seq2seq.sequence_loss`\n", 13 | "- Masking unvalid token with `tf.sequence_mask`\n", 14 | "- Creating the model as **Class**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "import sys\n", 25 | "import time\n", 26 | "import string\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "import tensorflow as tf\n", 33 | "\n", 34 | "slim = tf.contrib.slim\n", 35 | "rnn = tf.contrib.rnn\n", 36 | "\n", 37 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Prepare example data " 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "sentences = [['I', 'feel', 'hungry'],\n", 54 | " ['You', 'are', 'a', 'genius'],\n", 55 | " ['tensorflow', 'is', 'very', 'difficult'],\n", 56 | " ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n", 57 | " ['tensorflow', 'is', 'very', 'fast', 'changing']]\n", 58 | "pos = [['pronoun', 'verb', 'adjective'],\n", 59 | " ['pronoun', 'verb', 'preposition', 'noun'],\n", 60 | " ['noun', 'verb', 'adverb', 'adjective'],\n", 61 | " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n", 62 | " ['noun', 'verb', 'adverb', 'adjective', 'verb']]" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# word dictionary\n", 72 | "bag_of_words = []\n", 73 | "for sentence in sentences:\n", 74 | " bag_of_words += sentence\n", 75 | "bag_of_words = list(set(bag_of_words))\n", 76 | "bag_of_words.sort()\n", 77 | "bag_of_words = [''] + bag_of_words\n", 78 | "\n", 79 | "word2idx = {word : idx for idx, word in enumerate(bag_of_words)} # word to index\n", 80 | "idx2word = [word for word in bag_of_words] # index to word" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "#print(\"word2idx: {}\".format(word2idx))\n", 90 | "word2idx" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "#print(\"idx2word: {}\".format(idx2word))\n", 100 | "idx2word" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "# pos dictionary\n", 110 | "bag_of_pos = []\n", 111 | "for item in pos:\n", 112 | " bag_of_pos += item\n", 113 | "bag_of_pos = list(set(bag_of_pos))\n", 114 | "bag_of_pos.sort()\n", 115 | "bag_of_pos = [''] + bag_of_pos\n", 116 | "print(\"bag_of_pos: {}\".format(bag_of_pos))\n", 117 | "\n", 118 | "pos2idx = {pos : idx for idx, pos in enumerate(bag_of_pos)} # pos to index\n", 119 | "idx2pos = [pos for pos in bag_of_pos] # index to pos" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "#print(\"pos2idx: {}\".format(pos2idx))\n", 129 | "pos2idx" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "#print(\"idx2pos: {}\".format(idx2pos))\n", 139 | "idx2pos" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Create pad_seq function" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def pad_seq(sequences, max_length, dic):\n", 156 | " \"\"\"Padding sequences\n", 157 | " Padding a special charcter '' from the end of sentence to max_length\n", 158 | " \n", 159 | " Args:\n", 160 | " sequences (list of characters): input data\n", 161 | " max_length (int): max length for padding\n", 162 | " dic (dictionary): char to index\n", 163 | " \n", 164 | " Returns:\n", 165 | " seq_indices (2-rank np.array): \n", 166 | " seq_length (1-rank np.array): sequence lengthes of all data\n", 167 | " \"\"\"\n", 168 | " seq_length, seq_indices = [], []\n", 169 | " for sequence in sequences:\n", 170 | " seq_length.append(len(sequence))\n", 171 | " seq_idx = [dic.get(char) for char in sequence]\n", 172 | " seq_idx += (max_length - len(seq_idx)) * [dic.get('')] # 0 is idx of meaningless token \"\"\n", 173 | " seq_indices.append(seq_idx)\n", 174 | " return np.array(seq_indices), np.array(seq_length)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "### Pre-process data" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "max_length = 10\n", 191 | "X_indices, X_length = pad_seq(sequences=sentences, max_length=max_length, dic=word2idx)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "print(\"X_indices\")\n", 201 | "print(X_indices)\n", 202 | "print(\"X_length\")\n", 203 | "print(X_length)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "y_string = np.array([item + [''] * (max_length - len(item)) for item in pos])\n", 213 | "print(y_string)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "y = np.array([list(map(lambda el : pos2idx.get(el), item)) for item in y_string])\n", 223 | "print(y)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "### Define SimPosRNN" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "class PosRNN:\n", 240 | " def __init__(self, seq_indices, seq_length, labels, num_classes, hidden_dim, max_length, word2idx):\n", 241 | " # Data pipeline\n", 242 | " with tf.variable_scope('input_layer'):\n", 243 | " self._seq_indices = seq_indices\n", 244 | " self._seq_length = seq_length\n", 245 | " self._labels = labels\n", 246 | "\n", 247 | " one_hot = tf.eye(len(word2idx), dtype=tf.float32)\n", 248 | " self._one_hot = tf.get_variable(name='one_hot_embedding',\n", 249 | " initializer=one_hot,\n", 250 | " trainable=False) # embedding vector training 안할 것이기 때문\n", 251 | " self._seq_embeddings = tf.nn.embedding_lookup(params=self._one_hot,\n", 252 | " ids=self._seq_indices)\n", 253 | "\n", 254 | " # RNN cell (many to many)\n", 255 | " with tf.variable_scope('rnn_cell'):\n", 256 | " cell = rnn.BasicRNNCell(num_units=hidden_dim)\n", 257 | " score_cell = rnn.OutputProjectionWrapper(cell=cell,\n", 258 | " output_size=num_classes)\n", 259 | " self._outputs, _ = tf.nn.dynamic_rnn(cell=score_cell, inputs=self._seq_embeddings,\n", 260 | " sequence_length=self._seq_length,\n", 261 | " dtype=tf.float32)\n", 262 | "\n", 263 | " with tf.variable_scope('seq2seq_loss'):\n", 264 | " masks = tf.sequence_mask(lengths=self._seq_length, maxlen=max_length, dtype=tf.float32)\n", 265 | " self.seq2seq_loss = tf.contrib.seq2seq.sequence_loss(logits=self._outputs,\n", 266 | " targets=self._labels,\n", 267 | " weights=masks)\n", 268 | "\n", 269 | " with tf.variable_scope('prediction'):\n", 270 | " self._prediction = tf.argmax(input=self._outputs,\n", 271 | " axis=2, output_type=tf.int32)\n", 272 | "\n", 273 | " def predict(self, sess, seq_indices, seq_length):\n", 274 | " feed_dict = {self._seq_indices : seq_indices, self._seq_length : seq_length}\n", 275 | " return sess.run(self._prediction, feed_dict=feed_dict)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "markdown", 280 | "metadata": {}, 281 | "source": [ 282 | "### Create a model of SimPosRNN" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "metadata": {}, 289 | "outputs": [], 290 | "source": [ 291 | "# hyper-parameter\n", 292 | "num_classes = len(idx2pos)\n", 293 | "learning_rate = .003\n", 294 | "batch_size = 2\n", 295 | "max_epochs = 100" 296 | ] 297 | }, 298 | { 299 | "cell_type": "markdown", 300 | "metadata": {}, 301 | "source": [ 302 | "### Set up dataset with `tf.data`\n", 303 | "\n", 304 | "#### create input pipeline with `tf.data.Dataset`" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": null, 310 | "metadata": {}, 311 | "outputs": [], 312 | "source": [ 313 | "## create data pipeline with tf.data\n", 314 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_indices, X_length, y))\n", 315 | "train_dataset = train_dataset.shuffle(buffer_size = 100)\n", 316 | "train_dataset = train_dataset.batch(batch_size = batch_size)\n", 317 | "print(train_dataset)" 318 | ] 319 | }, 320 | { 321 | "cell_type": "markdown", 322 | "metadata": {}, 323 | "source": [ 324 | "#### Define Iterator" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": null, 330 | "metadata": {}, 331 | "outputs": [], 332 | "source": [ 333 | "train_iterator = train_dataset.make_initializable_iterator()\n", 334 | "seq_indices, seq_length, labels = train_iterator.get_next()" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": null, 340 | "metadata": {}, 341 | "outputs": [], 342 | "source": [ 343 | "pos_rnn = PosRNN(seq_indices=seq_indices, seq_length=seq_length,\n", 344 | " labels=labels, num_classes=num_classes,\n", 345 | " hidden_dim=16, max_length=max_length,\n", 346 | " word2idx=word2idx)" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": {}, 352 | "source": [ 353 | "### Creat training op and train model" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": null, 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [ 362 | "## create training op\n", 363 | "optimizer = tf.train.AdamOptimizer(learning_rate)\n", 364 | "train_op = optimizer.minimize(pos_rnn.seq2seq_loss)" 365 | ] 366 | }, 367 | { 368 | "cell_type": "markdown", 369 | "metadata": {}, 370 | "source": [ 371 | "### `tf.Session()` and train" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": null, 377 | "metadata": {}, 378 | "outputs": [], 379 | "source": [ 380 | "sess = tf.Session()\n", 381 | "sess.run(tf.global_variables_initializer())\n", 382 | "\n", 383 | "loss_history = []\n", 384 | "step = 0\n", 385 | "for epochs in range(max_epochs):\n", 386 | " start_time = time.time()\n", 387 | " sess.run(train_iterator.initializer)\n", 388 | " \n", 389 | " avg_loss = []\n", 390 | " while True:\n", 391 | " try:\n", 392 | " _, loss_ = sess.run([train_op, pos_rnn.seq2seq_loss])\n", 393 | " avg_loss.append(loss_)\n", 394 | " step += 1\n", 395 | "\n", 396 | " except tf.errors.OutOfRangeError:\n", 397 | " #print(\"End of dataset\") # ==> \"End of dataset\"\n", 398 | " break\n", 399 | "\n", 400 | " avg_loss_ = np.mean(avg_loss)\n", 401 | " loss_history.append(avg_loss_)\n", 402 | " \n", 403 | " duration = time.time() - start_time\n", 404 | " examples_per_sec = batch_size / float(duration)\n", 405 | " print(\"epochs: {}, step: {}, loss: {:g}, ({:.2f} examples/sec; {:.3f} sec/batch)\".format(epochs+1, step, avg_loss_, examples_per_sec, duration))" 406 | ] 407 | }, 408 | { 409 | "cell_type": "code", 410 | "execution_count": null, 411 | "metadata": {}, 412 | "outputs": [], 413 | "source": [ 414 | "plt.plot(loss_history, label='train')" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "metadata": {}, 421 | "outputs": [], 422 | "source": [ 423 | "y_pred = pos_rnn.predict(sess=sess, seq_indices=X_indices, seq_length=X_length)\n", 424 | "print(y_pred)" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": null, 430 | "metadata": {}, 431 | "outputs": [], 432 | "source": [ 433 | "result_str = []\n", 434 | "for example in y_pred:\n", 435 | " result_str.append([idx2pos[idx] for idx in example])\n", 436 | " \n", 437 | "for examples in zip(y_string, result_str):\n", 438 | " print(\" Label: \", ' '.join(examples[0]))\n", 439 | " print(\"Prediction: \", ' '.join(examples[1]))" 440 | ] 441 | } 442 | ], 443 | "metadata": { 444 | "kernelspec": { 445 | "display_name": "Python 3", 446 | "language": "python", 447 | "name": "python3" 448 | }, 449 | "language_info": { 450 | "codemirror_mode": { 451 | "name": "ipython", 452 | "version": 3 453 | }, 454 | "file_extension": ".py", 455 | "mimetype": "text/x-python", 456 | "name": "python", 457 | "nbconvert_exporter": "python", 458 | "pygments_lexer": "ipython3", 459 | "version": "3.6.5" 460 | } 461 | }, 462 | "nbformat": 4, 463 | "nbformat_minor": 2 464 | } 465 | -------------------------------------------------------------------------------- /tf.version.1/04.rnn/04.02.seq2seq.classification.GRU.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sequence to Sequence Classification by RNN\n", 8 | "\n", 9 | "- Creating the **data pipeline** with `tf.data`\n", 10 | "- Preprocessing word sequences (variable input sequence length) using `padding technique` by `user function (pad_seq)`\n", 11 | "- Using `tf.nn.embedding_lookup` for getting vector of tokens (eg. word, character)\n", 12 | "- Training **many to many classification** with `tf.contrib.seq2seq.sequence_loss`\n", 13 | "- Masking unvalid token with `tf.sequence_mask`\n", 14 | "- Creating the model as **Class**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "import sys\n", 25 | "import time\n", 26 | "import string\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "import tensorflow as tf\n", 33 | "\n", 34 | "slim = tf.contrib.slim\n", 35 | "rnn = tf.contrib.rnn\n", 36 | "\n", 37 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Prepare example data " 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "sentences = [['I', 'feel', 'hungry'],\n", 54 | " ['You', 'are', 'a', 'genius'],\n", 55 | " ['tensorflow', 'is', 'very', 'difficult'],\n", 56 | " ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n", 57 | " ['tensorflow', 'is', 'very', 'fast', 'changing']]\n", 58 | "pos = [['pronoun', 'verb', 'adjective'],\n", 59 | " ['pronoun', 'verb', 'preposition', 'noun'],\n", 60 | " ['noun', 'verb', 'adverb', 'adjective'],\n", 61 | " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n", 62 | " ['noun', 'verb', 'adverb', 'adjective', 'verb']]" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# word dictionary\n", 72 | "bag_of_words = []\n", 73 | "for sentence in sentences:\n", 74 | " bag_of_words += sentence\n", 75 | "bag_of_words = list(set(bag_of_words))\n", 76 | "bag_of_words.sort()\n", 77 | "bag_of_words = [''] + bag_of_words\n", 78 | "\n", 79 | "word2idx = {word : idx for idx, word in enumerate(bag_of_words)} # word to index\n", 80 | "idx2word = [word for word in bag_of_words] # index to word" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "#print(\"word2idx: {}\".format(word2idx))\n", 90 | "word2idx" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "#print(\"idx2word: {}\".format(idx2word))\n", 100 | "idx2word" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "# pos dictionary\n", 110 | "bag_of_pos = []\n", 111 | "for item in pos:\n", 112 | " bag_of_pos += item\n", 113 | "bag_of_pos = list(set(bag_of_pos))\n", 114 | "bag_of_pos.sort()\n", 115 | "bag_of_pos = [''] + bag_of_pos\n", 116 | "print(\"bag_of_pos: {}\".format(bag_of_pos))\n", 117 | "\n", 118 | "pos2idx = {pos : idx for idx, pos in enumerate(bag_of_pos)} # pos to index\n", 119 | "idx2pos = [pos for pos in bag_of_pos] # index to pos" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "#print(\"pos2idx: {}\".format(pos2idx))\n", 129 | "pos2idx" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "#print(\"idx2pos: {}\".format(idx2pos))\n", 139 | "idx2pos" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Create pad_seq function" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def pad_seq(sequences, max_length, dic):\n", 156 | " \"\"\"Padding sequences\n", 157 | " Padding a special charcter '' from the end of sentence to max_length\n", 158 | " \n", 159 | " Args:\n", 160 | " sequences (list of characters): input data\n", 161 | " max_length (int): max length for padding\n", 162 | " dic (dictionary): char to index\n", 163 | " \n", 164 | " Returns:\n", 165 | " seq_indices (2-rank np.array): \n", 166 | " seq_length (1-rank np.array): sequence lengthes of all data\n", 167 | " \"\"\"\n", 168 | " seq_length, seq_indices = [], []\n", 169 | " for sequence in sequences:\n", 170 | " seq_length.append(len(sequence))\n", 171 | " seq_idx = [dic.get(char) for char in sequence]\n", 172 | " seq_idx += (max_length - len(seq_idx)) * [dic.get('')] # 0 is idx of meaningless token \"\"\n", 173 | " seq_indices.append(seq_idx)\n", 174 | " return np.array(seq_indices), np.array(seq_length)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "### Pre-process data" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "max_length = 10\n", 191 | "X_indices, X_length = pad_seq(sequences=sentences, max_length=max_length, dic=word2idx)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "print(\"X_indices\")\n", 201 | "print(X_indices)\n", 202 | "print(\"X_length\")\n", 203 | "print(X_length)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "y_string = np.array([item + [''] * (max_length - len(item)) for item in pos])\n", 213 | "print(y_string)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "y = np.array([list(map(lambda el : pos2idx.get(el), item)) for item in y_string])\n", 223 | "print(y)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "### Define SimPosRNN" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "class PosRNN:\n", 240 | " def __init__(self, seq_indices, seq_length, labels, num_classes, hidden_dim, max_length, word2idx):\n", 241 | " # Data pipeline\n", 242 | " with tf.variable_scope('input_layer'):\n", 243 | " self._seq_indices = seq_indices\n", 244 | " self._seq_length = seq_length\n", 245 | " self._labels = labels\n", 246 | "\n", 247 | " one_hot = tf.eye(len(word2idx), dtype = tf.float32)\n", 248 | " self._one_hot = tf.get_variable(name='one_hot_embedding',\n", 249 | " initializer=one_hot,\n", 250 | " trainable=False) # embedding vector training 안할 것이기 때문\n", 251 | " self._seq_embeddings = tf.nn.embedding_lookup(params=self._one_hot,\n", 252 | " ids=self._seq_indices)\n", 253 | "\n", 254 | " # GRU cell (many to many)\n", 255 | " with tf.variable_scope('gru_cell'):\n", 256 | " cell = rnn.GRUCell(num_units=hidden_dim)\n", 257 | " score_cell = rnn.OutputProjectionWrapper(cell=cell,\n", 258 | " output_size=num_classes)\n", 259 | " self._outputs, _ = tf.nn.dynamic_rnn(cell=score_cell, inputs=self._seq_embeddings,\n", 260 | " sequence_length=self._seq_length,\n", 261 | " dtype=tf.float32)\n", 262 | "\n", 263 | " with tf.variable_scope('seq2seq_loss'):\n", 264 | " masks = tf.sequence_mask(lengths=self._seq_length, maxlen=max_length, dtype=tf.float32)\n", 265 | " self.seq2seq_loss = tf.contrib.seq2seq.sequence_loss(logits=self._outputs,\n", 266 | " targets=self._labels,\n", 267 | " weights=masks)\n", 268 | "\n", 269 | " with tf.variable_scope('prediction'):\n", 270 | " self._prediction = tf.argmax(input=self._outputs,\n", 271 | " axis=2, output_type=tf.int32)\n", 272 | "\n", 273 | " def predict(self, sess, seq_indices, seq_length):\n", 274 | " feed_dict = {self._seq_indices : seq_indices, self._seq_length : seq_length}\n", 275 | " return sess.run(self._prediction, feed_dict=feed_dict)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "markdown", 280 | "metadata": {}, 281 | "source": [ 282 | "### Create a model of SimPosRNN" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "metadata": {}, 289 | "outputs": [], 290 | "source": [ 291 | "# hyper-parameters\n", 292 | "num_classes = len(idx2pos)\n", 293 | "learning_rate = .003\n", 294 | "batch_size = 2\n", 295 | "max_epochs = 100" 296 | ] 297 | }, 298 | { 299 | "cell_type": "markdown", 300 | "metadata": {}, 301 | "source": [ 302 | "### Set up dataset with `tf.data`\n", 303 | "\n", 304 | "#### create input pipeline with `tf.data.Dataset`" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": null, 310 | "metadata": {}, 311 | "outputs": [], 312 | "source": [ 313 | "## create data pipeline with tf.data\n", 314 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_indices, X_length, y))\n", 315 | "train_dataset = train_dataset.shuffle(buffer_size = 100)\n", 316 | "train_dataset = train_dataset.batch(batch_size = batch_size)\n", 317 | "print(train_dataset)" 318 | ] 319 | }, 320 | { 321 | "cell_type": "markdown", 322 | "metadata": {}, 323 | "source": [ 324 | "#### Define Iterator" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": null, 330 | "metadata": {}, 331 | "outputs": [], 332 | "source": [ 333 | "train_iterator = train_dataset.make_initializable_iterator()\n", 334 | "seq_indices, seq_length, labels = train_iterator.get_next()" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": null, 340 | "metadata": {}, 341 | "outputs": [], 342 | "source": [ 343 | "pos_rnn = PosRNN(seq_indices=seq_indices, seq_length=seq_length,\n", 344 | " labels=labels, num_classes=num_classes,\n", 345 | " hidden_dim=16, max_length=max_length,\n", 346 | " word2idx=word2idx)" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": {}, 352 | "source": [ 353 | "### Creat training op and train model" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": null, 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [ 362 | "## create training op\n", 363 | "optimizer = tf.train.AdamOptimizer(learning_rate)\n", 364 | "train_op = optimizer.minimize(pos_rnn.seq2seq_loss)" 365 | ] 366 | }, 367 | { 368 | "cell_type": "markdown", 369 | "metadata": {}, 370 | "source": [ 371 | "### `tf.Session()` and train" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": null, 377 | "metadata": {}, 378 | "outputs": [], 379 | "source": [ 380 | "sess = tf.Session()\n", 381 | "sess.run(tf.global_variables_initializer())\n", 382 | "\n", 383 | "loss_history = []\n", 384 | "step = 0\n", 385 | "for epochs in range(max_epochs):\n", 386 | " start_time = time.time()\n", 387 | " sess.run(train_iterator.initializer)\n", 388 | " \n", 389 | " avg_loss = []\n", 390 | " while True:\n", 391 | " try:\n", 392 | " _, loss_ = sess.run([train_op, pos_rnn.seq2seq_loss])\n", 393 | " avg_loss.append(loss_)\n", 394 | " step += 1\n", 395 | "\n", 396 | " except tf.errors.OutOfRangeError:\n", 397 | " #print(\"End of dataset\") # ==> \"End of dataset\"\n", 398 | " break\n", 399 | "\n", 400 | " avg_loss_ = np.mean(avg_loss)\n", 401 | " loss_history.append(avg_loss_)\n", 402 | " \n", 403 | " duration = time.time() - start_time\n", 404 | " examples_per_sec = batch_size / float(duration)\n", 405 | " print(\"epochs: {}, step: {}, loss: {:g}, ({:.2f} examples/sec; {:.3f} sec/batch)\".format(epochs+1, step, avg_loss_, examples_per_sec, duration))" 406 | ] 407 | }, 408 | { 409 | "cell_type": "code", 410 | "execution_count": null, 411 | "metadata": {}, 412 | "outputs": [], 413 | "source": [ 414 | "plt.plot(loss_history, label='train')" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "metadata": {}, 421 | "outputs": [], 422 | "source": [ 423 | "y_pred = pos_rnn.predict(sess=sess, seq_indices=X_indices, seq_length=X_length)\n", 424 | "print(y_pred)" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": null, 430 | "metadata": {}, 431 | "outputs": [], 432 | "source": [ 433 | "result_str = []\n", 434 | "for example in y_pred:\n", 435 | " result_str.append([idx2pos[idx] for idx in example])\n", 436 | " \n", 437 | "for examples in zip(y_string, result_str):\n", 438 | " print(\" Label: \", ' '.join(examples[0]))\n", 439 | " print(\"Prediction: \", ' '.join(examples[1]))" 440 | ] 441 | } 442 | ], 443 | "metadata": { 444 | "kernelspec": { 445 | "display_name": "Python 3", 446 | "language": "python", 447 | "name": "python3" 448 | }, 449 | "language_info": { 450 | "codemirror_mode": { 451 | "name": "ipython", 452 | "version": 3 453 | }, 454 | "file_extension": ".py", 455 | "mimetype": "text/x-python", 456 | "name": "python", 457 | "nbconvert_exporter": "python", 458 | "pygments_lexer": "ipython3", 459 | "version": "3.6.5" 460 | } 461 | }, 462 | "nbformat": 4, 463 | "nbformat_minor": 2 464 | } 465 | -------------------------------------------------------------------------------- /tf.version.1/04.rnn/04.04.seq2seq.classification.biRNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sequence to Sequence Classification by RNN\n", 8 | "\n", 9 | "- Creating the **data pipeline** with `tf.data`\n", 10 | "- Preprocessing word sequences (variable input sequence length) using `padding technique` by `user function (pad_seq)`\n", 11 | "- Using `tf.nn.embedding_lookup` for getting vector of tokens (eg. word, character)\n", 12 | "- Training **many to many classification** with `tf.contrib.seq2seq.sequence_loss`\n", 13 | "- Masking unvalid token with `tf.sequence_mask`\n", 14 | "- Creating the model as **Class**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "import sys\n", 25 | "import time\n", 26 | "import string\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "import tensorflow as tf\n", 33 | "\n", 34 | "slim = tf.contrib.slim\n", 35 | "rnn = tf.contrib.rnn\n", 36 | "\n", 37 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Prepare example data " 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "sentences = [['I', 'feel', 'hungry'],\n", 54 | " ['You', 'are', 'a', 'genius'],\n", 55 | " ['tensorflow', 'is', 'very', 'difficult'],\n", 56 | " ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n", 57 | " ['tensorflow', 'is', 'very', 'fast', 'changing']]\n", 58 | "pos = [['pronoun', 'verb', 'adjective'],\n", 59 | " ['pronoun', 'verb', 'preposition', 'noun'],\n", 60 | " ['noun', 'verb', 'adverb', 'adjective'],\n", 61 | " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n", 62 | " ['noun', 'verb', 'adverb', 'adjective', 'verb']]" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# word dictionary\n", 72 | "bag_of_words = []\n", 73 | "for sentence in sentences:\n", 74 | " bag_of_words += sentence\n", 75 | "bag_of_words = list(set(bag_of_words))\n", 76 | "bag_of_words.sort()\n", 77 | "bag_of_words = [''] + bag_of_words\n", 78 | "\n", 79 | "word2idx = {word : idx for idx, word in enumerate(bag_of_words)} # word to index\n", 80 | "idx2word = [word for word in bag_of_words] # index to word" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "#print(\"word2idx: {}\".format(word2idx))\n", 90 | "word2idx" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "#print(\"idx2word: {}\".format(idx2word))\n", 100 | "idx2word" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "# pos dictionary\n", 110 | "bag_of_pos = []\n", 111 | "for item in pos:\n", 112 | " bag_of_pos += item\n", 113 | "bag_of_pos = list(set(bag_of_pos))\n", 114 | "bag_of_pos.sort()\n", 115 | "bag_of_pos = [''] + bag_of_pos\n", 116 | "print(\"bag_of_pos: {}\".format(bag_of_pos))\n", 117 | "\n", 118 | "pos2idx = {pos : idx for idx, pos in enumerate(bag_of_pos)} # pos to index\n", 119 | "idx2pos = [pos for pos in bag_of_pos] # index to pos" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "#print(\"pos2idx: {}\".format(pos2idx))\n", 129 | "pos2idx" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "#print(\"idx2pos: {}\".format(idx2pos))\n", 139 | "idx2pos" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Create pad_seq function" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def pad_seq(sequences, max_length, dic):\n", 156 | " \"\"\"Padding sequences\n", 157 | " Padding a special charcter '' from the end of sentence to max_length\n", 158 | " \n", 159 | " Args:\n", 160 | " sequences (list of characters): input data\n", 161 | " max_length (int): max length for padding\n", 162 | " dic (dictionary): char to index\n", 163 | " \n", 164 | " Returns:\n", 165 | " seq_indices (2-rank np.array): \n", 166 | " seq_length (1-rank np.array): sequence lengthes of all data\n", 167 | " \"\"\"\n", 168 | " seq_length, seq_indices = [], []\n", 169 | " for sequence in sequences:\n", 170 | " seq_length.append(len(sequence))\n", 171 | " seq_idx = [dic.get(char) for char in sequence]\n", 172 | " seq_idx += (max_length - len(seq_idx)) * [dic.get('')] # 0 is idx of meaningless token \"\"\n", 173 | " seq_indices.append(seq_idx)\n", 174 | " return np.array(seq_indices), np.array(seq_length)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "### Pre-process data" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "max_length = 10\n", 191 | "X_indices, X_length = pad_seq(sequences=sentences, max_length=max_length, dic=word2idx)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "print(\"X_indices\")\n", 201 | "print(X_indices)\n", 202 | "print(\"X_length\")\n", 203 | "print(X_length)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "y_string = np.array([item + [''] * (max_length - len(item)) for item in pos])\n", 213 | "print(y_string)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "y = np.array([list(map(lambda el : pos2idx.get(el), item)) for item in y_string])\n", 223 | "print(y)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "### Define SimPosRNN" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "class PosRNN:\n", 240 | " def __init__(self, seq_indices, seq_length, labels, num_classes, hidden_dim, max_length, word2idx):\n", 241 | " # Data pipeline\n", 242 | " with tf.variable_scope('input_layer'):\n", 243 | " self._seq_indices = seq_indices\n", 244 | " self._seq_length = seq_length\n", 245 | " self._labels = labels\n", 246 | "\n", 247 | " one_hot = tf.eye(len(word2idx), dtype=tf.float32)\n", 248 | " self._one_hot = tf.get_variable(name='one_hot_embedding',\n", 249 | " initializer=one_hot,\n", 250 | " trainable=False) # embedding vector training 안할 것이기 때문\n", 251 | " self._seq_embeddings = tf.nn.embedding_lookup(params=self._one_hot,\n", 252 | " ids=self._seq_indices)\n", 253 | "\n", 254 | " # bidirectional RNN cell (many to many)\n", 255 | " with tf.variable_scope('rnn_cell'):\n", 256 | " cell_fw = rnn.BasicRNNCell(num_units=hidden_dim)\n", 257 | " cell_bw = rnn.BasicRNNCell(num_units=hidden_dim)\n", 258 | " outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw,\n", 259 | " self._seq_embeddings,\n", 260 | " sequence_length=self._seq_length,\n", 261 | " dtype=tf.float32)\n", 262 | " concat_outputs = tf.concat([outputs[0], outputs[1]], axis=2)\n", 263 | " \n", 264 | " weights = tf.get_variable(name='weights', shape=[2 * hidden_dim, num_classes],\n", 265 | " initializer=slim.xavier_initializer())\n", 266 | " self._logits = tf.map_fn(lambda elm : tf.matmul(elm, weights), concat_outputs)\n", 267 | "\n", 268 | " with tf.variable_scope('seq2seq_loss'):\n", 269 | " masks = tf.sequence_mask(lengths=self._seq_length, maxlen=max_length, dtype=tf.float32)\n", 270 | " self.seq2seq_loss = tf.contrib.seq2seq.sequence_loss(logits=self._logits,\n", 271 | " targets=self._labels,\n", 272 | " weights=masks)\n", 273 | "\n", 274 | " with tf.variable_scope('prediction'):\n", 275 | " self._prediction = tf.argmax(input=self._logits,\n", 276 | " axis=2, output_type=tf.int32)\n", 277 | "\n", 278 | " def predict(self, sess, seq_indices, seq_length):\n", 279 | " feed_dict = {self._seq_indices : seq_indices, self._seq_length : seq_length}\n", 280 | " return sess.run(self._prediction, feed_dict=feed_dict)" 281 | ] 282 | }, 283 | { 284 | "cell_type": "markdown", 285 | "metadata": {}, 286 | "source": [ 287 | "### Create a model of SimPosRNN" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": null, 293 | "metadata": {}, 294 | "outputs": [], 295 | "source": [ 296 | "# hyper-parameter\n", 297 | "num_classes = len(idx2pos)\n", 298 | "learning_rate = .003\n", 299 | "batch_size = 2\n", 300 | "max_epochs = 100" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "### Set up dataset with `tf.data`\n", 308 | "\n", 309 | "#### create input pipeline with `tf.data.Dataset`" 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": null, 315 | "metadata": {}, 316 | "outputs": [], 317 | "source": [ 318 | "## create data pipeline with tf.data\n", 319 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_indices, X_length, y))\n", 320 | "train_dataset = train_dataset.shuffle(buffer_size = 100)\n", 321 | "train_dataset = train_dataset.batch(batch_size = batch_size)\n", 322 | "print(train_dataset)" 323 | ] 324 | }, 325 | { 326 | "cell_type": "markdown", 327 | "metadata": {}, 328 | "source": [ 329 | "#### Define Iterator" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": null, 335 | "metadata": {}, 336 | "outputs": [], 337 | "source": [ 338 | "train_iterator = train_dataset.make_initializable_iterator()\n", 339 | "seq_indices, seq_length, labels = train_iterator.get_next()" 340 | ] 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": null, 345 | "metadata": {}, 346 | "outputs": [], 347 | "source": [ 348 | "pos_rnn = PosRNN(seq_indices=seq_indices, seq_length=seq_length,\n", 349 | " labels=labels, num_classes=num_classes,\n", 350 | " hidden_dim=16, max_length=max_length,\n", 351 | " word2idx=word2idx)" 352 | ] 353 | }, 354 | { 355 | "cell_type": "markdown", 356 | "metadata": {}, 357 | "source": [ 358 | "### Creat training op and train model" 359 | ] 360 | }, 361 | { 362 | "cell_type": "code", 363 | "execution_count": null, 364 | "metadata": {}, 365 | "outputs": [], 366 | "source": [ 367 | "## create training op\n", 368 | "optimizer = tf.train.AdamOptimizer(learning_rate)\n", 369 | "train_op = optimizer.minimize(pos_rnn.seq2seq_loss)" 370 | ] 371 | }, 372 | { 373 | "cell_type": "markdown", 374 | "metadata": {}, 375 | "source": [ 376 | "### `tf.Session()` and train" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": null, 382 | "metadata": {}, 383 | "outputs": [], 384 | "source": [ 385 | "sess = tf.Session()\n", 386 | "sess.run(tf.global_variables_initializer())\n", 387 | "\n", 388 | "loss_history = []\n", 389 | "step = 0\n", 390 | "for epochs in range(max_epochs):\n", 391 | " start_time = time.time()\n", 392 | " sess.run(train_iterator.initializer)\n", 393 | " \n", 394 | " avg_loss = []\n", 395 | " while True:\n", 396 | " try:\n", 397 | " _, loss_ = sess.run([train_op, pos_rnn.seq2seq_loss])\n", 398 | " avg_loss.append(loss_)\n", 399 | " step += 1\n", 400 | "\n", 401 | " except tf.errors.OutOfRangeError:\n", 402 | " #print(\"End of dataset\") # ==> \"End of dataset\"\n", 403 | " break\n", 404 | "\n", 405 | " avg_loss_ = np.mean(avg_loss)\n", 406 | " loss_history.append(avg_loss_)\n", 407 | " \n", 408 | " duration = time.time() - start_time\n", 409 | " examples_per_sec = batch_size / float(duration)\n", 410 | " print(\"epochs: {}, step: {}, loss: {:g}, ({:.2f} examples/sec; {:.3f} sec/batch)\".format(epochs+1, step, avg_loss_, examples_per_sec, duration))" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": null, 416 | "metadata": {}, 417 | "outputs": [], 418 | "source": [ 419 | "plt.plot(loss_history, label='train')" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": null, 425 | "metadata": {}, 426 | "outputs": [], 427 | "source": [ 428 | "y_pred = pos_rnn.predict(sess=sess, seq_indices=X_indices, seq_length=X_length)\n", 429 | "print(y_pred)" 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "execution_count": null, 435 | "metadata": {}, 436 | "outputs": [], 437 | "source": [ 438 | "result_str = []\n", 439 | "for example in y_pred:\n", 440 | " result_str.append([idx2pos[idx] for idx in example])\n", 441 | " \n", 442 | "for examples in zip(y_string, result_str):\n", 443 | " print(\" Label: \", ' '.join(examples[0]))\n", 444 | " print(\"Prediction: \", ' '.join(examples[1]))" 445 | ] 446 | } 447 | ], 448 | "metadata": { 449 | "kernelspec": { 450 | "display_name": "Python 3", 451 | "language": "python", 452 | "name": "python3" 453 | }, 454 | "language_info": { 455 | "codemirror_mode": { 456 | "name": "ipython", 457 | "version": 3 458 | }, 459 | "file_extension": ".py", 460 | "mimetype": "text/x-python", 461 | "name": "python", 462 | "nbconvert_exporter": "python", 463 | "pygments_lexer": "ipython3", 464 | "version": "3.6.5" 465 | } 466 | }, 467 | "nbformat": 4, 468 | "nbformat_minor": 2 469 | } 470 | -------------------------------------------------------------------------------- /tf.version.1/04.rnn/04.05.seq2seq.classification.biLSTM.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sequence to Sequence Classification by RNN\n", 8 | "\n", 9 | "- Creating the **data pipeline** with `tf.data`\n", 10 | "- Preprocessing word sequences (variable input sequence length) using `padding technique` by `user function (pad_seq)`\n", 11 | "- Using `tf.nn.embedding_lookup` for getting vector of tokens (eg. word, character)\n", 12 | "- Training **many to many classification** with `tf.contrib.seq2seq.sequence_loss`\n", 13 | "- Masking unvalid token with `tf.sequence_mask`\n", 14 | "- Creating the model as **Class**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "import sys\n", 25 | "import time\n", 26 | "import string\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "import tensorflow as tf\n", 33 | "\n", 34 | "slim = tf.contrib.slim\n", 35 | "rnn = tf.contrib.rnn\n", 36 | "\n", 37 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Prepare example data " 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "sentences = [['I', 'feel', 'hungry'],\n", 54 | " ['You', 'are', 'a', 'genius'],\n", 55 | " ['tensorflow', 'is', 'very', 'difficult'],\n", 56 | " ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n", 57 | " ['tensorflow', 'is', 'very', 'fast', 'changing']]\n", 58 | "pos = [['pronoun', 'verb', 'adjective'],\n", 59 | " ['pronoun', 'verb', 'preposition', 'noun'],\n", 60 | " ['noun', 'verb', 'adverb', 'adjective'],\n", 61 | " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n", 62 | " ['noun', 'verb', 'adverb', 'adjective', 'verb']]" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# word dictionary\n", 72 | "bag_of_words = []\n", 73 | "for sentence in sentences:\n", 74 | " bag_of_words += sentence\n", 75 | "bag_of_words = list(set(bag_of_words))\n", 76 | "bag_of_words.sort()\n", 77 | "bag_of_words = [''] + bag_of_words\n", 78 | "\n", 79 | "word2idx = {word : idx for idx, word in enumerate(bag_of_words)} # word to index\n", 80 | "idx2word = [word for word in bag_of_words] # index to word" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "#print(\"word2idx: {}\".format(word2idx))\n", 90 | "word2idx" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "#print(\"idx2word: {}\".format(idx2word))\n", 100 | "idx2word" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "# pos dictionary\n", 110 | "bag_of_pos = []\n", 111 | "for item in pos:\n", 112 | " bag_of_pos += item\n", 113 | "bag_of_pos = list(set(bag_of_pos))\n", 114 | "bag_of_pos.sort()\n", 115 | "bag_of_pos = [''] + bag_of_pos\n", 116 | "print(\"bag_of_pos: {}\".format(bag_of_pos))\n", 117 | "\n", 118 | "pos2idx = {pos : idx for idx, pos in enumerate(bag_of_pos)} # pos to index\n", 119 | "idx2pos = [pos for pos in bag_of_pos] # index to pos" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "#print(\"pos2idx: {}\".format(pos2idx))\n", 129 | "pos2idx" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "#print(\"idx2pos: {}\".format(idx2pos))\n", 139 | "idx2pos" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Create pad_seq function" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def pad_seq(sequences, max_length, dic):\n", 156 | " \"\"\"Padding sequences\n", 157 | " Padding a special charcter '' from the end of sentence to max_length\n", 158 | " \n", 159 | " Args:\n", 160 | " sequences (list of characters): input data\n", 161 | " max_length (int): max length for padding\n", 162 | " dic (dictionary): char to index\n", 163 | " \n", 164 | " Returns:\n", 165 | " seq_indices (2-rank np.array): \n", 166 | " seq_length (1-rank np.array): sequence lengthes of all data\n", 167 | " \"\"\"\n", 168 | " seq_length, seq_indices = [], []\n", 169 | " for sequence in sequences:\n", 170 | " seq_length.append(len(sequence))\n", 171 | " seq_idx = [dic.get(char) for char in sequence]\n", 172 | " seq_idx += (max_length - len(seq_idx)) * [dic.get('')] # 0 is idx of meaningless token \"\"\n", 173 | " seq_indices.append(seq_idx)\n", 174 | " return np.array(seq_indices), np.array(seq_length)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "### Pre-process data" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "max_length = 10\n", 191 | "X_indices, X_length = pad_seq(sequences=sentences, max_length=max_length, dic=word2idx)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "print(\"X_indices\")\n", 201 | "print(X_indices)\n", 202 | "print(\"X_length\")\n", 203 | "print(X_length)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "y_string = np.array([item + [''] * (max_length - len(item)) for item in pos])\n", 213 | "print(y_string)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "y = np.array([list(map(lambda el : pos2idx.get(el), item)) for item in y_string])\n", 223 | "print(y)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "### Define SimPosRNN" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "class PosRNN:\n", 240 | " def __init__(self, seq_indices, seq_length, labels, num_classes, hidden_dim, max_length, word2idx):\n", 241 | " # Data pipeline\n", 242 | " with tf.variable_scope('input_layer'):\n", 243 | " self._seq_indices = seq_indices\n", 244 | " self._seq_length = seq_length\n", 245 | " self._labels = labels\n", 246 | "\n", 247 | " one_hot = tf.eye(len(word2idx), dtype=tf.float32)\n", 248 | " self._one_hot = tf.get_variable(name='one_hot_embedding',\n", 249 | " initializer=one_hot,\n", 250 | " trainable=False) # embedding vector training 안할 것이기 때문\n", 251 | " self._seq_embeddings = tf.nn.embedding_lookup(params=self._one_hot,\n", 252 | " ids=self._seq_indices)\n", 253 | "\n", 254 | " # bidirectional LSTM cell (many to many)\n", 255 | " with tf.variable_scope('rnn_cell'):\n", 256 | " cell_fw = rnn.BasicLSTMCell(num_units=hidden_dim, state_is_tuple=True)\n", 257 | " cell_bw = rnn.BasicLSTMCell(num_units=hidden_dim, state_is_tuple=True)\n", 258 | " outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw,\n", 259 | " self._seq_embeddings,\n", 260 | " sequence_length=self._seq_length,\n", 261 | " dtype=tf.float32)\n", 262 | " concat_outputs = tf.concat([outputs[0], outputs[1]], axis=2)\n", 263 | " \n", 264 | " weights = tf.get_variable(name='weights', shape=[2 * hidden_dim, num_classes],\n", 265 | " initializer=slim.xavier_initializer())\n", 266 | " self._logits = tf.map_fn(lambda elm : tf.matmul(elm, weights), concat_outputs)\n", 267 | "\n", 268 | " with tf.variable_scope('seq2seq_loss'):\n", 269 | " masks = tf.sequence_mask(lengths=self._seq_length, maxlen=max_length, dtype=tf.float32)\n", 270 | " self.seq2seq_loss = tf.contrib.seq2seq.sequence_loss(logits=self._logits,\n", 271 | " targets=self._labels,\n", 272 | " weights=masks)\n", 273 | "\n", 274 | " with tf.variable_scope('prediction'):\n", 275 | " self._prediction = tf.argmax(input=self._logits,\n", 276 | " axis=2, output_type=tf.int32)\n", 277 | "\n", 278 | " def predict(self, sess, seq_indices, seq_length):\n", 279 | " feed_dict = {self._seq_indices : seq_indices, self._seq_length : seq_length}\n", 280 | " return sess.run(self._prediction, feed_dict=feed_dict)" 281 | ] 282 | }, 283 | { 284 | "cell_type": "markdown", 285 | "metadata": {}, 286 | "source": [ 287 | "### Create a model of SimPosRNN" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": null, 293 | "metadata": {}, 294 | "outputs": [], 295 | "source": [ 296 | "# hyper-parameter\n", 297 | "num_classes = len(idx2pos)\n", 298 | "learning_rate = .003\n", 299 | "batch_size = 2\n", 300 | "max_epochs = 100" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "### Set up dataset with `tf.data`\n", 308 | "\n", 309 | "#### create input pipeline with `tf.data.Dataset`" 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": null, 315 | "metadata": {}, 316 | "outputs": [], 317 | "source": [ 318 | "## create data pipeline with tf.data\n", 319 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_indices, X_length, y))\n", 320 | "train_dataset = train_dataset.shuffle(buffer_size = 100)\n", 321 | "train_dataset = train_dataset.batch(batch_size = batch_size)\n", 322 | "print(train_dataset)" 323 | ] 324 | }, 325 | { 326 | "cell_type": "markdown", 327 | "metadata": {}, 328 | "source": [ 329 | "#### Define Iterator" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": null, 335 | "metadata": {}, 336 | "outputs": [], 337 | "source": [ 338 | "train_iterator = train_dataset.make_initializable_iterator()\n", 339 | "seq_indices, seq_length, labels = train_iterator.get_next()" 340 | ] 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": null, 345 | "metadata": {}, 346 | "outputs": [], 347 | "source": [ 348 | "pos_rnn = PosRNN(seq_indices=seq_indices, seq_length=seq_length,\n", 349 | " labels=labels, num_classes=num_classes,\n", 350 | " hidden_dim=16, max_length=max_length,\n", 351 | " word2idx=word2idx)" 352 | ] 353 | }, 354 | { 355 | "cell_type": "markdown", 356 | "metadata": {}, 357 | "source": [ 358 | "### Creat training op and train model" 359 | ] 360 | }, 361 | { 362 | "cell_type": "code", 363 | "execution_count": null, 364 | "metadata": {}, 365 | "outputs": [], 366 | "source": [ 367 | "## create training op\n", 368 | "optimizer = tf.train.AdamOptimizer(learning_rate)\n", 369 | "train_op = optimizer.minimize(pos_rnn.seq2seq_loss)" 370 | ] 371 | }, 372 | { 373 | "cell_type": "markdown", 374 | "metadata": {}, 375 | "source": [ 376 | "### `tf.Session()` and train" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": null, 382 | "metadata": {}, 383 | "outputs": [], 384 | "source": [ 385 | "sess = tf.Session()\n", 386 | "sess.run(tf.global_variables_initializer())\n", 387 | "\n", 388 | "loss_history = []\n", 389 | "step = 0\n", 390 | "for epochs in range(max_epochs):\n", 391 | " start_time = time.time()\n", 392 | " sess.run(train_iterator.initializer)\n", 393 | " \n", 394 | " avg_loss = []\n", 395 | " while True:\n", 396 | " try:\n", 397 | " _, loss_ = sess.run([train_op, pos_rnn.seq2seq_loss])\n", 398 | " avg_loss.append(loss_)\n", 399 | " step += 1\n", 400 | "\n", 401 | " except tf.errors.OutOfRangeError:\n", 402 | " #print(\"End of dataset\") # ==> \"End of dataset\"\n", 403 | " break\n", 404 | "\n", 405 | " avg_loss_ = np.mean(avg_loss)\n", 406 | " loss_history.append(avg_loss_)\n", 407 | " \n", 408 | " duration = time.time() - start_time\n", 409 | " examples_per_sec = batch_size / float(duration)\n", 410 | " print(\"epochs: {}, step: {}, loss: {:g}, ({:.2f} examples/sec; {:.3f} sec/batch)\".format(epochs+1, step, avg_loss_, examples_per_sec, duration))" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": null, 416 | "metadata": {}, 417 | "outputs": [], 418 | "source": [ 419 | "plt.plot(loss_history, label='train')" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": null, 425 | "metadata": {}, 426 | "outputs": [], 427 | "source": [ 428 | "y_pred = pos_rnn.predict(sess=sess, seq_indices=X_indices, seq_length=X_length)\n", 429 | "print(y_pred)" 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "execution_count": null, 435 | "metadata": {}, 436 | "outputs": [], 437 | "source": [ 438 | "result_str = []\n", 439 | "for example in y_pred:\n", 440 | " result_str.append([idx2pos[idx] for idx in example])\n", 441 | " \n", 442 | "for examples in zip(y_string, result_str):\n", 443 | " print(\" Label: \", ' '.join(examples[0]))\n", 444 | " print(\"Prediction: \", ' '.join(examples[1]))" 445 | ] 446 | } 447 | ], 448 | "metadata": { 449 | "kernelspec": { 450 | "display_name": "Python 3", 451 | "language": "python", 452 | "name": "python3" 453 | }, 454 | "language_info": { 455 | "codemirror_mode": { 456 | "name": "ipython", 457 | "version": 3 458 | }, 459 | "file_extension": ".py", 460 | "mimetype": "text/x-python", 461 | "name": "python", 462 | "nbconvert_exporter": "python", 463 | "pygments_lexer": "ipython3", 464 | "version": "3.6.5" 465 | } 466 | }, 467 | "nbformat": 4, 468 | "nbformat_minor": 2 469 | } 470 | -------------------------------------------------------------------------------- /tf.version.1/04.rnn/04.06.seq2seq.classification.Multi.RNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sequence to Sequence Classification by RNN\n", 8 | "\n", 9 | "- Creating the **data pipeline** with `tf.data`\n", 10 | "- Preprocessing word sequences (variable input sequence length) using `padding technique` by `user function (pad_seq)`\n", 11 | "- Using `tf.nn.embedding_lookup` for getting vector of tokens (eg. word, character)\n", 12 | "- Training **many to many classification** with `tf.contrib.seq2seq.sequence_loss`\n", 13 | "- Masking unvalid token with `tf.sequence_mask`\n", 14 | "- Creating the model as **Class**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "import sys\n", 25 | "import time\n", 26 | "import string\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "import tensorflow as tf\n", 33 | "\n", 34 | "slim = tf.contrib.slim\n", 35 | "rnn = tf.contrib.rnn\n", 36 | "\n", 37 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Prepare example data " 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "sentences = [['I', 'feel', 'hungry'],\n", 54 | " ['You', 'are', 'a', 'genius'],\n", 55 | " ['tensorflow', 'is', 'very', 'difficult'],\n", 56 | " ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n", 57 | " ['tensorflow', 'is', 'very', 'fast', 'changing']]\n", 58 | "pos = [['pronoun', 'verb', 'adjective'],\n", 59 | " ['pronoun', 'verb', 'preposition', 'noun'],\n", 60 | " ['noun', 'verb', 'adverb', 'adjective'],\n", 61 | " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n", 62 | " ['noun', 'verb', 'adverb', 'adjective', 'verb']]" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# word dictionary\n", 72 | "bag_of_words = []\n", 73 | "for sentence in sentences:\n", 74 | " bag_of_words += sentence\n", 75 | "bag_of_words = list(set(bag_of_words))\n", 76 | "bag_of_words.sort()\n", 77 | "bag_of_words = [''] + bag_of_words\n", 78 | "\n", 79 | "word2idx = {word : idx for idx, word in enumerate(bag_of_words)} # word to index\n", 80 | "idx2word = [word for word in bag_of_words] # index to word" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "#print(\"word2idx: {}\".format(word2idx))\n", 90 | "word2idx" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "#print(\"idx2word: {}\".format(idx2word))\n", 100 | "idx2word" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "# pos dictionary\n", 110 | "bag_of_pos = []\n", 111 | "for item in pos:\n", 112 | " bag_of_pos += item\n", 113 | "bag_of_pos = list(set(bag_of_pos))\n", 114 | "bag_of_pos.sort()\n", 115 | "bag_of_pos = [''] + bag_of_pos\n", 116 | "print(\"bag_of_pos: {}\".format(bag_of_pos))\n", 117 | "\n", 118 | "pos2idx = {pos : idx for idx, pos in enumerate(bag_of_pos)} # pos to index\n", 119 | "idx2pos = [pos for pos in bag_of_pos] # index to pos" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "#print(\"pos2idx: {}\".format(pos2idx))\n", 129 | "pos2idx" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "#print(\"idx2pos: {}\".format(idx2pos))\n", 139 | "idx2pos" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Create pad_seq function" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def pad_seq(sequences, max_length, dic):\n", 156 | " \"\"\"Padding sequences\n", 157 | " Padding a special charcter '' from the end of sentence to max_length\n", 158 | " \n", 159 | " Args:\n", 160 | " sequences (list of characters): input data\n", 161 | " max_length (int): max length for padding\n", 162 | " dic (dictionary): char to index\n", 163 | " \n", 164 | " Returns:\n", 165 | " seq_indices (2-rank np.array): \n", 166 | " seq_length (1-rank np.array): sequence lengthes of all data\n", 167 | " \"\"\"\n", 168 | " seq_length, seq_indices = [], []\n", 169 | " for sequence in sequences:\n", 170 | " seq_length.append(len(sequence))\n", 171 | " seq_idx = [dic.get(char) for char in sequence]\n", 172 | " seq_idx += (max_length - len(seq_idx)) * [dic.get('')] # 0 is idx of meaningless token \"\"\n", 173 | " seq_indices.append(seq_idx)\n", 174 | " return np.array(seq_indices), np.array(seq_length)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "### Pre-process data" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "max_length = 10\n", 191 | "X_indices, X_length = pad_seq(sequences=sentences, max_length=max_length, dic=word2idx)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "print(\"X_indices\")\n", 201 | "print(X_indices)\n", 202 | "print(\"X_length\")\n", 203 | "print(X_length)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "y_string = np.array([item + [''] * (max_length - len(item)) for item in pos])\n", 213 | "print(y_string)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "y = np.array([list(map(lambda el : pos2idx.get(el), item)) for item in y_string])\n", 223 | "print(y)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "### Define SimPosRNN" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "class PosRNN:\n", 240 | " def __init__(self, seq_indices, seq_length, labels, num_classes, hidden_dims, max_length, word2idx):\n", 241 | " # Data pipeline\n", 242 | " with tf.variable_scope('input_layer'):\n", 243 | " self._seq_indices = seq_indices\n", 244 | " self._seq_length = seq_length\n", 245 | " self._labels = labels\n", 246 | "\n", 247 | " one_hot = tf.eye(len(word2idx), dtype=tf.float32)\n", 248 | " self._one_hot = tf.get_variable(name='one_hot_embedding',\n", 249 | " initializer=one_hot,\n", 250 | " trainable=False) # embedding vector training 안할 것이기 때문\n", 251 | " self._seq_embeddings = tf.nn.embedding_lookup(params=self._one_hot,\n", 252 | " ids=self._seq_indices)\n", 253 | "\n", 254 | " # Multi RNN cell (many to many)\n", 255 | " with tf.variable_scope('rnn_cell'): \n", 256 | " multi_cells = rnn.MultiRNNCell([rnn.BasicRNNCell(hidden_dim) for hidden_dim in hidden_dims])\n", 257 | " score_cell = tf.contrib.rnn.OutputProjectionWrapper(cell=multi_cells,\n", 258 | " output_size=num_classes)\n", 259 | " self._outputs, _ = tf.nn.dynamic_rnn(cell=score_cell, \n", 260 | " inputs=self._seq_embeddings,\n", 261 | " sequence_length=self._seq_length,\n", 262 | " dtype=tf.float32)\n", 263 | " \n", 264 | " with tf.variable_scope('seq2seq_loss'):\n", 265 | " masks = tf.sequence_mask(lengths=self._seq_length, maxlen=max_length, dtype=tf.float32)\n", 266 | " self.seq2seq_loss = tf.contrib.seq2seq.sequence_loss(logits=self._outputs,\n", 267 | " targets=self._labels,\n", 268 | " weights=masks)\n", 269 | "\n", 270 | " with tf.variable_scope('prediction'):\n", 271 | " self._prediction = tf.argmax(input=self._outputs,\n", 272 | " axis=2, output_type=tf.int32)\n", 273 | "\n", 274 | " def predict(self, sess, seq_indices, seq_length):\n", 275 | " feed_dict = {self._seq_indices : seq_indices, self._seq_length : seq_length}\n", 276 | " return sess.run(self._prediction, feed_dict=feed_dict)" 277 | ] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "metadata": {}, 282 | "source": [ 283 | "### Create a model of SimPosRNN" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": null, 289 | "metadata": {}, 290 | "outputs": [], 291 | "source": [ 292 | "# hyper-parameter\n", 293 | "num_classes = len(idx2pos)\n", 294 | "learning_rate = .003\n", 295 | "batch_size = 2\n", 296 | "max_epochs = 100" 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": {}, 302 | "source": [ 303 | "### Set up dataset with `tf.data`\n", 304 | "\n", 305 | "#### create input pipeline with `tf.data.Dataset`" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": null, 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [ 314 | "## create data pipeline with tf.data\n", 315 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_indices, X_length, y))\n", 316 | "train_dataset = train_dataset.shuffle(buffer_size = 100)\n", 317 | "train_dataset = train_dataset.batch(batch_size = batch_size)\n", 318 | "print(train_dataset)" 319 | ] 320 | }, 321 | { 322 | "cell_type": "markdown", 323 | "metadata": {}, 324 | "source": [ 325 | "#### Define Iterator" 326 | ] 327 | }, 328 | { 329 | "cell_type": "code", 330 | "execution_count": null, 331 | "metadata": {}, 332 | "outputs": [], 333 | "source": [ 334 | "train_iterator = train_dataset.make_initializable_iterator()\n", 335 | "seq_indices, seq_length, labels = train_iterator.get_next()" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": null, 341 | "metadata": {}, 342 | "outputs": [], 343 | "source": [ 344 | "pos_rnn = PosRNN(seq_indices=seq_indices, seq_length=seq_length,\n", 345 | " labels=labels, num_classes=num_classes,\n", 346 | " hidden_dims=[32, 16], max_length=max_length,\n", 347 | " word2idx=word2idx)" 348 | ] 349 | }, 350 | { 351 | "cell_type": "markdown", 352 | "metadata": {}, 353 | "source": [ 354 | "### Creat training op and train model" 355 | ] 356 | }, 357 | { 358 | "cell_type": "code", 359 | "execution_count": null, 360 | "metadata": {}, 361 | "outputs": [], 362 | "source": [ 363 | "## create training op\n", 364 | "optimizer = tf.train.AdamOptimizer(learning_rate)\n", 365 | "train_op = optimizer.minimize(pos_rnn.seq2seq_loss)" 366 | ] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "metadata": {}, 371 | "source": [ 372 | "### `tf.Session()` and train" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": null, 378 | "metadata": {}, 379 | "outputs": [], 380 | "source": [ 381 | "sess = tf.Session()\n", 382 | "sess.run(tf.global_variables_initializer())\n", 383 | "\n", 384 | "loss_history = []\n", 385 | "step = 0\n", 386 | "for epochs in range(max_epochs):\n", 387 | " start_time = time.time()\n", 388 | " sess.run(train_iterator.initializer)\n", 389 | " \n", 390 | " avg_loss = []\n", 391 | " while True:\n", 392 | " try:\n", 393 | " _, loss_ = sess.run([train_op, pos_rnn.seq2seq_loss])\n", 394 | " avg_loss.append(loss_)\n", 395 | " step += 1\n", 396 | "\n", 397 | " except tf.errors.OutOfRangeError:\n", 398 | " #print(\"End of dataset\") # ==> \"End of dataset\"\n", 399 | " break\n", 400 | "\n", 401 | " avg_loss_ = np.mean(avg_loss)\n", 402 | " loss_history.append(avg_loss_)\n", 403 | " \n", 404 | " duration = time.time() - start_time\n", 405 | " examples_per_sec = batch_size / float(duration)\n", 406 | " print(\"epochs: {}, step: {}, loss: {:g}, ({:.2f} examples/sec; {:.3f} sec/batch)\".format(epochs+1, step, avg_loss_, examples_per_sec, duration))" 407 | ] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": null, 412 | "metadata": {}, 413 | "outputs": [], 414 | "source": [ 415 | "plt.plot(loss_history, label='train')" 416 | ] 417 | }, 418 | { 419 | "cell_type": "code", 420 | "execution_count": null, 421 | "metadata": {}, 422 | "outputs": [], 423 | "source": [ 424 | "y_pred = pos_rnn.predict(sess=sess, seq_indices=X_indices, seq_length=X_length)\n", 425 | "print(y_pred)" 426 | ] 427 | }, 428 | { 429 | "cell_type": "code", 430 | "execution_count": null, 431 | "metadata": {}, 432 | "outputs": [], 433 | "source": [ 434 | "result_str = []\n", 435 | "for example in y_pred:\n", 436 | " result_str.append([idx2pos[idx] for idx in example])\n", 437 | " \n", 438 | "for examples in zip(y_string, result_str):\n", 439 | " print(\" Label: \", ' '.join(examples[0]))\n", 440 | " print(\"Prediction: \", ' '.join(examples[1]))" 441 | ] 442 | } 443 | ], 444 | "metadata": { 445 | "kernelspec": { 446 | "display_name": "Python 3", 447 | "language": "python", 448 | "name": "python3" 449 | }, 450 | "language_info": { 451 | "codemirror_mode": { 452 | "name": "ipython", 453 | "version": 3 454 | }, 455 | "file_extension": ".py", 456 | "mimetype": "text/x-python", 457 | "name": "python", 458 | "nbconvert_exporter": "python", 459 | "pygments_lexer": "ipython3", 460 | "version": "3.6.5" 461 | } 462 | }, 463 | "nbformat": 4, 464 | "nbformat_minor": 2 465 | } 466 | -------------------------------------------------------------------------------- /tf.version.1/04.rnn/04.07.seq2seq.classification.Multi.LSTM.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sequence to Sequence Classification by RNN\n", 8 | "\n", 9 | "- Creating the **data pipeline** with `tf.data`\n", 10 | "- Preprocessing word sequences (variable input sequence length) using `padding technique` by `user function (pad_seq)`\n", 11 | "- Using `tf.nn.embedding_lookup` for getting vector of tokens (eg. word, character)\n", 12 | "- Training **many to many classification** with `tf.contrib.seq2seq.sequence_loss`\n", 13 | "- Masking unvalid token with `tf.sequence_mask`\n", 14 | "- Creating the model as **Class**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "import sys\n", 25 | "import time\n", 26 | "import string\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "import tensorflow as tf\n", 33 | "\n", 34 | "slim = tf.contrib.slim\n", 35 | "rnn = tf.contrib.rnn\n", 36 | "\n", 37 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Prepare example data " 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "sentences = [['I', 'feel', 'hungry'],\n", 54 | " ['You', 'are', 'a', 'genius'],\n", 55 | " ['tensorflow', 'is', 'very', 'difficult'],\n", 56 | " ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n", 57 | " ['tensorflow', 'is', 'very', 'fast', 'changing']]\n", 58 | "pos = [['pronoun', 'verb', 'adjective'],\n", 59 | " ['pronoun', 'verb', 'preposition', 'noun'],\n", 60 | " ['noun', 'verb', 'adverb', 'adjective'],\n", 61 | " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n", 62 | " ['noun', 'verb', 'adverb', 'adjective', 'verb']]" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# word dictionary\n", 72 | "bag_of_words = []\n", 73 | "for sentence in sentences:\n", 74 | " bag_of_words += sentence\n", 75 | "bag_of_words = list(set(bag_of_words))\n", 76 | "bag_of_words.sort()\n", 77 | "bag_of_words = [''] + bag_of_words\n", 78 | "\n", 79 | "word2idx = {word : idx for idx, word in enumerate(bag_of_words)} # word to index\n", 80 | "idx2word = [word for word in bag_of_words] # index to word" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "#print(\"word2idx: {}\".format(word2idx))\n", 90 | "word2idx" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "#print(\"idx2word: {}\".format(idx2word))\n", 100 | "idx2word" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "# pos dictionary\n", 110 | "bag_of_pos = []\n", 111 | "for item in pos:\n", 112 | " bag_of_pos += item\n", 113 | "bag_of_pos = list(set(bag_of_pos))\n", 114 | "bag_of_pos.sort()\n", 115 | "bag_of_pos = [''] + bag_of_pos\n", 116 | "print(\"bag_of_pos: {}\".format(bag_of_pos))\n", 117 | "\n", 118 | "pos2idx = {pos : idx for idx, pos in enumerate(bag_of_pos)} # pos to index\n", 119 | "idx2pos = [pos for pos in bag_of_pos] # index to pos" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "#print(\"pos2idx: {}\".format(pos2idx))\n", 129 | "pos2idx" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "#print(\"idx2pos: {}\".format(idx2pos))\n", 139 | "idx2pos" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Create pad_seq function" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def pad_seq(sequences, max_length, dic):\n", 156 | " \"\"\"Padding sequences\n", 157 | " Padding a special charcter '' from the end of sentence to max_length\n", 158 | " \n", 159 | " Args:\n", 160 | " sequences (list of characters): input data\n", 161 | " max_length (int): max length for padding\n", 162 | " dic (dictionary): char to index\n", 163 | " \n", 164 | " Returns:\n", 165 | " seq_indices (2-rank np.array): \n", 166 | " seq_length (1-rank np.array): sequence lengthes of all data\n", 167 | " \"\"\"\n", 168 | " seq_length, seq_indices = [], []\n", 169 | " for sequence in sequences:\n", 170 | " seq_length.append(len(sequence))\n", 171 | " seq_idx = [dic.get(char) for char in sequence]\n", 172 | " seq_idx += (max_length - len(seq_idx)) * [dic.get('')] # 0 is idx of meaningless token \"\"\n", 173 | " seq_indices.append(seq_idx)\n", 174 | " return np.array(seq_indices), np.array(seq_length)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "### Pre-process data" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "max_length = 10\n", 191 | "X_indices, X_length = pad_seq(sequences=sentences, max_length=max_length, dic=word2idx)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "print(\"X_indices\")\n", 201 | "print(X_indices)\n", 202 | "print(\"X_length\")\n", 203 | "print(X_length)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "y_string = np.array([item + [''] * (max_length - len(item)) for item in pos])\n", 213 | "print(y_string)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "y = np.array([list(map(lambda el : pos2idx.get(el), item)) for item in y_string])\n", 223 | "print(y)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "### Define SimPosRNN" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "class PosRNN:\n", 240 | " def __init__(self, seq_indices, seq_length, labels, num_classes, hidden_dims, max_length, word2idx):\n", 241 | " # Data pipeline\n", 242 | " with tf.variable_scope('input_layer'):\n", 243 | " self._seq_indices = seq_indices\n", 244 | " self._seq_length = seq_length\n", 245 | " self._labels = labels\n", 246 | "\n", 247 | " one_hot = tf.eye(len(word2idx), dtype=tf.float32)\n", 248 | " self._one_hot = tf.get_variable(name='one_hot_embedding',\n", 249 | " initializer=one_hot,\n", 250 | " trainable=False) # embedding vector training 안할 것이기 때문\n", 251 | " self._seq_embeddings = tf.nn.embedding_lookup(params=self._one_hot,\n", 252 | " ids=self._seq_indices)\n", 253 | "\n", 254 | " # Multi RNN cell (many to many)\n", 255 | " with tf.variable_scope('rnn_cell'): \n", 256 | " multi_cells = rnn.MultiRNNCell([rnn.BasicLSTMCell(num_units=hidden_dim, state_is_tuple=True) \\\n", 257 | " for hidden_dim in hidden_dims])\n", 258 | " score_cell = tf.contrib.rnn.OutputProjectionWrapper(cell=multi_cells,\n", 259 | " output_size=num_classes)\n", 260 | " self._outputs, _ = tf.nn.dynamic_rnn(cell=score_cell, \n", 261 | " inputs=self._seq_embeddings,\n", 262 | " sequence_length=self._seq_length,\n", 263 | " dtype=tf.float32)\n", 264 | " \n", 265 | " with tf.variable_scope('seq2seq_loss'):\n", 266 | " masks = tf.sequence_mask(lengths=self._seq_length, maxlen=max_length, dtype=tf.float32)\n", 267 | " self.seq2seq_loss = tf.contrib.seq2seq.sequence_loss(logits=self._outputs,\n", 268 | " targets=self._labels,\n", 269 | " weights=masks)\n", 270 | "\n", 271 | " with tf.variable_scope('prediction'):\n", 272 | " self._prediction = tf.argmax(input=self._outputs,\n", 273 | " axis=2, output_type=tf.int32)\n", 274 | "\n", 275 | " def predict(self, sess, seq_indices, seq_length):\n", 276 | " feed_dict = {self._seq_indices : seq_indices, self._seq_length : seq_length}\n", 277 | " return sess.run(self._prediction, feed_dict=feed_dict)" 278 | ] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "metadata": {}, 283 | "source": [ 284 | "### Create a model of SimPosRNN" 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": null, 290 | "metadata": {}, 291 | "outputs": [], 292 | "source": [ 293 | "# hyper-parameter\n", 294 | "num_classes = len(idx2pos)\n", 295 | "learning_rate = .003\n", 296 | "batch_size = 2\n", 297 | "max_epochs = 100" 298 | ] 299 | }, 300 | { 301 | "cell_type": "markdown", 302 | "metadata": {}, 303 | "source": [ 304 | "### Set up dataset with `tf.data`\n", 305 | "\n", 306 | "#### create input pipeline with `tf.data.Dataset`" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": null, 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [ 315 | "## create data pipeline with tf.data\n", 316 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_indices, X_length, y))\n", 317 | "train_dataset = train_dataset.shuffle(buffer_size = 100)\n", 318 | "train_dataset = train_dataset.batch(batch_size = batch_size)\n", 319 | "print(train_dataset)" 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": {}, 325 | "source": [ 326 | "#### Define Iterator" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [ 335 | "train_iterator = train_dataset.make_initializable_iterator()\n", 336 | "seq_indices, seq_length, labels = train_iterator.get_next()" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": null, 342 | "metadata": {}, 343 | "outputs": [], 344 | "source": [ 345 | "pos_rnn = PosRNN(seq_indices=seq_indices, seq_length=seq_length,\n", 346 | " labels=labels, num_classes=num_classes,\n", 347 | " hidden_dims=[32, 16], max_length=max_length,\n", 348 | " word2idx=word2idx)" 349 | ] 350 | }, 351 | { 352 | "cell_type": "markdown", 353 | "metadata": {}, 354 | "source": [ 355 | "### Creat training op and train model" 356 | ] 357 | }, 358 | { 359 | "cell_type": "code", 360 | "execution_count": null, 361 | "metadata": {}, 362 | "outputs": [], 363 | "source": [ 364 | "## create training op\n", 365 | "optimizer = tf.train.AdamOptimizer(learning_rate)\n", 366 | "train_op = optimizer.minimize(pos_rnn.seq2seq_loss)" 367 | ] 368 | }, 369 | { 370 | "cell_type": "markdown", 371 | "metadata": {}, 372 | "source": [ 373 | "### `tf.Session()` and train" 374 | ] 375 | }, 376 | { 377 | "cell_type": "code", 378 | "execution_count": null, 379 | "metadata": {}, 380 | "outputs": [], 381 | "source": [ 382 | "sess = tf.Session()\n", 383 | "sess.run(tf.global_variables_initializer())\n", 384 | "\n", 385 | "loss_history = []\n", 386 | "step = 0\n", 387 | "for epochs in range(max_epochs):\n", 388 | " start_time = time.time()\n", 389 | " sess.run(train_iterator.initializer)\n", 390 | " \n", 391 | " avg_loss = []\n", 392 | " while True:\n", 393 | " try:\n", 394 | " _, loss_ = sess.run([train_op, pos_rnn.seq2seq_loss])\n", 395 | " avg_loss.append(loss_)\n", 396 | " step += 1\n", 397 | "\n", 398 | " except tf.errors.OutOfRangeError:\n", 399 | " #print(\"End of dataset\") # ==> \"End of dataset\"\n", 400 | " break\n", 401 | "\n", 402 | " avg_loss_ = np.mean(avg_loss)\n", 403 | " loss_history.append(avg_loss_)\n", 404 | " \n", 405 | " duration = time.time() - start_time\n", 406 | " examples_per_sec = batch_size / float(duration)\n", 407 | " print(\"epochs: {}, step: {}, loss: {:g}, ({:.2f} examples/sec; {:.3f} sec/batch)\".format(epochs+1, step, avg_loss_, examples_per_sec, duration))" 408 | ] 409 | }, 410 | { 411 | "cell_type": "code", 412 | "execution_count": null, 413 | "metadata": {}, 414 | "outputs": [], 415 | "source": [ 416 | "plt.plot(loss_history, label='train')" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": null, 422 | "metadata": {}, 423 | "outputs": [], 424 | "source": [ 425 | "y_pred = pos_rnn.predict(sess=sess, seq_indices=X_indices, seq_length=X_length)\n", 426 | "print(y_pred)" 427 | ] 428 | }, 429 | { 430 | "cell_type": "code", 431 | "execution_count": null, 432 | "metadata": {}, 433 | "outputs": [], 434 | "source": [ 435 | "result_str = []\n", 436 | "for example in y_pred:\n", 437 | " result_str.append([idx2pos[idx] for idx in example])\n", 438 | " \n", 439 | "for examples in zip(y_string, result_str):\n", 440 | " print(\" Label: \", ' '.join(examples[0]))\n", 441 | " print(\"Prediction: \", ' '.join(examples[1]))" 442 | ] 443 | } 444 | ], 445 | "metadata": { 446 | "kernelspec": { 447 | "display_name": "Python 3", 448 | "language": "python", 449 | "name": "python3" 450 | }, 451 | "language_info": { 452 | "codemirror_mode": { 453 | "name": "ipython", 454 | "version": 3 455 | }, 456 | "file_extension": ".py", 457 | "mimetype": "text/x-python", 458 | "name": "python", 459 | "nbconvert_exporter": "python", 460 | "pygments_lexer": "ipython3", 461 | "version": "3.6.5" 462 | } 463 | }, 464 | "nbformat": 4, 465 | "nbformat_minor": 2 466 | } 467 | -------------------------------------------------------------------------------- /tf.version.1/04.rnn/04.08.seq2seq.classification.Multi.RNN.dropout.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sequence to Sequence Classification by RNN\n", 8 | "\n", 9 | "- Creating the **data pipeline** with `tf.data`\n", 10 | "- Preprocessing word sequences (variable input sequence length) using `padding technique` by `user function (pad_seq)`\n", 11 | "- Using `tf.nn.embedding_lookup` for getting vector of tokens (eg. word, character)\n", 12 | "- Training **many to many classification** with `tf.contrib.seq2seq.sequence_loss`\n", 13 | "- Masking unvalid token with `tf.sequence_mask`\n", 14 | "- Creating the model as **Class**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "import sys\n", 25 | "import time\n", 26 | "import string\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "import tensorflow as tf\n", 33 | "\n", 34 | "slim = tf.contrib.slim\n", 35 | "rnn = tf.contrib.rnn\n", 36 | "\n", 37 | "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Prepare example data " 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "sentences = [['I', 'feel', 'hungry'],\n", 54 | " ['You', 'are', 'a', 'genius'],\n", 55 | " ['tensorflow', 'is', 'very', 'difficult'],\n", 56 | " ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n", 57 | " ['tensorflow', 'is', 'very', 'fast', 'changing']]\n", 58 | "pos = [['pronoun', 'verb', 'adjective'],\n", 59 | " ['pronoun', 'verb', 'preposition', 'noun'],\n", 60 | " ['noun', 'verb', 'adverb', 'adjective'],\n", 61 | " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n", 62 | " ['noun', 'verb', 'adverb', 'adjective', 'verb']]" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# word dictionary\n", 72 | "bag_of_words = []\n", 73 | "for sentence in sentences:\n", 74 | " bag_of_words += sentence\n", 75 | "bag_of_words = list(set(bag_of_words))\n", 76 | "bag_of_words.sort()\n", 77 | "bag_of_words = [''] + bag_of_words\n", 78 | "\n", 79 | "word2idx = {word : idx for idx, word in enumerate(bag_of_words)} # word to index\n", 80 | "idx2word = [word for word in bag_of_words] # index to word" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "#print(\"word2idx: {}\".format(word2idx))\n", 90 | "word2idx" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "#print(\"idx2word: {}\".format(idx2word))\n", 100 | "idx2word" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "# pos dictionary\n", 110 | "bag_of_pos = []\n", 111 | "for item in pos:\n", 112 | " bag_of_pos += item\n", 113 | "bag_of_pos = list(set(bag_of_pos))\n", 114 | "bag_of_pos.sort()\n", 115 | "bag_of_pos = [''] + bag_of_pos\n", 116 | "print(\"bag_of_pos: {}\".format(bag_of_pos))\n", 117 | "\n", 118 | "pos2idx = {pos : idx for idx, pos in enumerate(bag_of_pos)} # pos to index\n", 119 | "idx2pos = [pos for pos in bag_of_pos] # index to pos" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "#print(\"pos2idx: {}\".format(pos2idx))\n", 129 | "pos2idx" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "#print(\"idx2pos: {}\".format(idx2pos))\n", 139 | "idx2pos" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Create pad_seq function" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def pad_seq(sequences, max_length, dic):\n", 156 | " \"\"\"Padding sequences\n", 157 | " Padding a special charcter '' from the end of sentence to max_length\n", 158 | " \n", 159 | " Args:\n", 160 | " sequences (list of characters): input data\n", 161 | " max_length (int): max length for padding\n", 162 | " dic (dictionary): char to index\n", 163 | " \n", 164 | " Returns:\n", 165 | " seq_indices (2-rank np.array): \n", 166 | " seq_length (1-rank np.array): sequence lengthes of all data\n", 167 | " \"\"\"\n", 168 | " seq_length, seq_indices = [], []\n", 169 | " for sequence in sequences:\n", 170 | " seq_length.append(len(sequence))\n", 171 | " seq_idx = [dic.get(char) for char in sequence]\n", 172 | " seq_idx += (max_length - len(seq_idx)) * [dic.get('')] # 0 is idx of meaningless token \"\"\n", 173 | " seq_indices.append(seq_idx)\n", 174 | " return np.array(seq_indices), np.array(seq_length)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "### Pre-process data" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "max_length = 10\n", 191 | "X_indices, X_length = pad_seq(sequences=sentences, max_length=max_length, dic=word2idx)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "print(\"X_indices\")\n", 201 | "print(X_indices)\n", 202 | "print(\"X_length\")\n", 203 | "print(X_length)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "y_string = np.array([item + [''] * (max_length - len(item)) for item in pos])\n", 213 | "print(y_string)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "y = np.array([list(map(lambda el : pos2idx.get(el), item)) for item in y_string])\n", 223 | "print(y)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "### Define SimPosRNN" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "class PosRNN:\n", 240 | " def __init__(self, seq_indices, seq_length, labels, num_classes, hidden_dims, max_length, word2idx):\n", 241 | " # Data pipeline\n", 242 | " with tf.variable_scope('input_layer'):\n", 243 | " self._seq_indices = seq_indices\n", 244 | " self._seq_length = seq_length\n", 245 | " self._labels = labels\n", 246 | " self._keep_prob = tf.placeholder(tf.float32)\n", 247 | "\n", 248 | " one_hot = tf.eye(len(word2idx), dtype=tf.float32)\n", 249 | " self._one_hot = tf.get_variable(name='one_hot_embedding',\n", 250 | " initializer=one_hot,\n", 251 | " trainable=False) # embedding vector training 안할 것이기 때문\n", 252 | " self._seq_embeddings = tf.nn.embedding_lookup(params=self._one_hot,\n", 253 | " ids=self._seq_indices)\n", 254 | "\n", 255 | " # Multi RNN cell (many to many)\n", 256 | " with tf.variable_scope('rnn_cell'):\n", 257 | " multi_cells = []\n", 258 | " for hidden_dim in hidden_dims:\n", 259 | " cell = rnn.BasicRNNCell(num_units=hidden_dim)\n", 260 | " cell = rnn.DropoutWrapper(cell=cell, output_keep_prob=self._keep_prob)\n", 261 | " multi_cells.append(cell)\n", 262 | " multi_cells = rnn.MultiRNNCell(cells=multi_cells)\n", 263 | " \n", 264 | " score_cell = rnn.OutputProjectionWrapper(cell=multi_cells,\n", 265 | " output_size=num_classes)\n", 266 | " self._outputs, _ = tf.nn.dynamic_rnn(cell=score_cell,\n", 267 | " inputs=self._seq_embeddings,\n", 268 | " sequence_length=self._seq_length,\n", 269 | " dtype=tf.float32)\n", 270 | " \n", 271 | " with tf.variable_scope('seq2seq_loss'):\n", 272 | " masks = tf.sequence_mask(lengths=self._seq_length, maxlen=max_length, dtype=tf.float32)\n", 273 | " self.seq2seq_loss = tf.contrib.seq2seq.sequence_loss(logits=self._outputs,\n", 274 | " targets=self._labels,\n", 275 | " weights=masks)\n", 276 | "\n", 277 | " with tf.variable_scope('prediction'):\n", 278 | " self._prediction = tf.argmax(input=self._outputs,\n", 279 | " axis=2, output_type=tf.int32)\n", 280 | "\n", 281 | " def predict(self, sess, seq_indices, seq_length):\n", 282 | " feed_dict = {self._seq_indices : seq_indices, self._seq_length : seq_length, self._keep_prob : 1.0}\n", 283 | " return sess.run(self._prediction, feed_dict=feed_dict)" 284 | ] 285 | }, 286 | { 287 | "cell_type": "markdown", 288 | "metadata": {}, 289 | "source": [ 290 | "### Create a model of SimPosRNN" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": {}, 297 | "outputs": [], 298 | "source": [ 299 | "# hyper-parameter\n", 300 | "num_classes = len(idx2pos)\n", 301 | "learning_rate = .003\n", 302 | "batch_size = 2\n", 303 | "max_epochs = 100" 304 | ] 305 | }, 306 | { 307 | "cell_type": "markdown", 308 | "metadata": {}, 309 | "source": [ 310 | "### Set up dataset with `tf.data`\n", 311 | "\n", 312 | "#### create input pipeline with `tf.data.Dataset`" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": null, 318 | "metadata": {}, 319 | "outputs": [], 320 | "source": [ 321 | "## create data pipeline with tf.data\n", 322 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_indices, X_length, y))\n", 323 | "train_dataset = train_dataset.shuffle(buffer_size = 100)\n", 324 | "train_dataset = train_dataset.batch(batch_size = batch_size)\n", 325 | "print(train_dataset)" 326 | ] 327 | }, 328 | { 329 | "cell_type": "markdown", 330 | "metadata": {}, 331 | "source": [ 332 | "#### Define Iterator" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": null, 338 | "metadata": {}, 339 | "outputs": [], 340 | "source": [ 341 | "train_iterator = train_dataset.make_initializable_iterator()\n", 342 | "seq_indices, seq_length, labels = train_iterator.get_next()" 343 | ] 344 | }, 345 | { 346 | "cell_type": "code", 347 | "execution_count": null, 348 | "metadata": {}, 349 | "outputs": [], 350 | "source": [ 351 | "pos_rnn = PosRNN(seq_indices=seq_indices, seq_length=seq_length,\n", 352 | " labels=labels, num_classes=num_classes,\n", 353 | " hidden_dims=[32, 16], max_length=max_length,\n", 354 | " word2idx=word2idx)" 355 | ] 356 | }, 357 | { 358 | "cell_type": "markdown", 359 | "metadata": {}, 360 | "source": [ 361 | "### Creat training op and train model" 362 | ] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "execution_count": null, 367 | "metadata": {}, 368 | "outputs": [], 369 | "source": [ 370 | "## create training op\n", 371 | "optimizer = tf.train.AdamOptimizer(learning_rate)\n", 372 | "train_op = optimizer.minimize(pos_rnn.seq2seq_loss)" 373 | ] 374 | }, 375 | { 376 | "cell_type": "markdown", 377 | "metadata": {}, 378 | "source": [ 379 | "### `tf.Session()` and train" 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": null, 385 | "metadata": {}, 386 | "outputs": [], 387 | "source": [ 388 | "sess = tf.Session()\n", 389 | "sess.run(tf.global_variables_initializer())\n", 390 | "\n", 391 | "loss_history = []\n", 392 | "step = 0\n", 393 | "for epochs in range(max_epochs):\n", 394 | " start_time = time.time()\n", 395 | " sess.run(train_iterator.initializer)\n", 396 | " \n", 397 | " avg_loss = []\n", 398 | " while True:\n", 399 | " try:\n", 400 | " _, loss_ = sess.run([train_op, pos_rnn.seq2seq_loss],\n", 401 | " feed_dict={pos_rnn._keep_prob: 0.5})\n", 402 | " avg_loss.append(loss_)\n", 403 | " step += 1\n", 404 | "\n", 405 | " except tf.errors.OutOfRangeError:\n", 406 | " #print(\"End of dataset\") # ==> \"End of dataset\"\n", 407 | " break\n", 408 | "\n", 409 | " avg_loss_ = np.mean(avg_loss)\n", 410 | " loss_history.append(avg_loss_)\n", 411 | " \n", 412 | " duration = time.time() - start_time\n", 413 | " examples_per_sec = batch_size / float(duration)\n", 414 | " print(\"epochs: {}, step: {}, loss: {:g}, ({:.2f} examples/sec; {:.3f} sec/batch)\".format(epochs+1, step, avg_loss_, examples_per_sec, duration))" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "metadata": {}, 421 | "outputs": [], 422 | "source": [ 423 | "plt.plot(loss_history, label='train')" 424 | ] 425 | }, 426 | { 427 | "cell_type": "code", 428 | "execution_count": null, 429 | "metadata": {}, 430 | "outputs": [], 431 | "source": [ 432 | "y_pred = pos_rnn.predict(sess=sess, seq_indices=X_indices, seq_length=X_length)\n", 433 | "print(y_pred)" 434 | ] 435 | }, 436 | { 437 | "cell_type": "code", 438 | "execution_count": null, 439 | "metadata": {}, 440 | "outputs": [], 441 | "source": [ 442 | "result_str = []\n", 443 | "for example in y_pred:\n", 444 | " result_str.append([idx2pos[idx] for idx in example])\n", 445 | " \n", 446 | "for examples in zip(y_string, result_str):\n", 447 | " print(\" Label: \", ' '.join(examples[0]))\n", 448 | " print(\"Prediction: \", ' '.join(examples[1]))" 449 | ] 450 | } 451 | ], 452 | "metadata": { 453 | "kernelspec": { 454 | "display_name": "Python 3", 455 | "language": "python", 456 | "name": "python3" 457 | }, 458 | "language_info": { 459 | "codemirror_mode": { 460 | "name": "ipython", 461 | "version": 3 462 | }, 463 | "file_extension": ".py", 464 | "mimetype": "text/x-python", 465 | "name": "python", 466 | "nbconvert_exporter": "python", 467 | "pygments_lexer": "ipython3", 468 | "version": "3.6.5" 469 | } 470 | }, 471 | "nbformat": 4, 472 | "nbformat_minor": 2 473 | } 474 | -------------------------------------------------------------------------------- /tf.version.1/07.sequences/download.py: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # 3 | # Functions for downloading and extracting data-files from the internet. 4 | # 5 | # Implemented in Python 3.5 6 | # 7 | ######################################################################## 8 | # 9 | # This file is part of the TensorFlow Tutorials available at: 10 | # 11 | # https://github.com/Hvass-Labs/TensorFlow-Tutorials 12 | # 13 | # Published under the MIT License. See the file LICENSE for details. 14 | # 15 | # Copyright 2016 by Magnus Erik Hvass Pedersen 16 | # 17 | ######################################################################## 18 | 19 | import sys 20 | import os 21 | import urllib.request 22 | import tarfile 23 | import zipfile 24 | 25 | ######################################################################## 26 | 27 | 28 | def _print_download_progress(count, block_size, total_size): 29 | """ 30 | Function used for printing the download progress. 31 | Used as a call-back function in maybe_download_and_extract(). 32 | """ 33 | 34 | # Percentage completion. 35 | pct_complete = float(count * block_size) / total_size 36 | 37 | # Limit it because rounding errors may cause it to exceed 100%. 38 | pct_complete = min(1.0, pct_complete) 39 | 40 | # Status-message. Note the \r which means the line should overwrite itself. 41 | msg = "\r- Download progress: {0:.1%}".format(pct_complete) 42 | 43 | # Print it. 44 | sys.stdout.write(msg) 45 | sys.stdout.flush() 46 | 47 | 48 | ######################################################################## 49 | 50 | def download(base_url, filename, download_dir): 51 | """ 52 | Download the given file if it does not already exist in the download_dir. 53 | 54 | :param base_url: The internet URL without the filename. 55 | :param filename: The filename that will be added to the base_url. 56 | :param download_dir: Local directory for storing the file. 57 | :return: Nothing. 58 | """ 59 | 60 | # Path for local file. 61 | save_path = os.path.join(download_dir, filename) 62 | 63 | # Check if the file already exists, otherwise we need to download it now. 64 | if not os.path.exists(save_path): 65 | # Check if the download directory exists, otherwise create it. 66 | if not os.path.exists(download_dir): 67 | os.makedirs(download_dir) 68 | 69 | print("Downloading", filename, "...") 70 | 71 | # Download the file from the internet. 72 | url = base_url + filename 73 | file_path, _ = urllib.request.urlretrieve(url=url, 74 | filename=save_path, 75 | reporthook=_print_download_progress) 76 | 77 | print(" Done!") 78 | 79 | 80 | def maybe_download_and_extract(url, download_dir): 81 | """ 82 | Download and extract the data if it doesn't already exist. 83 | Assumes the url is a tar-ball file. 84 | 85 | :param url: 86 | Internet URL for the tar-file to download. 87 | Example: "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" 88 | 89 | :param download_dir: 90 | Directory where the downloaded file is saved. 91 | Example: "data/CIFAR-10/" 92 | 93 | :return: 94 | Nothing. 95 | """ 96 | 97 | # Filename for saving the file downloaded from the internet. 98 | # Use the filename from the URL and add it to the download_dir. 99 | filename = url.split('/')[-1] 100 | file_path = os.path.join(download_dir, filename) 101 | 102 | # Check if the file already exists. 103 | # If it exists then we assume it has also been extracted, 104 | # otherwise we need to download and extract it now. 105 | if not os.path.exists(file_path): 106 | # Check if the download directory exists, otherwise create it. 107 | if not os.path.exists(download_dir): 108 | os.makedirs(download_dir) 109 | 110 | # Download the file from the internet. 111 | file_path, _ = urllib.request.urlretrieve(url=url, 112 | filename=file_path, 113 | reporthook=_print_download_progress) 114 | 115 | print() 116 | print("Download finished. Extracting files.") 117 | 118 | if file_path.endswith(".zip"): 119 | # Unpack the zip-file. 120 | zipfile.ZipFile(file=file_path, mode="r").extractall(download_dir) 121 | elif file_path.endswith((".tar.gz", ".tgz")): 122 | # Unpack the tar-ball. 123 | tarfile.open(name=file_path, mode="r:gz").extractall(download_dir) 124 | 125 | print("Done.") 126 | else: 127 | print("Data has apparently already been downloaded and unpacked.") 128 | 129 | 130 | ######################################################################## 131 | -------------------------------------------------------------------------------- /tf.version.1/07.sequences/imdb.py: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # 3 | # Functions for downloading the IMDB Review data-set from the internet 4 | # and loading it into memory. 5 | # 6 | # Implemented in Python 3.6 7 | # 8 | # Usage: 9 | # 1) Set the variable data_dir with the desired storage directory. 10 | # 2) Call maybe_download_and_extract() to download the data-set 11 | # if it is not already located in the given data_dir. 12 | # 3) Call load_data(train=True) to load the training-set. 13 | # 4) Call load_data(train=False) to load the test-set. 14 | # 5) Use the returned data in your own program. 15 | # 16 | # Format: 17 | # The IMDB Review data-set consists of 50000 reviews of movies 18 | # that are split into 25000 reviews for the training- and test-set, 19 | # and each of those is split into 12500 positive and 12500 negative reviews. 20 | # These are returned as lists of strings by the load_data() function. 21 | # 22 | ######################################################################## 23 | # 24 | # This file is part of the TensorFlow Tutorials available at: 25 | # 26 | # https://github.com/Hvass-Labs/TensorFlow-Tutorials 27 | # 28 | # Published under the MIT License. See the file LICENSE for details. 29 | # 30 | # Copyright 2018 by Magnus Erik Hvass Pedersen 31 | # 32 | ######################################################################## 33 | 34 | import os 35 | import download 36 | import glob 37 | 38 | ######################################################################## 39 | 40 | # Directory where you want to download and save the data-set. 41 | # Set this before you start calling any of the functions below. 42 | data_dir = "data/IMDB/" 43 | 44 | # URL for the data-set on the internet. 45 | data_url = "http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz" 46 | 47 | 48 | ######################################################################## 49 | # Private helper-functions. 50 | 51 | def _read_text_file(path): 52 | """ 53 | Read and return all the contents of the text-file with the given path. 54 | It is returned as a single string where all lines are concatenated. 55 | """ 56 | 57 | with open(path, 'rt') as file: 58 | # Read a list of strings. 59 | lines = file.readlines() 60 | 61 | # Concatenate to a single string. 62 | text = " ".join(lines) 63 | 64 | return text 65 | 66 | 67 | ######################################################################## 68 | # Public functions that you may call to download the data-set from 69 | # the internet and load the data into memory. 70 | 71 | 72 | def maybe_download_and_extract(): 73 | """ 74 | Download and extract the IMDB Review data-set if it doesn't already exist 75 | in data_dir (set this variable first to the desired directory). 76 | """ 77 | 78 | download.maybe_download_and_extract(url=data_url, download_dir=data_dir) 79 | 80 | 81 | def load_data(train=True): 82 | """ 83 | Load all the data from the IMDB Review data-set for sentiment analysis. 84 | 85 | :param train: Boolean whether to load the training-set (True) 86 | or the test-set (False). 87 | 88 | :return: A list of all the reviews as text-strings, 89 | and a list of the corresponding sentiments 90 | where 1.0 is positive and 0.0 is negative. 91 | """ 92 | 93 | # Part of the path-name for either training or test-set. 94 | train_test_path = "train" if train else "test" 95 | 96 | # Base-directory where the extracted data is located. 97 | dir_base = os.path.join(data_dir, "aclImdb", train_test_path) 98 | 99 | # Filename-patterns for the data-files. 100 | path_pattern_pos = os.path.join(dir_base, "pos", "*.txt") 101 | path_pattern_neg = os.path.join(dir_base, "neg", "*.txt") 102 | 103 | # Get lists of all the file-paths for the data. 104 | paths_pos = glob.glob(path_pattern_pos) 105 | paths_neg = glob.glob(path_pattern_neg) 106 | 107 | # Read all the text-files. 108 | data_pos = [_read_text_file(path) for path in paths_pos] 109 | data_neg = [_read_text_file(path) for path in paths_neg] 110 | 111 | # Concatenate the positive and negative data. 112 | x = data_pos + data_neg 113 | 114 | # Create a list of the sentiments for the text-data. 115 | # 1.0 is a positive sentiment, 0.0 is a negative sentiment. 116 | y = [1.0] * len(data_pos) + [0.0] * len(data_neg) 117 | 118 | return x, y 119 | 120 | 121 | ######################################################################## 122 | -------------------------------------------------------------------------------- /tf.version.2/01.tf.basic/02.guideline.eager.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Guideline on Eager Execution\n", 8 | "\n", 9 | "* 이 코드는 [TensorFlow official Guide `eager execution` 문서](https://www.tensorflow.org/guide/eager)를 정리한 것이다.\n", 10 | "\n", 11 | "[Eager execution](https://www.tensorflow.org/guide/eager#build_a_model) is a flexible machine learning platform for research and experimentation, providing:\n", 12 | "\n", 13 | "* **An intuitive interface**: Structure your code naturally and use Python data structures. Quickly iterate on small models and small data.\n", 14 | "* **Easier debugging**: Call ops directly to inspect running models and test changes. Use standard Python debugging tools for immediate error reporting.\n", 15 | "* **Natural control flow**: Use Python control flow instead of graph control flow, simplifying the specification of dynamic models." 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": null, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "from __future__ import absolute_import\n", 25 | "from __future__ import division\n", 26 | "from __future__ import print_function\n", 27 | "from __future__ import unicode_literals\n", 28 | "\n", 29 | "import os\n", 30 | "import numpy as np\n", 31 | "import matplotlib.pyplot as plt\n", 32 | "%matplotlib inline\n", 33 | "\n", 34 | "import tensorflow as tf\n", 35 | "from tensorflow.keras import layers\n", 36 | "tf.enable_eager_execution()\n", 37 | "\n", 38 | "print(tf.VERSION)" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "## Guideline on building a model\n", 46 | "\n", 47 | "* Use only [`tf.keras.layers`](https://www.tensorflow.org/api_docs/python/tf/keras/layers), do **NOT** use `tf.layers` or `tf.contrib.layers`." 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "### Example of a layer made myself" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "class MySimpleLayer(tf.keras.layers.Layer):\n", 64 | " def __init__(self, output_units):\n", 65 | " super(MySimpleLayer, self).__init__()\n", 66 | " self.output_units = output_units\n", 67 | "\n", 68 | " def build(self, input_shape):\n", 69 | " # The build method gets called the first time your layer is used.\n", 70 | " # Creating variables on build() allows you to make their shape depend\n", 71 | " # on the input shape and hence removes the need for the user to specify\n", 72 | " # full shapes. It is possible to create variables during __init__() if\n", 73 | " # you already know their full shapes.\n", 74 | " self.kernel = self.add_variable(\n", 75 | " \"kernel\", [input_shape[-1], self.output_units])\n", 76 | "\n", 77 | " def call(self, input):\n", 78 | " # Override call() instead of __call__ so we can perform some bookkeeping.\n", 79 | " return tf.matmul(input, self.kernel)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "### Simple method of building a model\n", 87 | "* Use [`tf.keras.Sequential`](https://www.tensorflow.org/api_docs/python/tf/keras/models/Sequential)\n", 88 | "* Must set input shape" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "model = tf.keras.Sequential([\n", 98 | " layers.Dense(10, input_shape=(784,)), # must declare input shape\n", 99 | " layers.Dense(10)\n", 100 | "])" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "### Another method of builing a model\n", 108 | "\n", 109 | "* Organize models in classes by inheriting from [`tf.keras.Model`](https://www.tensorflow.org/api_docs/python/tf/keras/models/Model).\n", 110 | "* It's not required to set an input shape for the [`tf.keras.Model`](https://www.tensorflow.org/api_docs/python/tf/keras/models/Model) class since the parameters are set the first time input is passed to the layer.\n", 111 | "* [`tf.keras.layers`](https://www.tensorflow.org/api_docs/python/tf/keras/layers) classes create and contain their own model variables that are tied to the lifetime of their layer objects. To share layer variables, share their objects." 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "metadata": {}, 118 | "outputs": [], 119 | "source": [ 120 | "class MNISTModel(tf.keras.Model):\n", 121 | " def __init__(self):\n", 122 | " super(MNISTModel, self).__init__()\n", 123 | " self.dense1 = layers.Dense(units=10)\n", 124 | " self.dense2 = layers.Dense(units=10)\n", 125 | "\n", 126 | " def call(self, input):\n", 127 | " \"\"\"Run the model.\"\"\"\n", 128 | " result = self.dense1(input)\n", 129 | " result = self.dense2(result)\n", 130 | " result = self.dense2(result) # reuse variables from dense2 layer\n", 131 | " return result\n", 132 | "\n", 133 | "model = MNISTModel()" 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": {}, 139 | "source": [ 140 | "## Eager Training" 141 | ] 142 | }, 143 | { 144 | "cell_type": "markdown", 145 | "metadata": {}, 146 | "source": [ 147 | "### Computing gradients\n", 148 | "* Use [`tf.GradientTape`](https://www.tensorflow.org/api_docs/python/tf/GradientTape)" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": null, 154 | "metadata": {}, 155 | "outputs": [], 156 | "source": [ 157 | "w = tf.Variable([[1.0]])\n", 158 | "with tf.GradientTape() as tape:\n", 159 | " loss = w * w\n", 160 | "\n", 161 | "grad = tape.gradient(loss, w)\n", 162 | "print(grad) # => tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "### Train a model" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "# Fetch and format the mnist data\n", 179 | "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n", 180 | "\n", 181 | "dataset = tf.data.Dataset.from_tensor_slices(\n", 182 | " (tf.cast(mnist_images[..., tf.newaxis]/255., tf.float32), # [..., tf.newaxis] for just expand_dims(last_axis)\n", 183 | " tf.cast(mnist_labels,tf.int64)))\n", 184 | "dataset = dataset.shuffle(1000).batch(32)" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "# Build the model\n", 194 | "mnist_model = tf.keras.Sequential([\n", 195 | " layers.Conv2D(16, [3,3], activation='relu'),\n", 196 | " layers.Conv2D(16, [3,3], activation='relu'),\n", 197 | " layers.GlobalAveragePooling2D(),\n", 198 | " layers.Dense(10)\n", 199 | "])" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "# without training, just inference a model in eager execution:\n", 209 | "for images,labels in dataset.take(1):\n", 210 | " print(\"Logits: \", mnist_model(images[0:1]).numpy())" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": null, 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "# Train the model\n", 220 | "optimizer = tf.train.AdamOptimizer()\n", 221 | "loss_history = []" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": null, 227 | "metadata": {}, 228 | "outputs": [], 229 | "source": [ 230 | "for (batch, (images, labels)) in enumerate(dataset.take(400)): # just 400 steps (iterations), NOT epochs\n", 231 | " if batch % 80 == 0:\n", 232 | " print()\n", 233 | " print('.', end='')\n", 234 | " with tf.GradientTape() as tape:\n", 235 | " logits = mnist_model(images, training=True)\n", 236 | " loss_value = tf.losses.sparse_softmax_cross_entropy(labels, logits)\n", 237 | "\n", 238 | " loss_history.append(loss_value.numpy())\n", 239 | " grads = tape.gradient(loss_value, mnist_model.variables)\n", 240 | " optimizer.apply_gradients(zip(grads, mnist_model.variables),\n", 241 | " global_step=tf.train.get_or_create_global_step())" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "plt.plot(loss_history)\n", 251 | "plt.xlabel('Batch #')\n", 252 | "plt.ylabel('Loss [entropy]')\n", 253 | "plt.show()" 254 | ] 255 | }, 256 | { 257 | "cell_type": "markdown", 258 | "metadata": {}, 259 | "source": [ 260 | "### Train a model made myself" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "class Model(tf.keras.Model):\n", 270 | " def __init__(self):\n", 271 | " super(Model, self).__init__()\n", 272 | " self.W = tf.Variable(5., name='weight')\n", 273 | " self.B = tf.Variable(10., name='bias')\n", 274 | "\n", 275 | " def call(self, inputs):\n", 276 | " return inputs * self.W + self.B" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": null, 282 | "metadata": {}, 283 | "outputs": [], 284 | "source": [ 285 | "# A toy dataset of points around 3 * x + 2\n", 286 | "NUM_EXAMPLES = 2000\n", 287 | "training_inputs = tf.random_normal([NUM_EXAMPLES])\n", 288 | "noise = tf.random_normal([NUM_EXAMPLES])\n", 289 | "training_outputs = training_inputs * 3 + 2 + noise" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": null, 295 | "metadata": {}, 296 | "outputs": [], 297 | "source": [ 298 | "# The loss function to be optimized\n", 299 | "def loss(model, inputs, targets):\n", 300 | " error = model(inputs) - targets\n", 301 | " return tf.reduce_mean(tf.square(error))\n", 302 | "\n", 303 | "def grad(model, inputs, targets):\n", 304 | " with tf.GradientTape() as tape:\n", 305 | " loss_value = loss(model, inputs, targets)\n", 306 | " return tape.gradient(loss_value, [model.W, model.B])" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": null, 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [ 315 | "# Define:\n", 316 | "# 1. A model.\n", 317 | "# 2. Derivatives of a loss function with respect to model parameters.\n", 318 | "# 3. A strategy for updating the variables based on the derivatives.\n", 319 | "model = Model()\n", 320 | "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n", 321 | "\n", 322 | "print(\"Initial loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", 323 | "\n", 324 | "# Training loop\n", 325 | "for i in range(300):\n", 326 | " grads = grad(model, training_inputs, training_outputs)\n", 327 | " optimizer.apply_gradients(zip(grads, [model.W, model.B]),\n", 328 | " global_step=tf.train.get_or_create_global_step())\n", 329 | " if i % 20 == 0:\n", 330 | " print(\"Loss at step {:03d}: {:.3f}\".format(i, loss(model, training_inputs, training_outputs)))\n", 331 | "\n", 332 | "print(\"Final loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", 333 | "print(\"W = {}, B = {}\".format(model.W.numpy(), model.B.numpy()))" 334 | ] 335 | }, 336 | { 337 | "cell_type": "markdown", 338 | "metadata": {}, 339 | "source": [ 340 | "### Save a model" 341 | ] 342 | }, 343 | { 344 | "cell_type": "code", 345 | "execution_count": null, 346 | "metadata": {}, 347 | "outputs": [], 348 | "source": [ 349 | "model = tf.keras.Sequential([\n", 350 | " layers.Conv2D(16, [3,3], activation='relu'),\n", 351 | " layers.GlobalAveragePooling2D(),\n", 352 | " layers.Dense(10)\n", 353 | "])\n", 354 | "\n", 355 | "optimizer = tf.train.AdamOptimizer(learning_rate=0.001)\n", 356 | "\n", 357 | "checkpoint_dir = './model_dir'\n", 358 | "os.makedirs(checkpoint_dir, exist_ok=True)\n", 359 | "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", 360 | "root = tf.train.Checkpoint(optimizer=optimizer,\n", 361 | " model=model,\n", 362 | " optimizer_step=tf.train.get_or_create_global_step())" 363 | ] 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": null, 368 | "metadata": {}, 369 | "outputs": [], 370 | "source": [ 371 | "# Save a model\n", 372 | "root.save(checkpoint_prefix)" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": null, 378 | "metadata": {}, 379 | "outputs": [], 380 | "source": [ 381 | "# Restore a model\n", 382 | "root.restore(tf.train.latest_checkpoint(checkpoint_dir))" 383 | ] 384 | } 385 | ], 386 | "metadata": { 387 | "kernelspec": { 388 | "display_name": "venv", 389 | "language": "python", 390 | "name": "venv" 391 | }, 392 | "language_info": { 393 | "codemirror_mode": { 394 | "name": "ipython", 395 | "version": 3 396 | }, 397 | "file_extension": ".py", 398 | "mimetype": "text/x-python", 399 | "name": "python", 400 | "nbconvert_exporter": "python", 401 | "pygments_lexer": "ipython3", 402 | "version": "3.6.5" 403 | }, 404 | "latex_envs": { 405 | "LaTeX_envs_menu_present": true, 406 | "autoclose": false, 407 | "autocomplete": true, 408 | "bibliofile": "biblio.bib", 409 | "cite_by": "apalike", 410 | "current_citInitial": 1, 411 | "eqLabelWithNumbers": true, 412 | "eqNumInitial": 1, 413 | "hotkeys": { 414 | "equation": "Ctrl-E", 415 | "itemize": "Ctrl-I" 416 | }, 417 | "labels_anchors": false, 418 | "latex_user_defs": false, 419 | "report_style_numbering": false, 420 | "user_envs_cfg": false 421 | }, 422 | "toc": { 423 | "base_numbering": 1, 424 | "nav_menu": {}, 425 | "number_sections": true, 426 | "sideBar": true, 427 | "skip_h1_title": false, 428 | "title_cell": "Table of Contents", 429 | "title_sidebar": "Contents", 430 | "toc_cell": false, 431 | "toc_position": {}, 432 | "toc_section_display": true, 433 | "toc_window_display": false 434 | }, 435 | "varInspector": { 436 | "cols": { 437 | "lenName": 16, 438 | "lenType": 16, 439 | "lenVar": 40 440 | }, 441 | "kernels_config": { 442 | "python": { 443 | "delete_cmd_postfix": "", 444 | "delete_cmd_prefix": "del ", 445 | "library": "var_list.py", 446 | "varRefreshCmd": "print(var_dic_list())" 447 | }, 448 | "r": { 449 | "delete_cmd_postfix": ") ", 450 | "delete_cmd_prefix": "rm(", 451 | "library": "var_list.r", 452 | "varRefreshCmd": "cat(var_dic_list()) " 453 | } 454 | }, 455 | "types_to_exclude": [ 456 | "module", 457 | "function", 458 | "builtin_function_or_method", 459 | "instance", 460 | "_Feature" 461 | ], 462 | "window_display": false 463 | } 464 | }, 465 | "nbformat": 4, 466 | "nbformat_minor": 2 467 | } 468 | -------------------------------------------------------------------------------- /tf.version.2/01.tf.basic/03.tf.data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# `tf.data` API\n", 8 | "\n", 9 | "* References\n", 10 | " * [`tf.data.Dataset` API](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)\n", 11 | " * [Importing data](https://www.tensorflow.org/guide/datasets) (Guideline Documents)\n", 12 | " * [Data Input Pipeline Performance](https://www.tensorflow.org/guide/performance/datasets)" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "`tf.data` API has two (or three) new abstractions\n", 20 | "* `tf.data.Dataset` represents a sequence of elements, in which each element contains one or more Tensor objects. For example, in an image pipeline, the image data and a label\n", 21 | " * Creating a source (e.g. Dataset.from_tensor_slices()) constructs a dataset from one or more `tf.Tensor` objects.\n", 22 | " * `tf.data.Dataset.from_tensors()`\n", 23 | " * `tf.data.Dataset.from_tensor_slices()`\n", 24 | " * `tf.data.TFRecordDataset`: TFRecord format을 읽을 때\n", 25 | " * Applying a transformation (e.g. Dataset.batch()) constructs a dataset from one or more `tf.data.Dataset` objects.\n", 26 | "* `tf.data.Iterator` provides the main way to extract elements from a dataset. The operation returned by `Iterator.get_next()` yields the next element of a `Dataset` when executed, and typically acts as the interface between input pipeline code and your model." 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "from __future__ import absolute_import\n", 36 | "from __future__ import division\n", 37 | "from __future__ import print_function\n", 38 | "from __future__ import unicode_literals\n", 39 | "\n", 40 | "import numpy as np\n", 41 | "import matplotlib.pyplot as plt\n", 42 | "%matplotlib inline\n", 43 | "\n", 44 | "import tensorflow as tf\n", 45 | "from tensorflow.keras import layers\n", 46 | "tf.enable_eager_execution()" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "### Loading MNIST dataset from `tf.keras.datasets`" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "# Load training and eval data from tf.keras\n", 63 | "(train_data, train_labels), (test_data, test_labels) = \\\n", 64 | " tf.keras.datasets.mnist.load_data()" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "### Show the MNIST" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "index = 10\n", 81 | "print(\"label = {}\".format(train_labels[index]))\n", 82 | "plt.imshow(train_data[index].reshape(28, 28))\n", 83 | "plt.colorbar()\n", 84 | "#plt.gca().grid(False)\n", 85 | "plt.show()" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "# set N=50 for small dataset loading\n", 95 | "N = 50\n", 96 | "train_data = train_data[:N]\n", 97 | "train_labels = train_labels[:N]\n", 98 | "train_data = train_data / 255.\n", 99 | "train_data = train_data.astype(np.float32)\n", 100 | "train_labels = train_labels.astype(np.int32)\n", 101 | "\n", 102 | "test_data = test_data[:N]\n", 103 | "test_labels = test_labels[:N]\n", 104 | "test_data = test_data / 255.\n", 105 | "test_data = test_data.astype(np.float32)\n", 106 | "test_labels = test_labels.astype(np.int32)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "## Input pipeline\n", 114 | "\n", 115 | "1. You must define a source. `tf.data.Dataset`.\n", 116 | " * To construct a Dataset from some tensors in memory, you can use `tf.data.Dataset.from_tensors()` or `tf.data.Dataset.from_tensor_slices()`.\n", 117 | " * Other methods\n", 118 | " * `tf.data.TextLineDataset(filenames)`\n", 119 | " * `tf.data.FixedLengthRecordDataset(filenames)`\n", 120 | " * `tf.data.TFRecordDataset(filenames)`\n", 121 | "2. Transformation\n", 122 | " * `Dataset.map()`: to apply a function to each element\n", 123 | " * `Dataset.batch()`\n", 124 | " * `Dataset.shuffle()`\n", 125 | "3. `Iterator`\n", 126 | " * `Iterator.initializer`: which enables you to (re)initialize the iterator's state\n", 127 | " * `Iterator.get_next()`" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "### 1. Store data in `tf.data.Dataset`\n", 135 | "\n", 136 | "* `tf.data.Dataset.from_tensor_slices((features, labels))`\n", 137 | "* `tf.data.Dataset.from_generator(gen, output_types, output_shapes)`" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": null, 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [ 146 | "train_dataset = tf.data.Dataset.from_tensor_slices((train_data, train_labels))\n", 147 | "print(train_dataset)\n", 148 | "print(train_dataset.output_shapes)\n", 149 | "print(train_dataset.output_types)" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "### 2. Transformaion\n", 157 | "\n", 158 | "* `apply(transformation_func)`\n", 159 | "* `batch(batch_size)`\n", 160 | "* `concatenate(dataset)`\n", 161 | "* `map(map_func)`\n", 162 | "* `repeat(count=None)`\n", 163 | " * count=max_epochs\n", 164 | "* `shuffle(buffer_size, seed=None, reshuffle_each_iteration=None)`" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "def cast_label(image, label):\n", 174 | " label = tf.cast(label, dtype=tf.float32)\n", 175 | " return image, label" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "batch_size = 16\n", 185 | "\n", 186 | "train_dataset = tf.data.Dataset.from_tensor_slices((train_data, train_labels))\n", 187 | "train_dataset = train_dataset.shuffle(buffer_size = 10000)\n", 188 | "train_dataset = train_dataset.repeat(count=2)\n", 189 | "train_dataset = train_dataset.map(cast_label)\n", 190 | "train_dataset = train_dataset.batch(batch_size = batch_size)" 191 | ] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "metadata": {}, 196 | "source": [ 197 | "### 3. Iterator\n", 198 | "\n", 199 | "* In eager execution, we can directly sample the batch dataset from `tf.data.Dataset` instead of using `Iterator`.\n", 200 | "* We do not need to explicitly create an `tf.data.Iterator` object. " 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "for step, (x, y) in enumerate(train_dataset.take(10)):\n", 210 | " print(\"step: {} label: {}\".format(step, y))" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "#### General method for importing data" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": null, 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "batch_size = 16\n", 227 | "\n", 228 | "train_dataset = tf.data.Dataset.from_tensor_slices((train_data, train_labels))\n", 229 | "train_dataset = train_dataset.shuffle(buffer_size = 10000)\n", 230 | "train_dataset = train_dataset.batch(batch_size = batch_size, drop_remainder=True)" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "max_epochs = 2\n", 240 | "for epoch in range(max_epochs):\n", 241 | " for step, (x, y) in enumerate(train_dataset):\n", 242 | " print(\"step: {} label: {}\".format(step, y))" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [] 251 | } 252 | ], 253 | "metadata": { 254 | "kernelspec": { 255 | "display_name": "venv", 256 | "language": "python", 257 | "name": "venv" 258 | }, 259 | "language_info": { 260 | "codemirror_mode": { 261 | "name": "ipython", 262 | "version": 3 263 | }, 264 | "file_extension": ".py", 265 | "mimetype": "text/x-python", 266 | "name": "python", 267 | "nbconvert_exporter": "python", 268 | "pygments_lexer": "ipython3", 269 | "version": "3.6.5" 270 | }, 271 | "latex_envs": { 272 | "LaTeX_envs_menu_present": true, 273 | "autoclose": false, 274 | "autocomplete": true, 275 | "bibliofile": "biblio.bib", 276 | "cite_by": "apalike", 277 | "current_citInitial": 1, 278 | "eqLabelWithNumbers": true, 279 | "eqNumInitial": 1, 280 | "hotkeys": { 281 | "equation": "Ctrl-E", 282 | "itemize": "Ctrl-I" 283 | }, 284 | "labels_anchors": false, 285 | "latex_user_defs": false, 286 | "report_style_numbering": false, 287 | "user_envs_cfg": false 288 | }, 289 | "toc": { 290 | "base_numbering": 1, 291 | "nav_menu": {}, 292 | "number_sections": true, 293 | "sideBar": true, 294 | "skip_h1_title": false, 295 | "title_cell": "Table of Contents", 296 | "title_sidebar": "Contents", 297 | "toc_cell": false, 298 | "toc_position": {}, 299 | "toc_section_display": true, 300 | "toc_window_display": false 301 | }, 302 | "varInspector": { 303 | "cols": { 304 | "lenName": 16, 305 | "lenType": 16, 306 | "lenVar": 40 307 | }, 308 | "kernels_config": { 309 | "python": { 310 | "delete_cmd_postfix": "", 311 | "delete_cmd_prefix": "del ", 312 | "library": "var_list.py", 313 | "varRefreshCmd": "print(var_dic_list())" 314 | }, 315 | "r": { 316 | "delete_cmd_postfix": ") ", 317 | "delete_cmd_prefix": "rm(", 318 | "library": "var_list.r", 319 | "varRefreshCmd": "cat(var_dic_list()) " 320 | } 321 | }, 322 | "types_to_exclude": [ 323 | "module", 324 | "function", 325 | "builtin_function_or_method", 326 | "instance", 327 | "_Feature" 328 | ], 329 | "window_display": false 330 | } 331 | }, 332 | "nbformat": 4, 333 | "nbformat_minor": 2 334 | } 335 | -------------------------------------------------------------------------------- /tf.version.2/02.regression/02.mnist.model.fit.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# MNIST softmax with `tf.data`\n", 8 | "\n", 9 | "* Make a very simple networks which have just one layer. (No hidden layers)\n", 10 | "* input pipeline: `tf.data`\n", 11 | "* Eager execution" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "### Training Pseudo Code\n", 19 | "\n", 20 | "```python\n", 21 | "for epoch in max_epochs: # 1 epoch: 모든 데이터(N)를 한번 학습 시켰을 때\n", 22 | " for step in num_batches: # num_batches = int(data_size / batch_size)\n", 23 | " 1. sampling mini-batches with batch_size\n", 24 | " 1-1. data augmentation [if you need (actually always)]\n", 25 | " 2. calculate the predictions # predictions = model(x)\n", 26 | " 3. calculate the loss # loss = loss(labels, predictions)\n", 27 | " 4. calculate the gradient with respect to weights\n", 28 | " 5. update weights\n", 29 | "```" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": {}, 35 | "source": [ 36 | "## Import modules" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": { 43 | "ExecuteTime": { 44 | "end_time": "2019-03-11T06:50:09.386388Z", 45 | "start_time": "2019-03-11T06:50:07.756789Z" 46 | } 47 | }, 48 | "outputs": [], 49 | "source": [ 50 | "from __future__ import absolute_import\n", 51 | "from __future__ import division\n", 52 | "from __future__ import print_function\n", 53 | "from __future__ import unicode_literals\n", 54 | "\n", 55 | "import os\n", 56 | "import time\n", 57 | "\n", 58 | "import numpy as np\n", 59 | "import matplotlib.pyplot as plt\n", 60 | "%matplotlib inline\n", 61 | "from IPython.display import clear_output\n", 62 | "\n", 63 | "import tensorflow as tf\n", 64 | "from tensorflow.keras import layers\n", 65 | "tf.enable_eager_execution()\n", 66 | "\n", 67 | "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": { 74 | "ExecuteTime": { 75 | "end_time": "2019-03-11T06:50:09.409839Z", 76 | "start_time": "2019-03-11T06:50:09.398159Z" 77 | } 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "print('TensorFlow version: {}'.format(tf.__version__))" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "### Import data" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": { 95 | "ExecuteTime": { 96 | "end_time": "2019-03-11T06:50:10.971805Z", 97 | "start_time": "2019-03-11T06:50:10.202174Z" 98 | } 99 | }, 100 | "outputs": [], 101 | "source": [ 102 | "# Load training and eval data from tf.keras\n", 103 | "(train_data, train_labels), (test_data, test_labels) = \\\n", 104 | " tf.keras.datasets.mnist.load_data()\n", 105 | "\n", 106 | "train_data = train_data / 255.\n", 107 | "train_data = train_data.reshape(-1, 784)\n", 108 | "train_data = train_data.astype(np.float32)\n", 109 | "train_labels = train_labels.astype(np.int32)\n", 110 | "\n", 111 | "test_data = test_data / 255.\n", 112 | "test_data = test_data.reshape(-1, 784)\n", 113 | "test_data = test_data.astype(np.float32)\n", 114 | "test_labels = test_labels.astype(np.int32)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "### Show the MNIST" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": { 128 | "ExecuteTime": { 129 | "end_time": "2019-03-11T06:50:12.516972Z", 130 | "start_time": "2019-03-11T06:50:12.314191Z" 131 | } 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "index = 1234\n", 136 | "print(\"label = {}\".format(train_labels[index]))\n", 137 | "plt.imshow(train_data[index].reshape(28, 28))\n", 138 | "plt.colorbar()\n", 139 | "#plt.gca().grid(False)\n", 140 | "plt.show()" 141 | ] 142 | }, 143 | { 144 | "cell_type": "markdown", 145 | "metadata": {}, 146 | "source": [ 147 | "## Create the model\n", 148 | "\n", 149 | "* Use `tf.keras.layers`" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": { 156 | "ExecuteTime": { 157 | "end_time": "2019-03-11T06:50:13.169781Z", 158 | "start_time": "2019-03-11T06:50:13.146599Z" 159 | } 160 | }, 161 | "outputs": [], 162 | "source": [ 163 | "model = tf.keras.models.Sequential()\n", 164 | "\n", 165 | "# Adds a densely-connected layer with 64 units to the model:\n", 166 | "#model.add(layers.Dense(64, activation='relu'))\n", 167 | "# Add another:\n", 168 | "#model.add(layers.Dense(64, activation='relu'))\n", 169 | "# Add a softmax layer with 10 output units:\n", 170 | "model.add(layers.Dense(10, activation='softmax'))" 171 | ] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": { 176 | "ExecuteTime": { 177 | "end_time": "2019-02-27T13:01:35.724742Z", 178 | "start_time": "2019-02-27T13:01:35.721465Z" 179 | } 180 | }, 181 | "source": [ 182 | "### Compile a model" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "metadata": { 189 | "ExecuteTime": { 190 | "end_time": "2019-03-11T06:50:17.750494Z", 191 | "start_time": "2019-03-11T06:50:17.740962Z" 192 | } 193 | }, 194 | "outputs": [], 195 | "source": [ 196 | "model.compile(optimizer=tf.train.AdamOptimizer(1e-4),\n", 197 | " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", 198 | " metrics=['accuracy'])" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [ 207 | "# without training, just inference a model in eager execution:\n", 208 | "predictions = model(train_data[0:1])\n", 209 | "print(\"Predictions: \", predictions.numpy())" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": { 216 | "ExecuteTime": { 217 | "end_time": "2019-03-11T06:50:26.807140Z", 218 | "start_time": "2019-03-11T06:50:26.803762Z" 219 | } 220 | }, 221 | "outputs": [], 222 | "source": [ 223 | "model.summary()" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "## Train a model\n", 231 | "\n", 232 | "### Using `model.fit()`" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": { 239 | "ExecuteTime": { 240 | "end_time": "2019-03-11T06:51:01.958859Z", 241 | "start_time": "2019-03-11T06:50:38.996084Z" 242 | } 243 | }, 244 | "outputs": [], 245 | "source": [ 246 | "batch_size = 32\n", 247 | "max_epochs = 10\n", 248 | "\n", 249 | "# using `numpy type` data\n", 250 | "history = model.fit(train_data, train_labels,\n", 251 | " batch_size=batch_size, epochs=max_epochs,\n", 252 | " validation_split=0.05)\n", 253 | "# using `tf.data.Dataset`\n", 254 | "#history = model.fit(train_dataset, epochs=max_epochs,\n", 255 | "# steps_per_epoch=int(len(train_data) / batch_size))" 256 | ] 257 | }, 258 | { 259 | "cell_type": "markdown", 260 | "metadata": {}, 261 | "source": [ 262 | "### Plot the loss funtion and accuracy" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "metadata": { 269 | "ExecuteTime": { 270 | "end_time": "2019-03-11T06:51:03.948937Z", 271 | "start_time": "2019-03-11T06:51:03.941971Z" 272 | } 273 | }, 274 | "outputs": [], 275 | "source": [ 276 | "history.history.keys()" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": null, 282 | "metadata": { 283 | "ExecuteTime": { 284 | "end_time": "2019-03-11T06:51:12.686509Z", 285 | "start_time": "2019-03-11T06:51:12.363440Z" 286 | } 287 | }, 288 | "outputs": [], 289 | "source": [ 290 | "acc = history.history['acc']\n", 291 | "val_acc = history.history['val_acc']\n", 292 | "\n", 293 | "loss = history.history['loss']\n", 294 | "val_loss = history.history['val_loss']\n", 295 | "\n", 296 | "epochs_range = range(max_epochs)\n", 297 | "\n", 298 | "plt.figure(figsize=(8, 8))\n", 299 | "plt.subplot(1, 2, 1)\n", 300 | "plt.plot(epochs_range, acc, label='Training Accuracy')\n", 301 | "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", 302 | "plt.legend(loc='lower right')\n", 303 | "plt.title('Training and Validation Accuracy')\n", 304 | "\n", 305 | "plt.subplot(1, 2, 2)\n", 306 | "plt.plot(epochs_range, loss, label='Training Loss')\n", 307 | "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", 308 | "plt.legend(loc='upper right')\n", 309 | "plt.title('Training and Validation Loss')\n", 310 | "plt.show()" 311 | ] 312 | }, 313 | { 314 | "cell_type": "markdown", 315 | "metadata": {}, 316 | "source": [ 317 | "## Evaluate a model\n", 318 | "\n", 319 | "### Test trained model\n", 320 | "\n", 321 | "* Compare test accuracy for 1 epoch and 10 epochs" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": null, 327 | "metadata": { 328 | "ExecuteTime": { 329 | "end_time": "2019-03-11T06:51:51.458065Z", 330 | "start_time": "2019-03-11T06:51:51.223058Z" 331 | } 332 | }, 333 | "outputs": [], 334 | "source": [ 335 | "results = model.evaluate(test_data, test_labels, batch_size=batch_size)" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": null, 341 | "metadata": { 342 | "ExecuteTime": { 343 | "end_time": "2019-03-11T06:51:53.209509Z", 344 | "start_time": "2019-03-11T06:51:53.205148Z" 345 | } 346 | }, 347 | "outputs": [], 348 | "source": [ 349 | "# loss\n", 350 | "print(\"loss value: {:.3f}\".format(results[0]))\n", 351 | "# accuracy\n", 352 | "print(\"accuracy value: {:.4f}%\".format(results[1]*100))" 353 | ] 354 | }, 355 | { 356 | "cell_type": "markdown", 357 | "metadata": {}, 358 | "source": [ 359 | "### Plot test set" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": null, 365 | "metadata": { 366 | "ExecuteTime": { 367 | "end_time": "2019-02-27T13:33:19.292217Z", 368 | "start_time": "2019-02-27T13:33:19.288174Z" 369 | } 370 | }, 371 | "outputs": [], 372 | "source": [ 373 | "np.random.seed(219)" 374 | ] 375 | }, 376 | { 377 | "cell_type": "code", 378 | "execution_count": null, 379 | "metadata": { 380 | "ExecuteTime": { 381 | "end_time": "2019-02-27T13:33:20.615446Z", 382 | "start_time": "2019-02-27T13:33:19.545969Z" 383 | } 384 | }, 385 | "outputs": [], 386 | "source": [ 387 | "test_batch_size = 16\n", 388 | "batch_index = np.random.choice(len(test_data), size=test_batch_size, replace=False)\n", 389 | "\n", 390 | "batch_xs = test_data[batch_index]\n", 391 | "batch_ys = test_labels[batch_index]\n", 392 | "y_pred_ = model(batch_xs, training=False)\n", 393 | "\n", 394 | "fig = plt.figure(figsize=(16, 10))\n", 395 | "for i, (px, py) in enumerate(zip(batch_xs, y_pred_)):\n", 396 | " p = fig.add_subplot(4, 8, i+1)\n", 397 | " if np.argmax(py) == batch_ys[i]:\n", 398 | " p.set_title(\"y_pred: {}\".format(np.argmax(py)), color='blue')\n", 399 | " else:\n", 400 | " p.set_title(\"y_pred: {}\".format(np.argmax(py)), color='red')\n", 401 | " p.imshow(px.reshape(28, 28))\n", 402 | " p.axis('off')" 403 | ] 404 | } 405 | ], 406 | "metadata": { 407 | "kernelspec": { 408 | "display_name": "venv", 409 | "language": "python", 410 | "name": "venv" 411 | }, 412 | "language_info": { 413 | "codemirror_mode": { 414 | "name": "ipython", 415 | "version": 3 416 | }, 417 | "file_extension": ".py", 418 | "mimetype": "text/x-python", 419 | "name": "python", 420 | "nbconvert_exporter": "python", 421 | "pygments_lexer": "ipython3", 422 | "version": "3.6.5" 423 | }, 424 | "latex_envs": { 425 | "LaTeX_envs_menu_present": true, 426 | "autoclose": false, 427 | "autocomplete": true, 428 | "bibliofile": "biblio.bib", 429 | "cite_by": "apalike", 430 | "current_citInitial": 1, 431 | "eqLabelWithNumbers": true, 432 | "eqNumInitial": 1, 433 | "hotkeys": { 434 | "equation": "Ctrl-E", 435 | "itemize": "Ctrl-I" 436 | }, 437 | "labels_anchors": false, 438 | "latex_user_defs": false, 439 | "report_style_numbering": false, 440 | "user_envs_cfg": false 441 | }, 442 | "toc": { 443 | "base_numbering": 1, 444 | "nav_menu": {}, 445 | "number_sections": true, 446 | "sideBar": true, 447 | "skip_h1_title": false, 448 | "title_cell": "Table of Contents", 449 | "title_sidebar": "Contents", 450 | "toc_cell": false, 451 | "toc_position": {}, 452 | "toc_section_display": true, 453 | "toc_window_display": false 454 | }, 455 | "varInspector": { 456 | "cols": { 457 | "lenName": 16, 458 | "lenType": 16, 459 | "lenVar": 40 460 | }, 461 | "kernels_config": { 462 | "python": { 463 | "delete_cmd_postfix": "", 464 | "delete_cmd_prefix": "del ", 465 | "library": "var_list.py", 466 | "varRefreshCmd": "print(var_dic_list())" 467 | }, 468 | "r": { 469 | "delete_cmd_postfix": ") ", 470 | "delete_cmd_prefix": "rm(", 471 | "library": "var_list.r", 472 | "varRefreshCmd": "cat(var_dic_list()) " 473 | } 474 | }, 475 | "types_to_exclude": [ 476 | "module", 477 | "function", 478 | "builtin_function_or_method", 479 | "instance", 480 | "_Feature" 481 | ], 482 | "window_display": false 483 | } 484 | }, 485 | "nbformat": 4, 486 | "nbformat_minor": 2 487 | } 488 | -------------------------------------------------------------------------------- /tf.version.2/04.rnn/00.rnn.custom.layers.implementation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import tensorflow as tf" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "tf.enable_eager_execution()" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "print('TensorFlow version: {}'.format(tf.__version__))" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "layers = tf.keras.layers" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "class MySimpleRNN(layers.Layer):\n", 46 | " def __init__(self, output_dim, return_sequences=False, **kwargs):\n", 47 | " super(MySimpleRNN, self).__init__(**kwargs)\n", 48 | " self.output_dim = output_dim\n", 49 | " self.return_sequences = return_sequences\n", 50 | " \n", 51 | " def build(self, input_shape):\n", 52 | " shape1 = tf.TensorShape((input_shape[-1], self.output_dim))\n", 53 | " shape2 = tf.TensorShape((self.output_dim, self.output_dim))\n", 54 | " # Create a trainable weight variable for this layer.\n", 55 | " self.kernel = self.add_weight(name='kernel',\n", 56 | " shape=shape1,\n", 57 | " initializer='uniform',\n", 58 | " trainable=True)\n", 59 | " self.recurrent_kernel = self.add_weight(name='recurrent_kernel',\n", 60 | " shape=shape2,\n", 61 | " initializer='uniform',\n", 62 | " trainable=True)\n", 63 | " self.bias = self.add_weight(name='bias',\n", 64 | " shape=self.output_dim,\n", 65 | " initializer='zeros',\n", 66 | " trainable=True)\n", 67 | " # Make sure to call the `build` method at the end\n", 68 | " super(MySimpleRNN, self).build(input_shape)\n", 69 | "\n", 70 | " def call(self, inputs):\n", 71 | " batch = inputs.get_shape()[0]\n", 72 | " seq_length = inputs.get_shape()[1]\n", 73 | " h = tf.zeros([batch, self.output_dim])\n", 74 | " if self.return_sequences:\n", 75 | " h_list = []\n", 76 | " for i in range(seq_length):\n", 77 | " h = tf.math.tanh(tf.matmul(inputs[:, i, :], self.kernel) + \n", 78 | " tf.matmul(h, self.recurrent_kernel) + self.bias)\n", 79 | " if self.return_sequences:\n", 80 | " h_list.append(h)\n", 81 | " \n", 82 | " if self.return_sequences:\n", 83 | " h_list = np.array(h_list)\n", 84 | " h_list = np.transpose(h_list, axes=[1, 0, 2])\n", 85 | " return tf.convert_to_tensor(h_list)\n", 86 | " else:\n", 87 | " return h" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "model = tf.keras.Sequential()" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "#model.add(layers.SimpleRNN(units=3, return_sequences=True))\n", 106 | "#model.add(layers.LSTM(2))\n", 107 | "#model.add(layers.GRU(4, return_sequences=True))\n", 108 | "#model.add(layers.LSTM(8, return_sequences=True))\n", 109 | "#model.add(layers.GRU(16))\n", 110 | "model.add(MySimpleRNN(3, return_sequences=True))\n", 111 | "model.add(MySimpleRNN(3))\n", 112 | "model.add(layers.Dense(1))" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "$$h_t = \\tanh (Wh + Ux)$$\n", 120 | "$$h_t = ReLU (Ux)$$\n" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "inputs = tf.random_normal([2, 4, 2])\n", 130 | "\n", 131 | "outputs = model(inputs)" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "outputs.shape" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "outputs.numpy()" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "model.trainable_variables" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [ 167 | "opt = tf.train.AdamOptimizer()" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": null, 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [ 176 | "for i in range(10):\n", 177 | " with tf.GradientTape() as tape:\n", 178 | " inputs = tf.random_normal([1, 4, 2])\n", 179 | " outputs = model(inputs)\n", 180 | " \n", 181 | " loss = tf.losses.mean_squared_error(labels=tf.ones(outputs.get_shape()),\n", 182 | " predictions=outputs)\n", 183 | " grad = tape.gradient(loss, model.variables)\n", 184 | " opt.apply_gradients(zip(grad, model.variables))\n", 185 | "\n" 186 | ] 187 | } 188 | ], 189 | "metadata": { 190 | "kernelspec": { 191 | "display_name": "venv", 192 | "language": "python", 193 | "name": "venv" 194 | }, 195 | "language_info": { 196 | "codemirror_mode": { 197 | "name": "ipython", 198 | "version": 3 199 | }, 200 | "file_extension": ".py", 201 | "mimetype": "text/x-python", 202 | "name": "python", 203 | "nbconvert_exporter": "python", 204 | "pygments_lexer": "ipython3", 205 | "version": "3.6.5" 206 | }, 207 | "latex_envs": { 208 | "LaTeX_envs_menu_present": true, 209 | "autoclose": false, 210 | "autocomplete": true, 211 | "bibliofile": "biblio.bib", 212 | "cite_by": "apalike", 213 | "current_citInitial": 1, 214 | "eqLabelWithNumbers": true, 215 | "eqNumInitial": 1, 216 | "hotkeys": { 217 | "equation": "Ctrl-E", 218 | "itemize": "Ctrl-I" 219 | }, 220 | "labels_anchors": false, 221 | "latex_user_defs": false, 222 | "report_style_numbering": false, 223 | "user_envs_cfg": false 224 | }, 225 | "toc": { 226 | "base_numbering": 1, 227 | "nav_menu": {}, 228 | "number_sections": true, 229 | "sideBar": true, 230 | "skip_h1_title": false, 231 | "title_cell": "Table of Contents", 232 | "title_sidebar": "Contents", 233 | "toc_cell": false, 234 | "toc_position": {}, 235 | "toc_section_display": true, 236 | "toc_window_display": false 237 | }, 238 | "varInspector": { 239 | "cols": { 240 | "lenName": 16, 241 | "lenType": 16, 242 | "lenVar": 40 243 | }, 244 | "kernels_config": { 245 | "python": { 246 | "delete_cmd_postfix": "", 247 | "delete_cmd_prefix": "del ", 248 | "library": "var_list.py", 249 | "varRefreshCmd": "print(var_dic_list())" 250 | }, 251 | "r": { 252 | "delete_cmd_postfix": ") ", 253 | "delete_cmd_prefix": "rm(", 254 | "library": "var_list.r", 255 | "varRefreshCmd": "cat(var_dic_list()) " 256 | } 257 | }, 258 | "types_to_exclude": [ 259 | "module", 260 | "function", 261 | "builtin_function_or_method", 262 | "instance", 263 | "_Feature" 264 | ], 265 | "window_display": false 266 | } 267 | }, 268 | "nbformat": 4, 269 | "nbformat_minor": 2 270 | } 271 | --------------------------------------------------------------------------------