├── .gitignore ├── .travis.yml ├── 01_Introduction ├── 01_How_TensorFlow_Works │ ├── 01_How_TensorFlow_Works.ipynb │ └── readme.md ├── 02_Creating_and_Using_Tensors │ ├── 02_tensors.ipynb │ ├── 02_tensors.py │ └── readme.md ├── 03_Using_Variables_and_Placeholders │ ├── 03_placeholders.ipynb │ ├── 03_placeholders.py │ └── readme.md ├── 04_Working_with_Matrices │ ├── 04_matrices.ipynb │ ├── 04_matrices.py │ └── readme.md ├── 05_Declaring_Operations │ ├── 05_operations.ipynb │ ├── 05_operations.py │ └── readme.md ├── 06_Implementing_Activation_Functions │ ├── 06_activation_functions.ipynb │ ├── 06_activation_functions.py │ └── readme.md ├── 07_Working_with_Data_Sources │ ├── 07_data_gathering.ipynb │ ├── 07_data_gathering.py │ ├── birthweight_data │ │ └── birthweight.dat │ └── readme.md ├── 08_Additional_Resources │ └── readme.md ├── images │ ├── 01_outline.png │ ├── 02_variable.png │ ├── 03_placeholder.png │ ├── 06_activation_funs1.png │ └── 06_activation_funs2.png └── readme.md ├── 02_TensorFlow_Way ├── 01_Operations_as_a_Computational_Graph │ ├── 01_operations_on_a_graph.ipynb │ ├── 01_operations_on_a_graph.py │ └── readme.md ├── 02_Layering_Nested_Operations │ ├── 02_layering_nested_operations.ipynb │ ├── 02_layering_nested_operations.py │ └── readme.md ├── 03_Working_with_Multiple_Layers │ ├── 03_multiple_layers.ipynb │ ├── 03_multiple_layers.py │ └── readme.md ├── 04_Implementing_Loss_Functions │ ├── 04_loss_functions.ipynb │ ├── 04_loss_functions.py │ └── readme.md ├── 05_Implementing_Back_Propagation │ ├── 05_back_propagation.ipynb │ ├── 05_back_propagation.py │ └── readme.md ├── 06_Working_with_Batch_and_Stochastic_Training │ ├── 06_batch_stochastic_training.ipynb │ ├── 06_batch_stochastic_training.py │ └── readme.md ├── 07_Combining_Everything_Together │ ├── 07_combining_everything_together.ipynb │ ├── 07_combining_everything_together.py │ └── readme.md ├── 08_Evaluating_Models │ ├── 08_evaluating_models.ipynb │ ├── 08_evaluating_models.py │ └── readme.md ├── images │ ├── 01_Operations_on_a_Graph.png │ ├── 02_Multiple_Operations.png │ ├── 03_Multiple_Layers.png │ ├── 04_loss_fun1.png │ ├── 04_loss_fun2.png │ ├── 06_Back_Propagation.png │ ├── 07_Combing_Everything_Together.png │ └── 08_Evaluating_Models.png └── readme.md ├── 03_Linear_Regression ├── 01_Using_the_Matrix_Inverse_Method │ ├── 01_lin_reg_inverse.ipynb │ ├── 01_lin_reg_inverse.py │ └── readme.md ├── 02_Implementing_a_Decomposition_Method │ ├── 02_lin_reg_decomposition.ipynb │ ├── 02_lin_reg_decomposition.py │ └── readme.md ├── 03_TensorFlow_Way_of_Linear_Regression │ ├── 03_lin_reg_tensorflow_way.ipynb │ ├── 03_lin_reg_tensorflow_way.py │ └── readme.md ├── 04_Loss_Functions_in_Linear_Regressions │ ├── 04_lin_reg_l1_vs_l2.ipynb │ ├── 04_lin_reg_l1_vs_l2.py │ └── readme.md ├── 05_Implementing_Deming_Regression │ ├── 05_deming_regression.ipynb │ ├── 05_deming_regression.py │ └── readme.md ├── 06_Implementing_Lasso_and_Ridge_Regression │ ├── 06_lasso_and_ridge_regression.ipynb │ ├── 06_lasso_and_ridge_regression.py │ └── readme.md ├── 07_Implementing_Elasticnet_Regression │ ├── 07_elasticnet_regression.ipynb │ ├── 07_elasticnet_regression.py │ └── readme.md ├── 08_Implementing_Logistic_Regression │ ├── 08_logistic_regression.ipynb │ ├── 08_logistic_regression.py │ └── readme.md ├── images │ ├── 01_Inverse_Matrix_Method.png │ ├── 02_Cholesky_Decomposition.png │ ├── 03_lin_reg_fit.png │ ├── 03_lin_reg_loss.png │ ├── 04_L1_L2_learningrates.png │ ├── 04_L1_L2_loss.png │ ├── 04_L1_L2_loss2.png │ ├── 05_demming_reg.png │ ├── 05_demming_vs_linear_reg.png │ ├── 07_elasticnet_reg_loss.png │ ├── 08_logistic_reg_acc.png │ └── 08_logistic_reg_loss.png └── readme.md ├── 04_Support_Vector_Machines ├── 01_Introduction │ └── readme.md ├── 02_Working_with_Linear_SVMs │ ├── 02_linear_svm.ipynb │ ├── 02_linear_svm.py │ └── readme.md ├── 03_Reduction_to_Linear_Regression │ ├── 03_support_vector_regression.ipynb │ ├── 03_support_vector_regression.py │ └── readme.md ├── 04_Working_with_Kernels │ ├── 04_svm_kernels.ipynb │ ├── 04_svm_kernels.py │ └── readme.md ├── 05_Implementing_Nonlinear_SVMs │ ├── 05_nonlinear_svm.ipynb │ ├── 05_nonlinear_svm.py │ └── readme.md ├── 06_Implementing_Multiclass_SVMs │ ├── 06_multiclass_svm.ipynb │ ├── 06_multiclass_svm.py │ └── readme.md ├── images │ ├── 01_introduction.png │ ├── 02_linear_svm_accuracy.png │ ├── 02_linear_svm_loss.png │ ├── 02_linear_svm_output.png │ ├── 03_linear_svm_loss.png │ ├── 03_svm_regression_output.png │ ├── 04_linear_svm_gaussian.png │ ├── 04_nonlinear_data_linear_kernel.png │ ├── 05_non_linear_svms.png │ └── 06_multiclass_svm.png └── readme.md ├── 05_Nearest_Neighbor_Methods ├── 01_Introduction │ └── readme.md ├── 02_Working_with_Nearest_Neighbors │ ├── 02_nearest_neighbor.ipynb │ ├── 02_nearest_neighbor.py │ └── readme.md ├── 03_Working_with_Text_Distances │ ├── 03_text_distances.ipynb │ ├── 03_text_distances.py │ └── readme.md ├── 04_Computing_with_Mixed_Distance_Functions │ ├── 04_mixed_distance_functions_knn.ipynb │ ├── 04_mixed_distance_functions_knn.py │ └── readme.md ├── 05_An_Address_Matching_Example │ ├── 05_address_matching.ipynb │ ├── 05_address_matching.py │ └── readme.md ├── 06_Nearest_Neighbors_for_Image_Recognition │ ├── 06_image_recognition.ipynb │ ├── 06_image_recognition.py │ └── readme.md ├── images │ ├── 02_mse_vs_variance.png │ ├── 02_nn_histogram.png │ ├── 04_pred_vs_actual.png │ ├── 06_nn_image_recognition.png │ ├── image.png │ └── nearest_neighbor_intro.jpg └── readme.md ├── 06_Neural_Networks ├── 01_Introduction │ ├── Introduction_to_Neural_Networks.ipynb │ └── readme.md ├── 02_Implementing_an_Operational_Gate │ ├── 02_gates.ipynb │ ├── 02_gates.py │ └── readme.md ├── 03_Working_with_Activation_Functions │ ├── 03_activation_functions.ipynb │ ├── 03_activation_functions.py │ └── readme.md ├── 04_Single_Hidden_Layer_Network │ ├── 04_single_hidden_layer_network.ipynb │ ├── 04_single_hidden_layer_network.py │ └── readme.md ├── 05_Implementing_Different_Layers │ ├── 05_implementing_different_layers.ipynb │ ├── 05_implementing_different_layers.py │ └── readme.md ├── 06_Using_Multiple_Layers │ ├── 06_using_a_multiple_layer_network.ipynb │ ├── 06_using_a_multiple_layer_network.py │ └── readme.md ├── 07_Improving_Linear_Regression │ ├── 07_improving_linear_regression.ipynb │ ├── 07_improving_linear_regression.py │ └── readme.md ├── 08_Learning_Tic_Tac_Toe │ ├── base_tic_tac_toe_moves.csv │ ├── readme.md │ └── tic_tac_toe_moves.py ├── images │ ├── 02_operational_gates.png │ ├── 03_activation1.png │ ├── 03_activation2.png │ ├── 04_nn_layout.png │ ├── 04_nn_loss.png │ ├── 06_nn_multiple_layers_loss.png │ ├── 07_lin_reg_acc.png │ ├── 07_lin_reg_loss.png │ ├── 08_tic_tac_toe_architecture.png │ ├── 08_tictactoe_layout.png │ ├── 08_tictactoe_loss.png │ └── image.png └── readme.md ├── 07_Natural_Language_Processing ├── 01_Introduction │ ├── 01_introduction.ipynb │ └── readme.md ├── 02_Working_with_Bag_of_Words │ ├── 02_bag_of_words.ipynb │ ├── 02_bag_of_words.py │ └── readme.md ├── 03_Implementing_tf_idf │ ├── 03_implementing_tf_idf.ipynb │ ├── 03_implementing_tf_idf.py │ └── readme.md ├── 04_Working_With_Skip_Gram_Embeddings │ ├── 04_working_with_skipgram.ipynb │ ├── 04_working_with_skipgram.py │ └── readme.md ├── 05_Working_With_CBOW_Embeddings │ ├── 05_Working_With_CBOW.ipynb │ ├── 05_Working_With_CBOW.py │ ├── readme.md │ └── text_helpers.py ├── 06_Using_Word2Vec_Embeddings │ ├── 06_using_word2vec.ipynb │ ├── 06_using_word2vec.py │ ├── readme.md │ └── text_helpers.py ├── 07_Sentiment_Analysis_With_Doc2Vec │ ├── 07_sentiment_with_doc2vec.ipynb │ ├── 07_sentiment_with_doc2vec.py │ ├── readme.md │ └── text_helpers.py ├── images │ ├── 02_bag_of_words.png │ ├── 03_tfidf_acc.png │ ├── 03_tfidf_loss.png │ ├── 04_skipgram_model.png │ ├── 05_cbow_model.png │ ├── 06_word2vec_acc.png │ ├── 06_word2vec_loss.png │ ├── 07_sentiment_doc2vec_loss.png │ └── image.png └── readme.md ├── 08_Convolutional_Neural_Networks ├── 01_Intro_to_CNN │ └── readme.md ├── 02_Intro_to_CNN_MNIST │ ├── 02_introductory_cnn.ipynb │ ├── 02_introductory_cnn.py │ └── readme.md ├── 03_CNN_CIFAR10 │ ├── 03_cnn_cifar10.ipynb │ ├── 03_cnn_cifar10.py │ └── readme.md ├── 04_Retraining_Current_Architectures │ ├── 04_download_cifar10.ipynb │ ├── 04_download_cifar10.py │ └── readme.md ├── 05_Stylenet_NeuralStyle │ ├── 05_stylenet.ipynb │ ├── 05_stylenet.py │ └── readme.md ├── 06_Deepdream │ ├── 06_deepdream.ipynb │ ├── 06_deepdream.py │ ├── book_cover.jpg │ └── readme.md ├── images │ ├── 01_intro_cnn.png │ ├── 01_intro_cnn2.png │ ├── 02_cnn1_loss_acc.png │ ├── 02_cnn1_mnist_output.png │ ├── 03_cnn2_loss_acc.png │ ├── 05_stylenet_ex.png │ ├── 06_deepdream_ex.png │ ├── book_cover.jpg │ ├── image.png │ └── starry_night.jpg └── readme.md ├── 09_Recurrent_Neural_Networks ├── 01_Introduction │ └── readme.md ├── 02_Implementing_RNN_for_Spam_Prediction │ ├── 02_implementing_rnn.ipynb │ ├── 02_implementing_rnn.py │ └── readme.md ├── 03_Implementing_LSTM │ ├── 03_implementing_lstm.ipynb │ ├── 03_implementing_lstm.py │ └── readme.md ├── 04_Stacking_Multiple_LSTM_Layers │ ├── 04_stacking_multiple_lstm.ipynb │ ├── 04_stacking_multiple_lstm.py │ └── readme.md ├── 05_Creating_A_Sequence_To_Sequence_Model │ ├── 05_seq2seq_translation.ipynb │ ├── 05_seq2seq_translation.py │ ├── 05_translation_model_sample.py │ └── readme.md ├── 06_Training_A_Siamese_Similarity_Measure │ ├── 06_siamese_similarity_driver.ipynb │ ├── 06_siamese_similarity_driver.py │ ├── readme.md │ └── siamese_similarity_model.py ├── images │ ├── 01_RNN_Seq2Seq.png │ ├── 01_RNN_Single_Target.png │ ├── 02_RNN_Spam_Acc_Loss.png │ ├── 03_LSTM_Loss.png │ ├── 04_MultipleLSTM_Loss.png │ ├── 04_MultipleRNN_Architecture.png │ ├── 05_Seq2Seq_Loss.png │ ├── 06_Similarity_RNN.png │ ├── 06_Similarity_RNN_Architecture.png │ └── 06_Similarity_RNN_Diagram.png └── readme.md ├── 10_Taking_TensorFlow_to_Production ├── 01_Implementing_Unit_Tests │ ├── 01_implementing_unit_tests.py │ └── readme.md ├── 02_Using_Multiple_Devices │ ├── 02_using_multiple_devices.py │ └── readme.md ├── 03_Parallelizing_TensorFlow │ ├── 03_parallelizing_tensorflow.py │ └── readme.md ├── 04_Production_Tips │ ├── 04_production_tips_for_tf.py │ └── readme.md ├── 05_Production_Example │ ├── 05_production_ex_eval.py │ ├── 05_production_ex_train.py │ └── readme.md ├── 06_Using_TensorFlow_Serving │ ├── 06_Using_TensorFlow_Serving_Client.py │ └── 06_Using_TensorFlow_Serving_Train.py ├── images │ ├── file_structure.jpg │ └── image.png └── readme.md ├── 11_More_with_TensorFlow ├── 01_Visualizing_Computational_Graphs │ ├── 01_using_tensorboard.ipynb │ ├── 01_using_tensorboard.py │ └── readme.md ├── 02_Working_with_a_Genetic_Algorithm │ ├── 02_genetic_algorithm.py │ └── readme.md ├── 03_Clustering_Using_KMeans │ ├── 03_k_means.py │ └── readme.md ├── 04_Solving_A_System_of_ODEs │ ├── 04_solving_ode_system.py │ └── readme.md ├── 05_Using_a_Random_Forest │ └── 05_Using_a_Random_Forest.py ├── 06_Using_TensorFlow_with_Keras │ └── 06_Using_TensorFlow_with_Keras.py ├── images │ ├── 01_tensorboard1.png │ ├── 01_tensorboard2.png │ ├── 01_tensorboard3.png │ ├── 02_genetic_algorithm.png │ ├── 03_kmeans.png │ └── 04_ode_system.png └── readme.md ├── CODE_OF_CONDUCT.md ├── LICENSE ├── images └── book_covers.jpg ├── readme.md ├── requirements.txt └── test_script.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Standard python gitignore file 2 | *.csv 3 | !/06_Neural_Networks/08_Learning_Tic_Tac_Toe/base_tic_tac_toe_moves.csv 4 | *.en 5 | *.de 6 | *.tsv 7 | *.Rdata 8 | *.rdata 9 | *.ppt 10 | *.pptx 11 | *.zip 12 | *.pb 13 | *.docx 14 | *.xls 15 | *.xlsx 16 | *.gz 17 | vocab2ix.json 18 | vocab2ix_dict.json 19 | .gitignore~ 20 | .Rhistory 21 | Readme.md~ 22 | readme.md~ 23 | LICENSE~ 24 | *.pkl 25 | *.ckpt 26 | checkpoint 27 | *.ckpt 28 | *.ckpy.meta 29 | *.pkl 30 | *.neg 31 | *.pos 32 | **/temp/ 33 | **/temp 34 | **/tmp/ 35 | **/tmp 36 | */tensorboard_logs/* 37 | */tensorboard_logs/ 38 | */tensorboard_logs 39 | ***/tensorboard_logs/* 40 | **/tensorboard_logs/ 41 | **/tensorboard_logs 42 | tensorboard_logs/ 43 | tensorboard_logs/* 44 | */tensorboard/* 45 | */tensorboard/ 46 | */tensorboard 47 | ***/tensorboard/* 48 | **/tensorboard/ 49 | **/tensorboard 50 | tensorboard/ 51 | tensorboard/* 52 | temp_* 53 | 54 | **/12_Reinforcement_Learning/ 55 | **/12_Reinforcement_Learning 56 | **/12_Reinforcement_Learning/* 57 | 58 | # Byte-compiled / optimized / DLL files 59 | __pycache__/ 60 | *.py[cod] 61 | *$py.class 62 | *.py~ 63 | 64 | # C extensions 65 | *.so 66 | 67 | # Distribution / packaging 68 | .Python 69 | env/ 70 | build/ 71 | develop-eggs/ 72 | dist/ 73 | downloads/ 74 | eggs/ 75 | .eggs/ 76 | lib/ 77 | lib64/ 78 | parts/ 79 | sdist/ 80 | var/ 81 | *.egg-info/ 82 | .installed.cfg 83 | *.egg 84 | 85 | temp/ 86 | temp*/ 87 | temp_*.* 88 | 89 | 90 | # PyInstaller 91 | # Usually these files are written by a python script from a template 92 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 93 | *.manifest 94 | *.spec 95 | 96 | # Installer logs 97 | pip-log.txt 98 | pip-delete-this-directory.txt 99 | 100 | # Unit test / coverage reports 101 | htmlcov/ 102 | .tox/ 103 | .coverage 104 | .coverage.* 105 | .cache 106 | nosetests.xml 107 | coverage.xml 108 | *,cover 109 | .hypothesis/ 110 | 111 | # Translations 112 | *.mo 113 | *.pot 114 | 115 | # Django stuff: 116 | *.log 117 | local_settings.py 118 | 119 | # Flask stuff: 120 | instance/ 121 | .webassets-cache 122 | 123 | # Scrapy stuff: 124 | .scrapy 125 | 126 | # Sphinx documentation 127 | docs/_build/ 128 | 129 | # PyBuilder 130 | target/ 131 | 132 | # IPython Notebook 133 | .ipynb_checkpoints 134 | 135 | # pyenv 136 | .python-version 137 | 138 | # celery beat schedule file 139 | celerybeat-schedule 140 | 141 | # dotenv 142 | .env 143 | 144 | # virtualenv 145 | venv/ 146 | ENV/ 147 | 148 | # Spyder/pycharm project settings 149 | .spyderproject 150 | .idea/ 151 | 152 | # Rope project settings 153 | .ropeproject 154 | 155 | # Ignore data sources 156 | */MNIST_data/* 157 | */birthweight_data/* 158 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | language: python 3 | addons: 4 | apt: 5 | update: true 6 | python: 7 | - "3.6" 8 | # command to install dependencies 9 | install: 10 | - "pip install -r requirements.txt" 11 | # command to run tests 12 | script: 13 | - python test_script.py 14 | -------------------------------------------------------------------------------- /01_Introduction/01_How_TensorFlow_Works/readme.md: -------------------------------------------------------------------------------- 1 | # Introduction to How TensorFlow Graphs Work 2 | 3 | For more detail, see the [Jupyter Notebook](01_How_TensorFlow_Works.ipynb). 4 | 5 | TensorFlow has a unique way of solving problems. This unique way allows for solving of machine learning problems very efficiently. There are a few common steps to most TensorFlow algorithms. 6 | 7 | 1. Import data, generate data, or setup a data-pipeline through placeholders. 8 | 9 | 2. Feed data through computational graph. 10 | 11 | 3. Evaluate output on loss function. 12 | 13 | 4. Use backpropagation to modify the variables. 14 | 15 | 5. Repeat until stopping condition. 16 | 17 | ![Computational Graph](../images/01_outline.png "A general outline of computational graphs") 18 | 19 | 20 | Now there are many more subtleties to many of the algorithms that we will cover in the book. We will talk about how to transform data, normalize data, use variables, create placeholders, initialize objects, define the computational graph structure, create loss functions, train models, and evaluate the results. 21 | 22 | ### TO DO 23 | 24 | - Add more documentation, explanations, and resources. 25 | -------------------------------------------------------------------------------- /01_Introduction/02_Creating_and_Using_Tensors/02_tensors.py: -------------------------------------------------------------------------------- 1 | # Tensors 2 | #---------------------------------- 3 | # 4 | # This function introduces various ways to create 5 | # tensors in TensorFlow 6 | 7 | import tensorflow as tf 8 | from tensorflow.python.framework import ops 9 | ops.reset_default_graph() 10 | 11 | # Introduce tensors in tf 12 | 13 | # Get graph handle 14 | sess = tf.Session() 15 | 16 | my_tensor = tf.zeros([1,20]) 17 | 18 | # Declare a variable 19 | my_var = tf.Variable(tf.zeros([1,20])) 20 | 21 | # Different kinds of variables 22 | row_dim = 2 23 | col_dim = 3 24 | 25 | # Zero initialized variable 26 | zero_var = tf.Variable(tf.zeros([row_dim, col_dim])) 27 | 28 | # One initialized variable 29 | ones_var = tf.Variable(tf.ones([row_dim, col_dim])) 30 | 31 | # shaped like other variable 32 | sess.run(zero_var.initializer) 33 | sess.run(ones_var.initializer) 34 | zero_similar = tf.Variable(tf.zeros_like(zero_var)) 35 | ones_similar = tf.Variable(tf.ones_like(ones_var)) 36 | 37 | sess.run(ones_similar.initializer) 38 | sess.run(zero_similar.initializer) 39 | 40 | # Fill shape with a constant 41 | fill_var = tf.Variable(tf.fill([row_dim, col_dim], -1)) 42 | 43 | # Create a variable from a constant 44 | const_var = tf.Variable(tf.constant([8, 6, 7, 5, 3, 0, 9])) 45 | # This can also be used to fill an array: 46 | const_fill_var = tf.Variable(tf.constant(-1, shape=[row_dim, col_dim])) 47 | 48 | # Sequence generation 49 | linear_var = tf.Variable(tf.linspace(start=0.0, stop=1.0, num=3)) # Generates [0.0, 0.5, 1.0] includes the end 50 | 51 | sequence_var = tf.Variable(tf.range(start=6, limit=15, delta=3)) # Generates [6, 9, 12] doesn't include the end 52 | 53 | # Random Numbers 54 | 55 | # Random Normal 56 | rnorm_var = tf.random_normal([row_dim, col_dim], mean=0.0, stddev=1.0) 57 | 58 | # Add summaries to tensorboard 59 | merged = tf.summary.merge_all() 60 | 61 | # Initialize graph writer: 62 | writer = tf.summary.FileWriter("/tmp/variable_logs", graph=sess.graph) 63 | 64 | # Initialize operation 65 | initialize_op = tf.global_variables_initializer() 66 | 67 | # Run initialization of variable 68 | sess.run(initialize_op) -------------------------------------------------------------------------------- /01_Introduction/02_Creating_and_Using_Tensors/readme.md: -------------------------------------------------------------------------------- 1 | # Creating and Using Tensors 2 | 3 | This script will produce a computational graph of initializing tensors. Here is the output if viewed in Tensorboard: 4 | 5 | ![Computational Graph](../images/02_variable.png "Tensor Computational Graph") 6 | -------------------------------------------------------------------------------- /01_Introduction/03_Using_Variables_and_Placeholders/03_placeholders.py: -------------------------------------------------------------------------------- 1 | # Placeholders 2 | #---------------------------------- 3 | # 4 | # This function introduces how to 5 | # use placeholders in TensorFlow 6 | 7 | import numpy as np 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # Using Placeholders 13 | sess = tf.Session() 14 | 15 | x = tf.placeholder(tf.float32, shape=(4, 4)) 16 | y = tf.identity(x) 17 | 18 | rand_array = np.random.rand(4, 4) 19 | 20 | merged = tf.summary.merge_all() 21 | 22 | writer = tf.summary.FileWriter("/tmp/variable_logs", sess.graph) 23 | 24 | print(sess.run(y, feed_dict={x: rand_array})) -------------------------------------------------------------------------------- /01_Introduction/03_Using_Variables_and_Placeholders/readme.md: -------------------------------------------------------------------------------- 1 | # Variables and Placeholders 2 | 3 | Here we initialize a simple placeholder for a piece of data. We then feed a random number into an identity function. If viewed in Tensorboard, the following graph will be created. 4 | 5 | ![Computational Graph](../images/03_placeholder.png "A Graph of a Variable and Placeholder") 6 | 7 | ### TO DO 8 | 9 | - Add more documentation, explanations, and resources. 10 | -------------------------------------------------------------------------------- /01_Introduction/04_Working_with_Matrices/04_matrices.py: -------------------------------------------------------------------------------- 1 | # Matrices and Matrix Operations 2 | #---------------------------------- 3 | # 4 | # This function introduces various ways to create 5 | # matrices and how to use them in TensorFlow 6 | 7 | import numpy as np 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # Declaring matrices 13 | sess = tf.Session() 14 | 15 | # Declaring matrices 16 | 17 | # Identity matrix 18 | identity_matrix = tf.diag([1.0,1.0,1.0]) 19 | print(sess.run(identity_matrix)) 20 | 21 | # 2x3 random norm matrix 22 | A = tf.truncated_normal([2,3]) 23 | print(sess.run(A)) 24 | 25 | # 2x3 constant matrix 26 | B = tf.fill([2,3], 5.0) 27 | print(sess.run(B)) 28 | 29 | # 3x2 random uniform matrix 30 | C = tf.random_uniform([3,2]) 31 | print(sess.run(C)) # Note that we are reinitializing, hence the new random variables 32 | 33 | # Create matrix from np array 34 | D = tf.convert_to_tensor(np.array([[1., 2., 3.], [-3., -7., -1.], [0., 5., -2.]])) 35 | print(sess.run(D)) 36 | 37 | # Matrix addition/subtraction 38 | print(sess.run(A+B)) 39 | print(sess.run(B-B)) 40 | 41 | # Matrix Multiplication 42 | print(sess.run(tf.matmul(B, identity_matrix))) 43 | 44 | # Matrix Transpose 45 | print(sess.run(tf.transpose(C))) # Again, new random variables 46 | 47 | # Matrix Determinant 48 | print(sess.run(tf.matrix_determinant(D))) 49 | 50 | # Matrix Inverse 51 | print(sess.run(tf.matrix_inverse(D))) 52 | 53 | # Cholesky Decomposition 54 | print(sess.run(tf.cholesky(identity_matrix))) 55 | 56 | # Eigenvalues and Eigenvectors 57 | print(sess.run(tf.self_adjoint_eig(D))) -------------------------------------------------------------------------------- /01_Introduction/04_Working_with_Matrices/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Matrices 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /01_Introduction/05_Declaring_Operations/05_operations.py: -------------------------------------------------------------------------------- 1 | # Operations 2 | #---------------------------------- 3 | # 4 | # This function introduces various operations 5 | # in TensorFlow 6 | 7 | # Declaring Operations 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # Open graph session 13 | sess = tf.Session() 14 | 15 | # div() vs truediv() vs floordiv() 16 | print(sess.run(tf.div(3, 4))) 17 | print(sess.run(tf.truediv(3, 4))) 18 | print(sess.run(tf.floordiv(3.0, 4.0))) 19 | 20 | # Mod function 21 | print(sess.run(tf.mod(22.0, 5.0))) 22 | 23 | # Cross Product 24 | print(sess.run(tf.cross([1., 0., 0.], [0., 1., 0.]))) 25 | 26 | # Trig functions 27 | print(sess.run(tf.sin(3.1416))) 28 | print(sess.run(tf.cos(3.1416))) 29 | print(sess.run(tf.tan(3.1416/4.))) 30 | 31 | # Custom operation 32 | test_nums = range(15) 33 | 34 | 35 | def custom_polynomial(x_val): 36 | # Return 3x^2 - x + 10 37 | return tf.subtract(3 * tf.square(x_val), x_val) + 10 38 | 39 | print(sess.run(custom_polynomial(11))) 40 | 41 | # What should we get with list comprehension 42 | expected_output = [3*x*x-x+10 for x in test_nums] 43 | print(expected_output) 44 | 45 | # TensorFlow custom function output 46 | for num in test_nums: 47 | print(sess.run(custom_polynomial(num))) 48 | -------------------------------------------------------------------------------- /01_Introduction/05_Declaring_Operations/readme.md: -------------------------------------------------------------------------------- 1 | # Declaring Operations 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /01_Introduction/06_Implementing_Activation_Functions/06_activation_functions.py: -------------------------------------------------------------------------------- 1 | # Activation Functions 2 | #---------------------------------- 3 | # 4 | # This function introduces activation 5 | # functions in TensorFlow 6 | 7 | # Implementing Activation Functions 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import tensorflow as tf 11 | from tensorflow.python.framework import ops 12 | ops.reset_default_graph() 13 | 14 | # Open graph session 15 | sess = tf.Session() 16 | 17 | # X range 18 | x_vals = np.linspace(start=-10., stop=10., num=100) 19 | 20 | # ReLU activation 21 | print(sess.run(tf.nn.relu([-3., 3., 10.]))) 22 | y_relu = sess.run(tf.nn.relu(x_vals)) 23 | 24 | # ReLU-6 activation 25 | print(sess.run(tf.nn.relu6([-3., 3., 10.]))) 26 | y_relu6 = sess.run(tf.nn.relu6(x_vals)) 27 | 28 | # Sigmoid activation 29 | print(sess.run(tf.nn.sigmoid([-1., 0., 1.]))) 30 | y_sigmoid = sess.run(tf.nn.sigmoid(x_vals)) 31 | 32 | # Hyper Tangent activation 33 | print(sess.run(tf.nn.tanh([-1., 0., 1.]))) 34 | y_tanh = sess.run(tf.nn.tanh(x_vals)) 35 | 36 | # Softsign activation 37 | print(sess.run(tf.nn.softsign([-1., 0., 1.]))) 38 | y_softsign = sess.run(tf.nn.softsign(x_vals)) 39 | 40 | # Softplus activation 41 | print(sess.run(tf.nn.softplus([-1., 0., 1.]))) 42 | y_softplus = sess.run(tf.nn.softplus(x_vals)) 43 | 44 | # Exponential linear activation 45 | print(sess.run(tf.nn.elu([-1., 0., 1.]))) 46 | y_elu = sess.run(tf.nn.elu(x_vals)) 47 | 48 | # Plot the different functions 49 | plt.plot(x_vals, y_softplus, 'r--', label='Softplus', linewidth=2) 50 | plt.plot(x_vals, y_relu, 'b:', label='ReLU', linewidth=2) 51 | plt.plot(x_vals, y_relu6, 'g-.', label='ReLU6', linewidth=2) 52 | plt.plot(x_vals, y_elu, 'k-', label='ExpLU', linewidth=0.5) 53 | plt.ylim([-1.5,7]) 54 | plt.legend(loc='upper left') 55 | plt.show() 56 | 57 | plt.plot(x_vals, y_sigmoid, 'r--', label='Sigmoid', linewidth=2) 58 | plt.plot(x_vals, y_tanh, 'b:', label='Tanh', linewidth=2) 59 | plt.plot(x_vals, y_softsign, 'g-.', label='Softsign', linewidth=2) 60 | plt.ylim([-2,2]) 61 | plt.legend(loc='upper left') 62 | plt.show() 63 | -------------------------------------------------------------------------------- /01_Introduction/06_Implementing_Activation_Functions/readme.md: -------------------------------------------------------------------------------- 1 | # Activation Functions 2 | 3 | This script plots many various activation functions in TensorFlow. 4 | 5 | ![Activation Functions](../images/06_activation_funs1.png "Activation Functions") 6 | 7 | ![Activation Functions](../images/06_activation_funs2.png "Activation Functions") 8 | 9 | ### TO DO 10 | 11 | - Add more documentation, explanations, and resources. 12 | -------------------------------------------------------------------------------- /01_Introduction/07_Working_with_Data_Sources/readme.md: -------------------------------------------------------------------------------- 1 | # Data Source Information 2 | 3 | Here are the sources of the data sets used in this book. The following links are the original sources with explanations and citations. The script in this directory demonstrates how to access these data sets. 4 | 5 | 1. [Iris Data](http://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html) 6 | 2. [Low Birthweight Data](https://github.com/nfmcclure/tensorflow_cookbook/raw/master/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat) 7 | 3. [Housing Price Data](https://archive.ics.uci.edu/ml/datasets/Housing) 8 | 4. [MNIST Dataset of Handwritten Digits](http://yann.lecun.com/exdb/mnist/) 9 | 5. [SMS Spam Data](https://archive.ics.uci.edu/ml/datasets/SMS+Spam+Collection) 10 | 6. [Movie Review Data](http://www.cs.cornell.edu/people/pabo/movie-review-data/) 11 | 7. [William Shakespeare Data](http://www.gutenberg.org/ebooks/100) 12 | 8. [German-English Sentence Data](http://www.manythings.org/anki/) 13 | 14 | 15 | -------------------------------------------------------------------------------- /01_Introduction/08_Additional_Resources/readme.md: -------------------------------------------------------------------------------- 1 | # Additional Resources 2 | 3 | ###Official Resources: 4 | 5 | - [TensorFlow Python API](https://www.tensorflow.org/api_docs/python/) 6 | - [TensorFlow on Github](https://github.com/tensorflow/tensorflow) 7 | - [TensorFlow Tutorials](https://www.tensorflow.org/tutorials/) 8 | - [Udacity Deep Learning Class](https://www.udacity.com/course/deep-learning--ud730) 9 | - [TensorFlow Playground](http://playground.tensorflow.org/) 10 | 11 | ###Github Tutorials and Examples: 12 | 13 | - [Tutorials by pkmital](https://github.com/pkmital/tensorflow_tutorials) 14 | - [Tutorials by nlintz](https://github.com/nlintz/TensorFlow-Tutorials) 15 | - [Examples by americdamien](https://github.com/aymericdamien/TensorFlow-Examples) 16 | - [TensorFlow Workshop by amygdala](https://github.com/amygdala/tensorflow-workshop) 17 | 18 | ###Deep Learning Resources 19 | 20 | - [Efficient Back Prop by Yann LeCun, et. al.](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf) 21 | - [Online Deep Learning Book, MIT Press](http://www.deeplearningbook.org/) 22 | - [An Overview of Gradient Descent Algorithms by Sebastian Ruder](http://sebastianruder.com/optimizing-gradient-descent/) 23 | - [Stochastic Optimization by John Duchi, et. al.](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) 24 | - [ADADELTA Method by Matthew Zeiler](http://arxiv.org/abs/1212.5701) 25 | - [A Friendly Introduction to Cross-Entropy Loss by Rob DiPietro](http://rdipietro.github.io/friendly-intro-to-cross-entropy-loss/) 26 | 27 | 28 | ###Additional Resources 29 | 30 | - [A Curated List of Dedicated TensorFlow Resources](https://github.com/jtoy/awesome-tensorflow/) 31 | 32 | ###Arxiv Papers 33 | 34 | - [TensorFlow: Large-Scale Machine Learning on Heterogeneous Distributed Systems](http://arxiv.org/abs/1603.04467) 35 | - [TensorFlow: A system for large-scale machine learning](http://arxiv.org/abs/1605.08695) 36 | - [Distributed TensorFlow with MPI](https://arxiv.org/abs/1603.02339) 37 | - [Comparative Study of Deep Learning Software Frameworks](https://arxiv.org/abs/1511.06435) 38 | - [Wide & Deep Learning for Recommender Systems](https://arxiv.org/abs/1606.07792) 39 | -------------------------------------------------------------------------------- /01_Introduction/images/01_outline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/01_Introduction/images/01_outline.png -------------------------------------------------------------------------------- /01_Introduction/images/02_variable.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/01_Introduction/images/02_variable.png -------------------------------------------------------------------------------- /01_Introduction/images/03_placeholder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/01_Introduction/images/03_placeholder.png -------------------------------------------------------------------------------- /01_Introduction/images/06_activation_funs1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/01_Introduction/images/06_activation_funs1.png -------------------------------------------------------------------------------- /01_Introduction/images/06_activation_funs2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/01_Introduction/images/06_activation_funs2.png -------------------------------------------------------------------------------- /01_Introduction/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 1: Getting Started with TensorFlow 2 | 3 | This chapter intends to introduce the main objects and concepts in TensorFlow. We also introduce how to access the data for the rest of the book and provide additional resources for learning about TensorFlow. 4 | 5 | 1. [How TensorFlow Works (General Outline of TF Algorithms)](01_How_TensorFlow_Works#introduction-to-how-tensorflow-graphs-work) 6 | * Here we introduce TensorFlow and the general outline of how most TensorFlow algorithms work. 7 | 2. [Creating and Using Tensors](02_Creating_and_Using_Tensors#creating-and-using-tensors) 8 | * How to create and initialize tensors in TensorFlow. We also depict how these operations appear in Tensorboard. 9 | 3. [Using Variables and Placeholders](03_Using_Variables_and_Placeholders#variables-and-placeholders) 10 | * How to create and use variables and placeholders in TensorFlow. We also depict how these operations appear in Tensorboard. 11 | 4. [Working with Matrices](04_Working_with_Matrices#working-with-matrices) 12 | * Understanding how TensorFlow can work with matrices is crucial to understanding how the algorithms work. 13 | 5. [Declaring Operations](05_Declaring_Operations#declaring-operations) 14 | * How to use various mathematical operations in TensorFlow. 15 | 6. [Implementing Activation Functions](06_Implementing_Activation_Functions#activation-functions) 16 | * Activation functions are unique functions that TensorFlow has built in for your use in algorithms. 17 | 7. [Working with Data Sources](07_Working_with_Data_Sources#data-source-information) 18 | * Here we show how to access all the various required data sources in the book. There are also links describing the data sources and where they come from. 19 | 8. [Additional Resources](08_Additional_Resources#additional-resources) 20 | * Mostly official resources and papers. The papers are TensorFlow papers or Deep Learning resources. 21 | -------------------------------------------------------------------------------- /02_TensorFlow_Way/01_Operations_as_a_Computational_Graph/01_operations_on_a_graph.py: -------------------------------------------------------------------------------- 1 | # Operations on a Computational Graph 2 | import os 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.python.framework import ops 6 | ops.reset_default_graph() 7 | 8 | # Create graph 9 | sess = tf.Session() 10 | 11 | # Create tensors 12 | 13 | # Create data to feed in 14 | x_vals = np.array([1., 3., 5., 7., 9.]) 15 | x_data = tf.placeholder(tf.float32) 16 | m_const = tf.constant(3.) 17 | 18 | # Multiplication 19 | my_product = tf.multiply(x_data, m_const) 20 | for x_val in x_vals: 21 | print(sess.run(my_product, feed_dict={x_data: x_val})) 22 | 23 | # View the tensorboard graph by running the following code and then 24 | # going to the terminal and typing: 25 | # $ tensorboard --logdir=tensorboard_logs 26 | merged = tf.summary.merge_all() 27 | if not os.path.exists('tensorboard_logs/'): 28 | os.makedirs('tensorboard_logs/') 29 | 30 | my_writer = tf.summary.FileWriter('tensorboard_logs/', sess.graph) 31 | -------------------------------------------------------------------------------- /02_TensorFlow_Way/01_Operations_as_a_Computational_Graph/readme.md: -------------------------------------------------------------------------------- 1 | # Operations as a Computational Graph 2 | 3 | ## Summary 4 | 5 | In this script, we create an array and feed it into a placeholder. We then multiply it by a constant. 6 | 7 | ## Computational Graph Output 8 | 9 | Viewing the computational graph in Tensorboard, as created by the script: 10 | 11 | ![One Operation](../images/01_Operations_on_a_Graph.png "An Operation on a Graph") -------------------------------------------------------------------------------- /02_TensorFlow_Way/02_Layering_Nested_Operations/02_layering_nested_operations.py: -------------------------------------------------------------------------------- 1 | # Layering Nested Operations 2 | 3 | import os 4 | import numpy as np 5 | import tensorflow as tf 6 | from tensorflow.python.framework import ops 7 | ops.reset_default_graph() 8 | 9 | # Start a graph session 10 | sess = tf.Session() 11 | 12 | # Create the data and variables 13 | my_array = np.array([[1., 3., 5., 7., 9.], 14 | [-2., 0., 2., 4., 6.], 15 | [-6., -3., 0., 3., 6.]]) 16 | x_vals = np.array([my_array, my_array + 1]) 17 | x_data = tf.placeholder(tf.float32, shape=(3, 5)) 18 | 19 | # Constants for matrix multiplication: 20 | m1 = tf.constant([[1.], [0.], [-1.], [2.], [4.]]) 21 | m2 = tf.constant([[2.]]) 22 | a1 = tf.constant([[10.]]) 23 | 24 | # Create our multiple operations 25 | prod1 = tf.matmul(x_data, m1) 26 | prod2 = tf.matmul(prod1, m2) 27 | add1 = tf.add(prod2, a1) 28 | 29 | # Now feed data through placeholder and print results 30 | for x_val in x_vals: 31 | print(sess.run(add1, feed_dict={x_data: x_val})) 32 | 33 | # View the tensorboard graph by running the following code and then 34 | # going to the terminal and typing: 35 | # $ tensorboard --logdir=tensorboard_logs 36 | merged = tf.summary.merge_all() 37 | if not os.path.exists('tensorboard_logs/'): 38 | os.makedirs('tensorboard_logs/') 39 | 40 | my_writer = tf.summary.FileWriter('tensorboard_logs/', sess.graph) 41 | -------------------------------------------------------------------------------- /02_TensorFlow_Way/02_Layering_Nested_Operations/readme.md: -------------------------------------------------------------------------------- 1 | # Multiple Operations on a Computational Graph 2 | 3 | ## Summary 4 | 5 | In this script, we will create an array and perform two multiplications on it, followed by addition: 6 | 7 | output = (input) * (m1) * (m2) + (a1) 8 | 9 | ## Computational Graph Output 10 | 11 | Viewing the computational graph in Tensorboard: 12 | 13 | ![Multiple Operations](../images/02_Multiple_Operations.png "Multiple Operations on a Graph") -------------------------------------------------------------------------------- /02_TensorFlow_Way/03_Working_with_Multiple_Layers/03_multiple_layers.py: -------------------------------------------------------------------------------- 1 | # Working with Multiple Layers 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | import os 6 | from tensorflow.python.framework import ops 7 | ops.reset_default_graph() 8 | 9 | # Create graph 10 | sess = tf.Session() 11 | 12 | # Create tensors 13 | 14 | # Create a small random 'image' of size 4x4 15 | x_shape = [1, 4, 4, 1] 16 | x_val = np.random.uniform(size=x_shape) 17 | 18 | x_data = tf.placeholder(tf.float32, shape=x_shape) 19 | 20 | # Create a layer that takes a spatial moving window average 21 | # Our window will be 2x2 with a stride of 2 for height and width 22 | # The filter value will be 0.25 because we want the average of the 2x2 window 23 | my_filter = tf.constant(0.25, shape=[2, 2, 1, 1]) 24 | my_strides = [1, 2, 2, 1] 25 | mov_avg_layer= tf.nn.conv2d(x_data, my_filter, my_strides, 26 | padding='SAME', name='Moving_Avg_Window') 27 | 28 | # Define a custom layer which will be sigmoid(Ax+b) where 29 | # x is a 2x2 matrix and A and b are 2x2 matrices 30 | def custom_layer(input_matrix): 31 | input_matrix_sqeezed = tf.squeeze(input_matrix) 32 | A = tf.constant([[1., 2.], [-1., 3.]]) 33 | b = tf.constant(1., shape=[2, 2]) 34 | temp1 = tf.matmul(A, input_matrix_sqeezed) 35 | temp = tf.add(temp1, b) # Ax + b 36 | return(tf.sigmoid(temp)) 37 | 38 | # Add custom layer to graph 39 | with tf.name_scope('Custom_Layer') as scope: 40 | custom_layer1 = custom_layer(mov_avg_layer) 41 | 42 | # The output should be an array that is 2x2, but size (1,2,2,1) 43 | print(sess.run(mov_avg_layer, feed_dict={x_data: x_val})) 44 | 45 | # After custom operation, size is now 2x2 (squeezed out size 1 dims) 46 | print(sess.run(custom_layer1, feed_dict={x_data: x_val})) 47 | 48 | merged = tf.summary.merge_all(key='summaries') 49 | 50 | if not os.path.exists('tensorboard_logs/'): 51 | os.makedirs('tensorboard_logs/') 52 | 53 | my_writer = tf.summary.FileWriter('tensorboard_logs/', sess.graph) 54 | -------------------------------------------------------------------------------- /02_TensorFlow_Way/03_Working_with_Multiple_Layers/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Multiple Layers 2 | 3 | ## Summary 4 | 5 | In this script, we will perform a 1D spatial moving average across a vector. Then we will create a "custom" operation by multiplying the output by a specific matrix: 6 | 7 | ## The Spatial Moving Window Layer 8 | We will create a layer that takes a spatial moving window average. Our window will be 2x2 with a stride of 2 for height and width. The filter value will be 0.25 because we want the average of the 2x2 window 9 | ``` 10 | my_filter = tf.constant(0.25, shape=[2, 2, 1, 1]) 11 | my_strides = [1, 2, 2, 1] 12 | mov_avg_layer= tf.nn.conv2d(x_data, my_filter, my_strides, padding='SAME', name='Moving_Avg_Window') 13 | ``` 14 | 15 | ## Custom Layer 16 | 17 | We create a custom layer which will be sigmoid(Ax+b) where x is a 2x2 matrix and A and b are 2x2 matrices. 18 | 19 | ``` 20 | output = sigmoid( input * A + b ) 21 | ``` 22 | 23 | ## Computational Graph Output 24 | 25 | Viewing the computational graph in Tensorboard: 26 | 27 | ![Multiple Layers](../images/03_Multiple_Layers.png "Multiple Layers on a Graph") -------------------------------------------------------------------------------- /02_TensorFlow_Way/04_Implementing_Loss_Functions/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Loss Functions 2 | 3 | ## Summary 4 | 5 | In this script, we will show how to implement different loss functions in tensorflow 6 | 7 | ## Plots of the Loss Functions 8 | 9 | The output of the script in this section plots the various loss functions: 10 | 11 | ![Loss Functions 1](../images/04_loss_fun1.png "Loss Functions 1") 12 | 13 | ![Loss Functions 2](../images/04_loss_fun2.png "Loss Functions 2") -------------------------------------------------------------------------------- /02_TensorFlow_Way/05_Implementing_Back_Propagation/05_back_propagation.py: -------------------------------------------------------------------------------- 1 | # Back Propagation 2 | #---------------------------------- 3 | # 4 | # This python function shows how to implement back propagation 5 | # in regression and classification models. 6 | 7 | import numpy as np 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # Create graph 13 | sess = tf.Session() 14 | 15 | # Regression Example: 16 | # We will create sample data as follows: 17 | # x-data: 100 random samples from a normal ~ N(1, 0.1) 18 | # target: 100 values of the value 10. 19 | # We will fit the model: 20 | # x-data * A = target 21 | # Theoretically, A = 10. 22 | 23 | # Create data 24 | x_vals = np.random.normal(1, 0.1, 100) 25 | y_vals = np.repeat(10., 100) 26 | x_data = tf.placeholder(shape=[1], dtype=tf.float32) 27 | y_target = tf.placeholder(shape=[1], dtype=tf.float32) 28 | 29 | # Create variable (one model parameter = A) 30 | A = tf.Variable(tf.random_normal(shape=[1])) 31 | 32 | # Add operation to graph 33 | my_output = tf.multiply(x_data, A) 34 | 35 | # Add L2 loss operation to graph 36 | loss = tf.square(my_output - y_target) 37 | 38 | # Create Optimizer 39 | my_opt = tf.train.GradientDescentOptimizer(0.02) 40 | train_step = my_opt.minimize(loss) 41 | 42 | # Initialize variables 43 | init = tf.global_variables_initializer() 44 | sess.run(init) 45 | 46 | # Run Loop 47 | for i in range(100): 48 | rand_index = np.random.choice(100) 49 | rand_x = [x_vals[rand_index]] 50 | rand_y = [y_vals[rand_index]] 51 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 52 | if (i+1)%25==0: 53 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 54 | print('Loss = ' + str(sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}))) 55 | 56 | # Classification Example 57 | # We will create sample data as follows: 58 | # x-data: sample 50 random values from a normal = N(-1, 1) 59 | # + sample 50 random values from a normal = N(1, 1) 60 | # target: 50 values of 0 + 50 values of 1. 61 | # These are essentially 100 values of the corresponding output index 62 | # We will fit the binary classification model: 63 | # If sigmoid(x+A) < 0.5 -> 0 else 1 64 | # Theoretically, A should be -(mean1 + mean2)/2 65 | 66 | ops.reset_default_graph() 67 | 68 | # Create graph 69 | sess = tf.Session() 70 | 71 | # Create data 72 | x_vals = np.concatenate((np.random.normal(-1, 1, 50), np.random.normal(3, 1, 50))) 73 | y_vals = np.concatenate((np.repeat(0., 50), np.repeat(1., 50))) 74 | x_data = tf.placeholder(shape=[1], dtype=tf.float32) 75 | y_target = tf.placeholder(shape=[1], dtype=tf.float32) 76 | 77 | # Create variable (one model parameter = A) 78 | A = tf.Variable(tf.random_normal(mean=10, shape=[1])) 79 | 80 | # Add operation to graph 81 | # Want to create the operstion sigmoid(x + A) 82 | # Note, the sigmoid() part is in the loss function 83 | my_output = tf.add(x_data, A) 84 | 85 | # Now we have to add another dimension to each (batch size of 1) 86 | my_output_expanded = tf.expand_dims(my_output, 0) 87 | y_target_expanded = tf.expand_dims(y_target, 0) 88 | 89 | # Initialize variables 90 | init = tf.global_variables_initializer() 91 | sess.run(init) 92 | 93 | # Add classification loss (cross entropy) 94 | xentropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=my_output_expanded, labels=y_target_expanded) 95 | 96 | # Create Optimizer 97 | my_opt = tf.train.GradientDescentOptimizer(0.05) 98 | train_step = my_opt.minimize(xentropy) 99 | 100 | # Run loop 101 | for i in range(1400): 102 | rand_index = np.random.choice(100) 103 | rand_x = [x_vals[rand_index]] 104 | rand_y = [y_vals[rand_index]] 105 | 106 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 107 | if (i+1)%200==0: 108 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 109 | print('Loss = ' + str(sess.run(xentropy, feed_dict={x_data: rand_x, y_target: rand_y}))) 110 | 111 | # Evaluate Predictions 112 | predictions = [] 113 | for i in range(len(x_vals)): 114 | x_val = [x_vals[i]] 115 | prediction = sess.run(tf.round(tf.sigmoid(my_output)), feed_dict={x_data: x_val}) 116 | predictions.append(prediction[0]) 117 | 118 | accuracy = sum(x==y for x,y in zip(predictions, y_vals))/100. 119 | print('Ending Accuracy = ' + str(np.round(accuracy, 2))) -------------------------------------------------------------------------------- /02_TensorFlow_Way/05_Implementing_Back_Propagation/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Back Propagation 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /02_TensorFlow_Way/06_Working_with_Batch_and_Stochastic_Training/06_batch_stochastic_training.py: -------------------------------------------------------------------------------- 1 | # Batch and Stochastic Training 2 | #---------------------------------- 3 | # 4 | # This python function illustrates two different training methods: 5 | # batch and stochastic training. For each model, we will use 6 | # a regression model that predicts one model variable. 7 | 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import tensorflow as tf 11 | from tensorflow.python.framework import ops 12 | ops.reset_default_graph() 13 | 14 | # We will implement a regression example in stochastic and batch training 15 | 16 | # Stochastic Training: 17 | # Create graph 18 | sess = tf.Session() 19 | 20 | # Create data 21 | x_vals = np.random.normal(1, 0.1, 100) 22 | y_vals = np.repeat(10., 100) 23 | x_data = tf.placeholder(shape=[1], dtype=tf.float32) 24 | y_target = tf.placeholder(shape=[1], dtype=tf.float32) 25 | 26 | # Create variable (one model parameter = A) 27 | A = tf.Variable(tf.random_normal(shape=[1])) 28 | 29 | # Add operation to graph 30 | my_output = tf.multiply(x_data, A) 31 | 32 | # Add L2 loss operation to graph 33 | loss = tf.square(my_output - y_target) 34 | 35 | # Create Optimizer 36 | my_opt = tf.train.GradientDescentOptimizer(0.02) 37 | train_step = my_opt.minimize(loss) 38 | 39 | # Initialize variables 40 | init = tf.global_variables_initializer() 41 | sess.run(init) 42 | 43 | loss_stochastic = [] 44 | # Run Loop 45 | for i in range(100): 46 | rand_index = np.random.choice(100) 47 | rand_x = [x_vals[rand_index]] 48 | rand_y = [y_vals[rand_index]] 49 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 50 | if (i+1)%5==0: 51 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 52 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 53 | print('Loss = ' + str(temp_loss)) 54 | loss_stochastic.append(temp_loss) 55 | 56 | 57 | # Batch Training: 58 | # Re-initialize graph 59 | ops.reset_default_graph() 60 | sess = tf.Session() 61 | 62 | # Declare batch size 63 | batch_size = 20 64 | 65 | # Create data 66 | x_vals = np.random.normal(1, 0.1, 100) 67 | y_vals = np.repeat(10., 100) 68 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 69 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 70 | 71 | # Create variable (one model parameter = A) 72 | A = tf.Variable(tf.random_normal(shape=[1,1])) 73 | 74 | # Add operation to graph 75 | my_output = tf.matmul(x_data, A) 76 | 77 | # Add L2 loss operation to graph 78 | loss = tf.reduce_mean(tf.square(my_output - y_target)) 79 | 80 | # Create Optimizer 81 | my_opt = tf.train.GradientDescentOptimizer(0.02) 82 | train_step = my_opt.minimize(loss) 83 | 84 | # Initialize variables 85 | init = tf.global_variables_initializer() 86 | sess.run(init) 87 | 88 | loss_batch = [] 89 | # Run Loop 90 | for i in range(100): 91 | rand_index = np.random.choice(100, size=batch_size) 92 | rand_x = np.transpose([x_vals[rand_index]]) 93 | rand_y = np.transpose([y_vals[rand_index]]) 94 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 95 | if (i+1)%5==0: 96 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 97 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 98 | print('Loss = ' + str(temp_loss)) 99 | loss_batch.append(temp_loss) 100 | 101 | plt.plot(range(0, 100, 5), loss_stochastic, 'b-', label='Stochastic Loss') 102 | plt.plot(range(0, 100, 5), loss_batch, 'r--', label='Batch Loss, size=20') 103 | plt.legend(loc='upper right', prop={'size': 11}) 104 | plt.show() -------------------------------------------------------------------------------- /02_TensorFlow_Way/06_Working_with_Batch_and_Stochastic_Training/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Batch and Stochastic Training 2 | 3 | ## Summary 4 | 5 | Here, we introduce the differences between batch and stochastic training and how to implement both in TensorFlow. Stochastic training is defined as training on one observation at once, and batch training is defined as training on a group of observations at once. 6 | 7 | ## Model 8 | In this script, we use generated data. We will generate input data that is distributed as Normal(mean=1, sd=0.1). All the target data will be is the value 10.0 repeated. The model will try to predict the multiplication factor to minimize the loss between the model output and the value 10.0. 9 | 10 | ## Notes 11 | 12 | It is important to note that TensorFlow works well with many dimensional matrices, so we can easily implement batch training by adding the batch dimension to our inputs, as illustrated in this script. 13 | 14 | ## Viewing the Difference 15 | 16 | Here we plot the loss function of stochastic and batch training on the same graph. Notice how stochastic training is less smooth in the convergence of a solution. This may sound like a bad thing, but it can help explore sample spaces and be less likely to be stuck in local minima. 17 | 18 | ![Stochastic and Batch](../images/06_Back_Propagation.png "Stochastic and Batch Loss") -------------------------------------------------------------------------------- /02_TensorFlow_Way/07_Combining_Everything_Together/07_combining_everything_together.py: -------------------------------------------------------------------------------- 1 | # Combining Everything Together 2 | #---------------------------------- 3 | # This file will perform binary classification on the 4 | # iris dataset. We will only predict if a flower is 5 | # I.setosa or not. 6 | # 7 | # We will create a simple binary classifier by creating a line 8 | # and running everything through a sigmoid to get a binary predictor. 9 | # The two features we will use are pedal length and pedal width. 10 | # 11 | # We will use batch training, but this can be easily 12 | # adapted to stochastic training. 13 | 14 | import matplotlib.pyplot as plt 15 | import numpy as np 16 | from sklearn import datasets 17 | import tensorflow as tf 18 | from tensorflow.python.framework import ops 19 | ops.reset_default_graph() 20 | 21 | # Load the iris data 22 | # iris.target = {0, 1, 2}, where '0' is setosa 23 | # iris.data ~ [sepal.width, sepal.length, pedal.width, pedal.length] 24 | iris = datasets.load_iris() 25 | binary_target = np.array([1. if x==0 else 0. for x in iris.target]) 26 | iris_2d = np.array([[x[2], x[3]] for x in iris.data]) 27 | 28 | # Declare batch size 29 | batch_size = 20 30 | 31 | # Create graph 32 | sess = tf.Session() 33 | 34 | # Declare placeholders 35 | x1_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 36 | x2_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 37 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 38 | 39 | # Create variables A and b (0 = x1 - A*x2 + b) 40 | A = tf.Variable(tf.random_normal(shape=[1, 1])) 41 | b = tf.Variable(tf.random_normal(shape=[1, 1])) 42 | 43 | # Add model to graph: 44 | # x1 - A*x2 + b 45 | my_mult = tf.matmul(x2_data, A) 46 | my_add = tf.add(my_mult, b) 47 | my_output = tf.subtract(x1_data, my_add) 48 | 49 | # Add classification loss (cross entropy) 50 | xentropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=my_output, labels=y_target) 51 | 52 | # Create Optimizer 53 | my_opt = tf.train.GradientDescentOptimizer(0.05) 54 | train_step = my_opt.minimize(xentropy) 55 | 56 | # Initialize variables 57 | init = tf.global_variables_initializer() 58 | sess.run(init) 59 | 60 | # Run Loop 61 | for i in range(1000): 62 | rand_index = np.random.choice(len(iris_2d), size=batch_size) 63 | #rand_x = np.transpose([iris_2d[rand_index]]) 64 | rand_x = iris_2d[rand_index] 65 | rand_x1 = np.array([[x[0]] for x in rand_x]) 66 | rand_x2 = np.array([[x[1]] for x in rand_x]) 67 | #rand_y = np.transpose([binary_target[rand_index]]) 68 | rand_y = np.array([[y] for y in binary_target[rand_index]]) 69 | sess.run(train_step, feed_dict={x1_data: rand_x1, x2_data: rand_x2, y_target: rand_y}) 70 | if (i+1)%200==0: 71 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ', b = ' + str(sess.run(b))) 72 | 73 | 74 | # Visualize Results 75 | # Pull out slope/intercept 76 | [[slope]] = sess.run(A) 77 | [[intercept]] = sess.run(b) 78 | 79 | # Create fitted line 80 | x = np.linspace(0, 3, num=50) 81 | ablineValues = [] 82 | for i in x: 83 | ablineValues.append(slope*i+intercept) 84 | 85 | # Plot the fitted line over the data 86 | setosa_x = [a[1] for i,a in enumerate(iris_2d) if binary_target[i]==1] 87 | setosa_y = [a[0] for i,a in enumerate(iris_2d) if binary_target[i]==1] 88 | non_setosa_x = [a[1] for i,a in enumerate(iris_2d) if binary_target[i]==0] 89 | non_setosa_y = [a[0] for i,a in enumerate(iris_2d) if binary_target[i]==0] 90 | plt.plot(setosa_x, setosa_y, 'rx', ms=10, mew=2, label='setosa') 91 | plt.plot(non_setosa_x, non_setosa_y, 'ro', label='Non-setosa') 92 | plt.plot(x, ablineValues, 'b-') 93 | plt.xlim([0.0, 2.7]) 94 | plt.ylim([0.0, 7.1]) 95 | plt.suptitle('Linear Separator For I.setosa', fontsize=20) 96 | plt.xlabel('Petal Length') 97 | plt.ylabel('Petal Width') 98 | plt.legend(loc='lower right') 99 | plt.show() -------------------------------------------------------------------------------- /02_TensorFlow_Way/07_Combining_Everything_Together/readme.md: -------------------------------------------------------------------------------- 1 | # Combining Everything Together 2 | 3 | ## Summary 4 | 5 | We will create a very simple linear classifier in this example. The data set of interest will be the Iris data. We will optimize a linear separator between pedal length and pedal width to classify if a flower is _I. setosa_ or not. The reason we setup the data this way is because it will end up being linear seperable. 6 | 7 | ## Notes 8 | We use the sigmoid cross entropy loss because we are predicting a binary class (sigmoid) and it is a classification problem (cross entropy). 9 | 10 | ## Results 11 | 12 | Viewing the resulting graph and separable line: 13 | 14 | ![Isetosa](../images/07_Combing_Everything_Together.png "I setosa Seperability") -------------------------------------------------------------------------------- /02_TensorFlow_Way/08_Evaluating_Models/readme.md: -------------------------------------------------------------------------------- 1 | # Evaluating Models 2 | 3 | ## Summary 4 | 5 | Here we will perform two modeling tasks: regression and classification. We will show how to evaluate the models in the TensorFlow algorithm 6 | 7 | ## Regression Model 8 | 9 | The regression model is from a prior section. We will generate input data which will be distributed as Normal(mean=1, sd=0.1) and target data or repeating 10.0 values. The model will optimize a multiplication factor (theoretically = 10) to predict the target data. 10 | 11 | ## Classification Model 12 | 13 | The classification model will be half values from Normal(mean=-1, sd=1) and half from Normal(mean=2, sd=1). There will be some overlap in the distributions. The classification model will optimize on predicting a middle point, of which is will classify everything less than it as the first distribution and everything greater than it as the latter distribution. Theoretically, we know that the ideal cutoff is between the two normals at x = 0.5. 14 | 15 | ## Outline 16 | 17 | The idea is that we want to split our labeled data set into a training and test set. We then train on the test set, and look at the accuracy on the test set. 18 | 19 | ## Classification Results 20 | 21 | Here is the classification results visualized as a histogram: 22 | 23 | ![Classification Results](../images/08_Evaluating_Models.png "Classification Results") -------------------------------------------------------------------------------- /02_TensorFlow_Way/images/01_Operations_on_a_Graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/02_TensorFlow_Way/images/01_Operations_on_a_Graph.png -------------------------------------------------------------------------------- /02_TensorFlow_Way/images/02_Multiple_Operations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/02_TensorFlow_Way/images/02_Multiple_Operations.png -------------------------------------------------------------------------------- /02_TensorFlow_Way/images/03_Multiple_Layers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/02_TensorFlow_Way/images/03_Multiple_Layers.png -------------------------------------------------------------------------------- /02_TensorFlow_Way/images/04_loss_fun1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/02_TensorFlow_Way/images/04_loss_fun1.png -------------------------------------------------------------------------------- /02_TensorFlow_Way/images/04_loss_fun2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/02_TensorFlow_Way/images/04_loss_fun2.png -------------------------------------------------------------------------------- /02_TensorFlow_Way/images/06_Back_Propagation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/02_TensorFlow_Way/images/06_Back_Propagation.png -------------------------------------------------------------------------------- /02_TensorFlow_Way/images/07_Combing_Everything_Together.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/02_TensorFlow_Way/images/07_Combing_Everything_Together.png -------------------------------------------------------------------------------- /02_TensorFlow_Way/images/08_Evaluating_Models.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/02_TensorFlow_Way/images/08_Evaluating_Models.png -------------------------------------------------------------------------------- /02_TensorFlow_Way/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 2: The TensorFlow Way 2 | 3 | After we have established the basic objects and methods in TensorFlow, we now want to establish the components that make up TensorFlow algorithms. We start by introducing computational graphs, and then move to loss functions and back propagation. We end with creating a simple classifier and then show an example of evaluating regression and classification algorithms. 4 | 5 | 1. [One Operation as a Computational Graph](01_Operations_as_a_Computational_Graph#operations-as-a-computational-graph) 6 | * We show how to create an operation on a computational graph and how to visualize it using Tensorboard. 7 | 2. [Layering Nested Operations](02_Layering_Nested_Operations#multiple-operations-on-a-computational-graph) 8 | * We show how to create multiple operations on a computational graph and how to visualize them using Tensorboard. 9 | 3. [Working with Multiple Layers](03_Working_with_Multiple_Layers#working-with-multiple-layers) 10 | * Here we extend the usage of the computational graph to create multiple layers and show how they appear in Tensorboard. 11 | 4. [Implementing Loss Functions](04_Implementing_Loss_Functions#implementing-loss-functions) 12 | * In order to train a model, we must be able to evaluate how well it is doing. This is given by loss functions. We plot various loss functions and talk about the benefits and limitations of some. 13 | 5. [Implementing Back Propagation](05_Implementing_Back_Propagation#implementing-back-propagation) 14 | * Here we show how to use loss functions to iterate through data and back propagate errors for regression and classification. 15 | 6. [Working with Stochastic and Batch Training](06_Working_with_Batch_and_Stochastic_Training#working-with-batch-and-stochastic-training) 16 | * TensorFlow makes it easy to use both batch and stochastic training. We show how to implement both and talk about the benefits and limitations of each. 17 | 7. [Combining Everything Together](07_Combining_Everything_Together#combining-everything-together) 18 | * We now combine everything together that we have learned and create a simple classifier. 19 | 8. [Evaluating Models](08_Evaluating_Models#evaluating-models) 20 | * Any model is only as good as it's evaluation. Here we show two examples of (1) evaluating a regression algorithm and (2) a classification algorithm. 21 | -------------------------------------------------------------------------------- /03_Linear_Regression/01_Using_the_Matrix_Inverse_Method/01_lin_reg_inverse.py: -------------------------------------------------------------------------------- 1 | # Linear Regression: Inverse Matrix Method 2 | #---------------------------------- 3 | # 4 | # This function shows how to use TensorFlow to 5 | # solve linear regression via the matrix inverse. 6 | # 7 | # Given Ax=b, solving for x: 8 | # x = (t(A) * A)^(-1) * t(A) * b 9 | # where t(A) is the transpose of A 10 | 11 | import matplotlib.pyplot as plt 12 | import numpy as np 13 | import tensorflow as tf 14 | from tensorflow.python.framework import ops 15 | ops.reset_default_graph() 16 | 17 | # Create graph 18 | sess = tf.Session() 19 | 20 | # Create the data 21 | x_vals = np.linspace(0, 10, 100) 22 | y_vals = x_vals + np.random.normal(0, 1, 100) 23 | 24 | # Create design matrix 25 | x_vals_column = np.transpose(np.matrix(x_vals)) 26 | ones_column = np.transpose(np.matrix(np.repeat(1, 100))) 27 | A = np.column_stack((x_vals_column, ones_column)) 28 | 29 | # Create b matrix 30 | b = np.transpose(np.matrix(y_vals)) 31 | 32 | # Create tensors 33 | A_tensor = tf.constant(A) 34 | b_tensor = tf.constant(b) 35 | 36 | # Matrix inverse solution 37 | tA_A = tf.matmul(tf.transpose(A_tensor), A_tensor) 38 | tA_A_inv = tf.matrix_inverse(tA_A) 39 | product = tf.matmul(tA_A_inv, tf.transpose(A_tensor)) 40 | solution = tf.matmul(product, b_tensor) 41 | 42 | solution_eval = sess.run(solution) 43 | 44 | # Extract coefficients 45 | slope = solution_eval[0][0] 46 | y_intercept = solution_eval[1][0] 47 | 48 | print('slope: ' + str(slope)) 49 | print('y_intercept: ' + str(y_intercept)) 50 | 51 | # Get best fit line 52 | best_fit = [] 53 | for i in x_vals: 54 | best_fit.append(slope*i+y_intercept) 55 | 56 | # Plot the results 57 | plt.plot(x_vals, y_vals, 'o', label='Data') 58 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 59 | plt.legend(loc='upper left') 60 | plt.show() -------------------------------------------------------------------------------- /03_Linear_Regression/01_Using_the_Matrix_Inverse_Method/readme.md: -------------------------------------------------------------------------------- 1 | # Using the Matrix Inverse Method 2 | 3 | Here we implement solving 2D linear regression via the matrix inverse method in TensorFlow. 4 | 5 | # Model 6 | 7 | Given A * x = b, we can solve for x via: 8 | 9 | (t(A) * A) * x = t(A) * b 10 | 11 | x = (t(A) * A)^(-1) * t(A) * b 12 | 13 | Here, note that t(A) is the transpose of A. 14 | 15 | # Graph of linear fit 16 | 17 | ![Matrix Inverse Method](../images/01_Inverse_Matrix_Method.png "Matrix Inverse Method") 18 | -------------------------------------------------------------------------------- /03_Linear_Regression/02_Implementing_a_Decomposition_Method/02_lin_reg_decomposition.py: -------------------------------------------------------------------------------- 1 | # Linear Regression: Decomposition Method 2 | #---------------------------------- 3 | # 4 | # This function shows how to use TensorFlow to 5 | # solve linear regression via the matrix inverse. 6 | # 7 | # Given Ax=b, and a Cholesky decomposition such that 8 | # A = L*L' then we can get solve for x via 9 | # 1) L*y=t(A)*b 10 | # 2) L'*x=y 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | # Create graph 19 | sess = tf.Session() 20 | 21 | # Create the data 22 | x_vals = np.linspace(0, 10, 100) 23 | y_vals = x_vals + np.random.normal(0, 1, 100) 24 | 25 | # Create design matrix 26 | x_vals_column = np.transpose(np.matrix(x_vals)) 27 | ones_column = np.transpose(np.matrix(np.repeat(1, 100))) 28 | A = np.column_stack((x_vals_column, ones_column)) 29 | 30 | # Create b matrix 31 | b = np.transpose(np.matrix(y_vals)) 32 | 33 | # Create tensors 34 | A_tensor = tf.constant(A) 35 | b_tensor = tf.constant(b) 36 | 37 | # Find Cholesky Decomposition 38 | tA_A = tf.matmul(tf.transpose(A_tensor), A_tensor) 39 | L = tf.cholesky(tA_A) 40 | 41 | # Solve L*y=t(A)*b 42 | tA_b = tf.matmul(tf.transpose(A_tensor), b) 43 | sol1 = tf.matrix_solve(L, tA_b) 44 | 45 | # Solve L' * y = sol1 46 | sol2 = tf.matrix_solve(tf.transpose(L), sol1) 47 | 48 | solution_eval = sess.run(sol2) 49 | 50 | # Extract coefficients 51 | slope = solution_eval[0][0] 52 | y_intercept = solution_eval[1][0] 53 | 54 | print('slope: ' + str(slope)) 55 | print('y_intercept: ' + str(y_intercept)) 56 | 57 | # Get best fit line 58 | best_fit = [] 59 | for i in x_vals: 60 | best_fit.append(slope*i+y_intercept) 61 | 62 | # Plot the results 63 | plt.plot(x_vals, y_vals, 'o', label='Data') 64 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 65 | plt.legend(loc='upper left') 66 | plt.show() -------------------------------------------------------------------------------- /03_Linear_Regression/02_Implementing_a_Decomposition_Method/readme.md: -------------------------------------------------------------------------------- 1 | # Using the Cholesky Decomposition Method 2 | 3 | Here we implement solving 2D linear regression via the Cholesky decomposition in TensorFlow. 4 | 5 | # Model 6 | 7 | Given A * x = b, and a Cholesky decomposition such that A = L*L' then we can get solve for x via 8 | 1. Solving L * y = t(A) * b for y 9 | 2. Solving L' * x = y for x. 10 | 11 | # Graph of linear fit 12 | 13 | ![Cholesky decomposition](../images/02_Cholesky_Decomposition.png "Cholesky decomposition") 14 | -------------------------------------------------------------------------------- /03_Linear_Regression/03_TensorFlow_Way_of_Linear_Regression/03_lin_reg_tensorflow_way.py: -------------------------------------------------------------------------------- 1 | # Linear Regression: TensorFlow Way 2 | #---------------------------------- 3 | # 4 | # This function shows how to use TensorFlow to 5 | # solve linear regression. 6 | # y = Ax + b 7 | # 8 | # We will use the iris data, specifically: 9 | # y = Sepal Length 10 | # x = Petal Width 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from sklearn import datasets 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Create graph 20 | sess = tf.Session() 21 | 22 | # Load the data 23 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 24 | iris = datasets.load_iris() 25 | x_vals = np.array([x[3] for x in iris.data]) 26 | y_vals = np.array([y[0] for y in iris.data]) 27 | 28 | # Declare batch size 29 | batch_size = 25 30 | 31 | # Initialize placeholders 32 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 33 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 34 | 35 | # Create variables for linear regression 36 | A = tf.Variable(tf.random_normal(shape=[1,1])) 37 | b = tf.Variable(tf.random_normal(shape=[1,1])) 38 | 39 | # Declare model operations 40 | model_output = tf.add(tf.matmul(x_data, A), b) 41 | 42 | # Declare loss function (L2 loss) 43 | loss = tf.reduce_mean(tf.square(y_target - model_output)) 44 | 45 | # Declare optimizer 46 | my_opt = tf.train.GradientDescentOptimizer(0.05) 47 | train_step = my_opt.minimize(loss) 48 | 49 | # Initialize variables 50 | init = tf.global_variables_initializer() 51 | sess.run(init) 52 | 53 | # Training loop 54 | loss_vec = [] 55 | for i in range(100): 56 | rand_index = np.random.choice(len(x_vals), size=batch_size) 57 | rand_x = np.transpose([x_vals[rand_index]]) 58 | rand_y = np.transpose([y_vals[rand_index]]) 59 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 60 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 61 | loss_vec.append(temp_loss) 62 | if (i+1)%25==0: 63 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 64 | print('Loss = ' + str(temp_loss)) 65 | 66 | # Get the optimal coefficients 67 | [slope] = sess.run(A) 68 | [y_intercept] = sess.run(b) 69 | 70 | # Get best fit line 71 | best_fit = [] 72 | for i in x_vals: 73 | best_fit.append(slope*i+y_intercept) 74 | 75 | # Plot the result 76 | plt.plot(x_vals, y_vals, 'o', label='Data Points') 77 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 78 | plt.legend(loc='upper left') 79 | plt.title('Sepal Length vs Petal Width') 80 | plt.xlabel('Petal Width') 81 | plt.ylabel('Sepal Length') 82 | plt.show() 83 | 84 | # Plot loss over time 85 | plt.plot(loss_vec, 'k-') 86 | plt.title('L2 Loss per Generation') 87 | plt.xlabel('Generation') 88 | plt.ylabel('L2 Loss') 89 | plt.show() 90 | -------------------------------------------------------------------------------- /03_Linear_Regression/03_TensorFlow_Way_of_Linear_Regression/readme.md: -------------------------------------------------------------------------------- 1 | # Learning the TensorFlow Way of Regression 2 | 3 | In this section we will implement linear regression as an iterative computational graph in TensorFlow. To make this more pertinent, instead of using generated data, we will instead use the Iris data set. Our x will be the Petal Width, our y will be the Sepal Length. Viewing the data in these two dimensions suggests a linear relationship. 4 | 5 | # Model 6 | 7 | The the output of our model is a 2D linear regression: 8 | 9 | y = A * x + b 10 | 11 | The x matrix input will be a 2D matrix, where it's dimensions will be (batch size x 1). The y target output will have the same dimensions, (batch size x 1). 12 | 13 | The loss function we will use will be the mean of the batch L2 Loss: 14 | 15 | loss = mean( (y\_target - model\_output)^2 ) 16 | 17 | We will then iterate through random batch size selections of the data. 18 | 19 | # Graph of Loss Function 20 | 21 | ![Regression Loss](../images/03_lin_reg_loss.png "Regression Loss") 22 | 23 | # Graph of Linear Fit 24 | 25 | ![TF Regression](../images/02_Cholesky_Decomposition.png "TF Regression") 26 | -------------------------------------------------------------------------------- /03_Linear_Regression/04_Loss_Functions_in_Linear_Regressions/04_lin_reg_l1_vs_l2.py: -------------------------------------------------------------------------------- 1 | # Linear Regression: L1 vs L2 2 | #---------------------------------- 3 | # 4 | # This function shows how to use TensorFlow to 5 | # solve linear regression via the matrix inverse. 6 | 7 | import matplotlib.pyplot as plt 8 | import numpy as np 9 | import tensorflow as tf 10 | from sklearn import datasets 11 | from tensorflow.python.framework import ops 12 | ops.reset_default_graph() 13 | 14 | # Create graph 15 | sess = tf.Session() 16 | 17 | # Load the data 18 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 19 | iris = datasets.load_iris() 20 | x_vals = np.array([x[3] for x in iris.data]) 21 | y_vals = np.array([y[0] for y in iris.data]) 22 | 23 | # Declare batch size and number of iterations 24 | batch_size = 25 25 | learning_rate = 0.4 # Will not converge with learning rate at 0.4 26 | iterations = 50 27 | 28 | # Initialize placeholders 29 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 30 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 31 | 32 | # Create variables for linear regression 33 | A = tf.Variable(tf.random_normal(shape=[1,1])) 34 | b = tf.Variable(tf.random_normal(shape=[1,1])) 35 | 36 | # Declare model operations 37 | model_output = tf.add(tf.matmul(x_data, A), b) 38 | 39 | # Declare loss functions 40 | loss_l1 = tf.reduce_mean(tf.abs(y_target - model_output)) 41 | 42 | # Declare optimizers 43 | my_opt_l1 = tf.train.GradientDescentOptimizer(learning_rate) 44 | train_step_l1 = my_opt_l1.minimize(loss_l1) 45 | 46 | # Initialize variables 47 | init = tf.global_variables_initializer() 48 | sess.run(init) 49 | 50 | # Training loop 51 | loss_vec_l1 = [] 52 | for i in range(iterations): 53 | rand_index = np.random.choice(len(x_vals), size=batch_size) 54 | rand_x = np.transpose([x_vals[rand_index]]) 55 | rand_y = np.transpose([y_vals[rand_index]]) 56 | sess.run(train_step_l1, feed_dict={x_data: rand_x, y_target: rand_y}) 57 | temp_loss_l1 = sess.run(loss_l1, feed_dict={x_data: rand_x, y_target: rand_y}) 58 | loss_vec_l1.append(temp_loss_l1) 59 | if (i+1)%25==0: 60 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 61 | 62 | 63 | # L2 Loss 64 | # Reinitialize graph 65 | ops.reset_default_graph() 66 | 67 | # Create graph 68 | sess = tf.Session() 69 | 70 | # Initialize placeholders 71 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 72 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 73 | 74 | # Create variables for linear regression 75 | A = tf.Variable(tf.random_normal(shape=[1,1])) 76 | b = tf.Variable(tf.random_normal(shape=[1,1])) 77 | 78 | # Declare model operations 79 | model_output = tf.add(tf.matmul(x_data, A), b) 80 | 81 | # Declare loss functions 82 | loss_l2 = tf.reduce_mean(tf.square(y_target - model_output)) 83 | 84 | # Declare optimizers 85 | my_opt_l2 = tf.train.GradientDescentOptimizer(learning_rate) 86 | train_step_l2 = my_opt_l2.minimize(loss_l2) 87 | 88 | # Initialize variables 89 | init = tf.global_variables_initializer() 90 | sess.run(init) 91 | 92 | loss_vec_l2 = [] 93 | for i in range(iterations): 94 | rand_index = np.random.choice(len(x_vals), size=batch_size) 95 | rand_x = np.transpose([x_vals[rand_index]]) 96 | rand_y = np.transpose([y_vals[rand_index]]) 97 | sess.run(train_step_l2, feed_dict={x_data: rand_x, y_target: rand_y}) 98 | temp_loss_l2 = sess.run(loss_l2, feed_dict={x_data: rand_x, y_target: rand_y}) 99 | loss_vec_l2.append(temp_loss_l2) 100 | if (i+1)%25==0: 101 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 102 | 103 | 104 | # Plot loss over time 105 | plt.plot(loss_vec_l1, 'k-', label='L1 Loss') 106 | plt.plot(loss_vec_l2, 'r--', label='L2 Loss') 107 | plt.title('L1 and L2 Loss per Generation') 108 | plt.xlabel('Generation') 109 | plt.ylabel('L1 Loss') 110 | plt.legend(loc='upper right') 111 | plt.show() 112 | -------------------------------------------------------------------------------- /03_Linear_Regression/04_Loss_Functions_in_Linear_Regressions/readme.md: -------------------------------------------------------------------------------- 1 | # Loss Functions in Linear Regression 2 | 3 | The choice of loss function can significantly impact the convergence of TensorFlow algorithms. We will compare and contrast using the L1 and L2 loss functions for linear regression. 4 | 5 | # L1 Loss 6 | 7 | The L1 loss is the absolute value of the difference between the target (y) and the prediction (p): 8 | 9 | L1(y,p) = |y - p| 10 | 11 | # L2 Loss 12 | 13 | The L2 loss is the squared difference between the target (y) and the prediction (p): 14 | 15 | L2(y,p) = (y - p)^2 16 | 17 | ## Summary 18 | 19 | | L2 Loss | L1 Loss | 20 | |-----------------|---------------| 21 | | More stable | Less Stable | 22 | | Not very robust | Robust | 23 | 24 | 25 | # Graph of L1 vs L2 Loss Functions 26 | 27 | ![L1 and L2 Loss](../images/04_L1_L2_loss.png "L1 and L2 Loss") 28 | 29 | # Graph of L1 vs L2 Loss Functions (L2 not converging) 30 | 31 | Here is an example of the L2 function not converging. Despite a large learning rate, L1 has converged but L2 has not. 32 | 33 | ![L2 Not Converging](../images/04_L1_L2_loss2.png "L2 Not Converging") 34 | 35 | # Graphical Summary of L1 and L2 with Learning Rates 36 | 37 | Here is a plot of a 1D example of L1 and L2 loss with a small and large learning rate. 38 | 39 | ![L1 and L2](../images/04_L1_L2_learningrates.png "L1 and L2") 40 | 41 | To note: 42 | 43 | ### Top Left 44 | - L1 loss with small learning rate: Robust, converges, but may take a while to converge. 45 | 46 | ### Top Right 47 | - L2 loss with small learning rate: Robust, converges, but may take a while to converge. 48 | 49 | ### Bottom Left 50 | - L1 loss with large learning rate: More robust, and less likely to explode, but may bounce around the optimum at the end. 51 | 52 | ### Bottom Right 53 | - L2 loss with large learning rate: Not robust, explodes because of large learning rate. Very sensitive to learning rate. 54 | 55 | Moral of the story: When your algorithm isn't converging, try decreasing the learning rate first. 56 | -------------------------------------------------------------------------------- /03_Linear_Regression/05_Implementing_Deming_Regression/05_deming_regression.py: -------------------------------------------------------------------------------- 1 | # Deming Regression 2 | #---------------------------------- 3 | # 4 | # This function shows how to use TensorFlow to 5 | # solve linear Deming regression. 6 | # y = Ax + b 7 | # 8 | # We will use the iris data, specifically: 9 | # y = Sepal Length 10 | # x = Petal Width 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from sklearn import datasets 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Set a random seed 20 | tf.set_random_seed(42) 21 | np.random.seed(42) 22 | 23 | # Create graph 24 | sess = tf.Session() 25 | 26 | # Load the data 27 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 28 | iris = datasets.load_iris() 29 | x_vals = np.array([x[3] for x in iris.data]) 30 | y_vals = np.array([y[0] for y in iris.data]) 31 | 32 | # Declare batch size 33 | batch_size = 50 34 | 35 | # Initialize placeholders 36 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 37 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 38 | 39 | # Create variables for linear regression 40 | A = tf.Variable(tf.random_normal(shape=[1,1])) 41 | b = tf.Variable(tf.random_normal(shape=[1,1])) 42 | 43 | # Declare model operations 44 | model_output = tf.add(tf.matmul(x_data, A), b) 45 | 46 | # Declare Deming loss function 47 | deming_numerator = tf.abs(tf.subtract(y_target, tf.add(tf.matmul(x_data, A), b))) 48 | deming_denominator = tf.sqrt(tf.add(tf.square(A),1)) 49 | loss = tf.reduce_mean(tf.truediv(deming_numerator, deming_denominator)) 50 | 51 | # Declare optimizer 52 | my_opt = tf.train.GradientDescentOptimizer(0.15) 53 | train_step = my_opt.minimize(loss) 54 | 55 | # Initialize variables 56 | init = tf.global_variables_initializer() 57 | sess.run(init) 58 | 59 | # Training loop 60 | loss_vec = [] 61 | for i in range(1000): 62 | rand_index = np.random.choice(len(x_vals), size=batch_size) 63 | rand_x = np.transpose([x_vals[rand_index]]) 64 | rand_y = np.transpose([y_vals[rand_index]]) 65 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 66 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 67 | loss_vec.append(temp_loss) 68 | if (i+1)%50 == 0: 69 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 70 | print('Loss = ' + str(temp_loss)) 71 | 72 | # Get the optimal coefficients 73 | [slope] = sess.run(A) 74 | [y_intercept] = sess.run(b) 75 | 76 | # Get best fit line 77 | best_fit = [] 78 | for i in x_vals: 79 | best_fit.append(slope*i+y_intercept) 80 | 81 | # Plot the result 82 | plt.plot(x_vals, y_vals, 'o', label='Data Points') 83 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 84 | plt.legend(loc='upper left') 85 | plt.title('Sepal Length vs Petal Width') 86 | plt.xlabel('Petal Width') 87 | plt.ylabel('Sepal Length') 88 | plt.show() 89 | 90 | # Plot loss over time 91 | plt.plot(loss_vec, 'k-') 92 | plt.title('L2 Loss per Generation') 93 | plt.xlabel('Generation') 94 | plt.ylabel('L2 Loss') 95 | plt.show() 96 | -------------------------------------------------------------------------------- /03_Linear_Regression/05_Implementing_Deming_Regression/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Deming Regression 2 | 3 | Deming regression, also known as total regression, is regular regression that minimizes the shortest distance to the line. Contrast this to regular regression, in which we aim to minimize the vertical distance between the model output and the y-target values. 4 | 5 | ![Deming Regression](../images/05_demming_vs_linear_reg.png "Deming Regression") 6 | 7 | # Model 8 | 9 | The model will be the same as regular linear regression: 10 | 11 | y = A * x + b 12 | 13 | Instead of measuring the vertical L2 distance, we will measure the shortest distance between the line and the predicted point in the loss function. 14 | 15 | loss = |y\_target - (A * x\_input + b)| / sqrt(A^2 + 1) 16 | 17 | # Graph of Linear Fit 18 | 19 | ![Deming Output](../images/05_demming_reg.png "Deming Output") 20 | -------------------------------------------------------------------------------- /03_Linear_Regression/06_Implementing_Lasso_and_Ridge_Regression/06_lasso_and_ridge_regression.py: -------------------------------------------------------------------------------- 1 | # LASSO and Ridge Regression 2 | # 3 | # This function shows how to use TensorFlow to solve LASSO or 4 | # Ridge regression for 5 | # y = Ax + b 6 | # 7 | # We will use the iris data, specifically: 8 | # y = Sepal Length 9 | # x = Petal Width 10 | 11 | # import required libraries 12 | import matplotlib.pyplot as plt 13 | import sys 14 | import numpy as np 15 | import tensorflow as tf 16 | from sklearn import datasets 17 | from tensorflow.python.framework import ops 18 | 19 | 20 | # Specify 'Ridge' or 'LASSO' 21 | #regression_type = 'LASSO' 22 | regression_type = 'Ridge' 23 | 24 | # clear out old graph 25 | ops.reset_default_graph() 26 | 27 | # Create graph 28 | sess = tf.Session() 29 | 30 | ### 31 | # Load iris data 32 | ### 33 | 34 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 35 | iris = datasets.load_iris() 36 | x_vals = np.array([x[3] for x in iris.data]) 37 | y_vals = np.array([y[0] for y in iris.data]) 38 | 39 | ### 40 | # Model Parameters 41 | ### 42 | 43 | # Declare batch size 44 | batch_size = 50 45 | 46 | # Initialize placeholders 47 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 48 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 49 | 50 | # make results reproducible 51 | seed = 13 52 | np.random.seed(seed) 53 | tf.set_random_seed(seed) 54 | 55 | # Create variables for linear regression 56 | A = tf.Variable(tf.random_normal(shape=[1,1])) 57 | b = tf.Variable(tf.random_normal(shape=[1,1])) 58 | 59 | # Declare model operations 60 | model_output = tf.add(tf.matmul(x_data, A), b) 61 | 62 | ### 63 | # Loss Functions 64 | ### 65 | 66 | # Select appropriate loss function based on regression type 67 | 68 | if regression_type == 'LASSO': 69 | # Declare Lasso loss function 70 | # Lasso Loss = L2_Loss + heavyside_step, 71 | # Where heavyside_step ~ 0 if A < constant, otherwise ~ 99 72 | lasso_param = tf.constant(0.9) 73 | heavyside_step = tf.truediv(1., tf.add(1., tf.exp(tf.multiply(-50., tf.subtract(A, lasso_param))))) 74 | regularization_param = tf.multiply(heavyside_step, 99.) 75 | loss = tf.add(tf.reduce_mean(tf.square(y_target - model_output)), regularization_param) 76 | 77 | elif regression_type == 'Ridge': 78 | # Declare the Ridge loss function 79 | # Ridge loss = L2_loss + L2 norm of slope 80 | ridge_param = tf.constant(1.) 81 | ridge_loss = tf.reduce_mean(tf.square(A)) 82 | loss = tf.expand_dims(tf.add(tf.reduce_mean(tf.square(y_target - model_output)), tf.multiply(ridge_param, ridge_loss)), 0) 83 | 84 | else: 85 | print('Invalid regression_type parameter value',file=sys.stderr) 86 | 87 | 88 | ### 89 | # Optimizer 90 | ### 91 | 92 | # Declare optimizer 93 | my_opt = tf.train.GradientDescentOptimizer(0.001) 94 | train_step = my_opt.minimize(loss) 95 | 96 | ### 97 | # Run regression 98 | ### 99 | 100 | # Initialize variables 101 | init = tf.global_variables_initializer() 102 | sess.run(init) 103 | 104 | # Training loop 105 | loss_vec = [] 106 | for i in range(1500): 107 | rand_index = np.random.choice(len(x_vals), size=batch_size) 108 | rand_x = np.transpose([x_vals[rand_index]]) 109 | rand_y = np.transpose([y_vals[rand_index]]) 110 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 111 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 112 | loss_vec.append(temp_loss[0]) 113 | if (i+1)%300==0: 114 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 115 | print('Loss = ' + str(temp_loss)) 116 | print('\n') 117 | 118 | ### 119 | # Extract regression results 120 | ### 121 | 122 | # Get the optimal coefficients 123 | [slope] = sess.run(A) 124 | [y_intercept] = sess.run(b) 125 | 126 | # Get best fit line 127 | best_fit = [] 128 | for i in x_vals: 129 | best_fit.append(slope*i+y_intercept) 130 | 131 | 132 | ### 133 | # Plot results 134 | ### 135 | 136 | # Plot regression line against data points 137 | plt.plot(x_vals, y_vals, 'o', label='Data Points') 138 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 139 | plt.legend(loc='upper left') 140 | plt.title('Sepal Length vs Pedal Width') 141 | plt.xlabel('Pedal Width') 142 | plt.ylabel('Sepal Length') 143 | plt.show() 144 | 145 | # Plot loss over time 146 | plt.plot(loss_vec, 'k-') 147 | plt.title(regression_type + ' Loss per Generation') 148 | plt.xlabel('Generation') 149 | plt.ylabel('Loss') 150 | plt.show() 151 | 152 | -------------------------------------------------------------------------------- /03_Linear_Regression/06_Implementing_Lasso_and_Ridge_Regression/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Lasso and Ridge Regression 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /03_Linear_Regression/07_Implementing_Elasticnet_Regression/07_elasticnet_regression.py: -------------------------------------------------------------------------------- 1 | # Elastic Net Regression 2 | #---------------------------------- 3 | # 4 | # This function shows how to use TensorFlow to 5 | # solve elastic net regression. 6 | # y = Ax + b 7 | # 8 | # We will use the iris data, specifically: 9 | # y = Sepal Length 10 | # x = Pedal Length, Petal Width, Sepal Width 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from sklearn import datasets 16 | from tensorflow.python.framework import ops 17 | 18 | ### 19 | # Set up for TensorFlow 20 | ### 21 | 22 | ops.reset_default_graph() 23 | 24 | # Create graph 25 | sess = tf.Session() 26 | 27 | ### 28 | # Obtain data 29 | ### 30 | 31 | # Load the data 32 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 33 | iris = datasets.load_iris() 34 | x_vals = np.array([[x[1], x[2], x[3]] for x in iris.data]) 35 | y_vals = np.array([y[0] for y in iris.data]) 36 | 37 | ### 38 | # Setup model 39 | ### 40 | 41 | # make results reproducible 42 | seed = 13 43 | np.random.seed(seed) 44 | tf.set_random_seed(seed) 45 | 46 | # Declare batch size 47 | batch_size = 50 48 | 49 | # Initialize placeholders 50 | x_data = tf.placeholder(shape=[None, 3], dtype=tf.float32) 51 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 52 | 53 | # Create variables for linear regression 54 | A = tf.Variable(tf.random_normal(shape=[3,1])) 55 | b = tf.Variable(tf.random_normal(shape=[1,1])) 56 | 57 | # Declare model operations 58 | model_output = tf.add(tf.matmul(x_data, A), b) 59 | 60 | # Declare the elastic net loss function 61 | elastic_param1 = tf.constant(1.) 62 | elastic_param2 = tf.constant(1.) 63 | l1_a_loss = tf.reduce_mean(tf.abs(A)) 64 | l2_a_loss = tf.reduce_mean(tf.square(A)) 65 | e1_term = tf.multiply(elastic_param1, l1_a_loss) 66 | e2_term = tf.multiply(elastic_param2, l2_a_loss) 67 | loss = tf.expand_dims(tf.add(tf.add(tf.reduce_mean(tf.square(y_target - model_output)), e1_term), e2_term), 0) 68 | 69 | # Declare optimizer 70 | my_opt = tf.train.GradientDescentOptimizer(0.001) 71 | train_step = my_opt.minimize(loss) 72 | 73 | ### 74 | # Train model 75 | ### 76 | 77 | # Initialize variables 78 | init = tf.global_variables_initializer() 79 | sess.run(init) 80 | 81 | # Training loop 82 | loss_vec = [] 83 | for i in range(1000): 84 | rand_index = np.random.choice(len(x_vals), size=batch_size) 85 | rand_x = x_vals[rand_index] 86 | rand_y = np.transpose([y_vals[rand_index]]) 87 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 88 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 89 | loss_vec.append(temp_loss[0]) 90 | if (i+1)%250==0: 91 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 92 | print('Loss = ' + str(temp_loss)) 93 | 94 | ### 95 | # Extract model results 96 | ### 97 | 98 | # Get the optimal coefficients 99 | [[sw_coef], [pl_coef], [pw_ceof]] = sess.run(A) 100 | [y_intercept] = sess.run(b) 101 | 102 | ### 103 | # Plot results 104 | ### 105 | 106 | # Plot loss over time 107 | plt.plot(loss_vec, 'k-') 108 | plt.title('Loss per Generation') 109 | plt.xlabel('Generation') 110 | plt.ylabel('Loss') 111 | plt.show() 112 | 113 | -------------------------------------------------------------------------------- /03_Linear_Regression/07_Implementing_Elasticnet_Regression/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Elasticnet Regression 2 | 3 | ![Elasticnet Regression Loss](../images/07_elasticnet_reg_loss.png "Elasticnet Regression Loss") 4 | -------------------------------------------------------------------------------- /03_Linear_Regression/08_Implementing_Logistic_Regression/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Logistic Regression 2 | 3 | Logistic regression is a way to predict a number between zero or one (usually we consider the output a probability). This prediction is classified into class value ‘1’ if the prediction is above a specified cut off value and class ‘0’ otherwise. The standard cutoff is 0.5. For the purpose of this example, we will specify that cut off to be 0.5, which will make the classification as simple as rounding the output. 4 | 5 | The data we will use for this example will be the [UMASS low birth weight data](https://www.umass.edu/statdata/statdata/data/lowbwt.txt). 6 | 7 | # Model 8 | 9 | The the output of our model is the standard logistic regression: 10 | 11 | y = sigmoid(A * x + b) 12 | 13 | The x matrix input will have dimensions (batch size x # features). The y target output will have the dimension batch size x 1. 14 | 15 | The loss function we will use will be the mean of the cross-entropy loss: 16 | 17 | loss = mean( - y * log(predicted) + (1-y) * log(1-predicted) ) 18 | 19 | TensorFlow has this cross entropy built in, and we can use the function, 'tf.nn.sigmoid\_cross\_entropy\_with\_logits()' 20 | 21 | We will then iterate through random batch size selections of the data. 22 | 23 | # Graph of Loss Function 24 | 25 | Running the script should result in a similar loss and accuracy output. 26 | 27 | ![Logistic Regression Loss](../images/08_logistic_reg_loss.png "Logistic Regression Loss") 28 | 29 | # Accuracy of Train and Test Sets 30 | 31 | ![Logistic Regression Accuracy](../images/08_logistic_reg_acc.png "Logistic Regression Accuracy") 32 | -------------------------------------------------------------------------------- /03_Linear_Regression/images/01_Inverse_Matrix_Method.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/01_Inverse_Matrix_Method.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/02_Cholesky_Decomposition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/02_Cholesky_Decomposition.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/03_lin_reg_fit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/03_lin_reg_fit.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/03_lin_reg_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/03_lin_reg_loss.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/04_L1_L2_learningrates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/04_L1_L2_learningrates.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/04_L1_L2_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/04_L1_L2_loss.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/04_L1_L2_loss2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/04_L1_L2_loss2.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/05_demming_reg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/05_demming_reg.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/05_demming_vs_linear_reg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/05_demming_vs_linear_reg.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/07_elasticnet_reg_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/07_elasticnet_reg_loss.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/08_logistic_reg_acc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/08_logistic_reg_acc.png -------------------------------------------------------------------------------- /03_Linear_Regression/images/08_logistic_reg_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/03_Linear_Regression/images/08_logistic_reg_loss.png -------------------------------------------------------------------------------- /03_Linear_Regression/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 3: Linear Regression 2 | 3 | Here we show how to implement various linear regression techniques in TensorFlow. The first two sections show how to do standard matrix linear regression solving in TensorFlow. The remaining six sections depict how to implement various types of regression using computational graphs in TensorFlow. 4 | 5 | 1. [Using the Matrix Inverse Method](01_Using_the_Matrix_Inverse_Method#using-the-matrix-inverse-method) 6 | * How to solve a 2D regression with a matrix inverse in TensorFlow. 7 | 2. [Implementing a Decomposition Method](02_Implementing_a_Decomposition_Method#using-the-cholesky-decomposition-method) 8 | * Solving a 2D linear regression with Cholesky decomposition. 9 | 3. [Learning the TensorFlow Way of Linear Regression](03_TensorFlow_Way_of_Linear_Regression#learning-the-tensorflow-way-of-regression) 10 | * Linear regression iterating through a computational graph with L2 Loss. 11 | 4. [Understanding Loss Functions in Linear Regression](04_Loss_Functions_in_Linear_Regressions#loss-functions-in-linear-regression) 12 | * L2 vs L1 loss in linear regression. We talk about the benefits and limitations of both. 13 | 5. [Implementing Deming Regression (Total Regression)](05_Implementing_Deming_Regression#implementing-deming-regression) 14 | * Deming (total) regression implemented in TensorFlow by changing the loss function. 15 | 6. [Implementing Lasso and Ridge Regression](06_Implementing_Lasso_and_Ridge_Regression#implementing-lasso-and-ridge-regression) 16 | * Lasso and Ridge regression are ways of regularizing the coefficients. We implement both of these in TensorFlow via changing the loss functions. 17 | 7. [Implementing Elastic Net Regression](07_Implementing_Elasticnet_Regression#implementing-elasticnet-regression) 18 | * Elastic net is a regularization technique that combines the L2 and L1 loss for coefficients. We show how to implement this in TensorFlow. 19 | 8. [Implementing Logistic Regression](08_Implementing_Logistic_Regression#implementing-logistic-regression) 20 | * We implement logistic regression by the use of an activation function in our computational graph. 21 | -------------------------------------------------------------------------------- /04_Support_Vector_Machines/01_Introduction/readme.md: -------------------------------------------------------------------------------- 1 | # Support Vector Machine Introduction 2 | 3 | ![Linear Separator](../images/01_introduction.png "Linear Separator") 4 | 5 | Support Vector Machines (SVMs) are a machine learning method to separate binary classes. This is accomplished by maximizing a margin width between two classes. It is extended to allow for noise in the data, when the two classes are not explicitly linear separable. SVMs can be extended to separate non linear classes by using non-linear kernels. After this, we finish this chapter by showing how to extend these binary classifies into multi-class problems via a one-vs-all method. 6 | -------------------------------------------------------------------------------- /04_Support_Vector_Machines/02_Working_with_Linear_SVMs/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Linear SVMs 2 | 3 | We introduce a linear SVM on a binary set, which will be a subset of the Iris data. We know for I. setosa, that petal width and sepal length are completely separable. We will create a linear SVM to predict I. setosa based on two features: petal width and sepal length. 4 | 5 | It is worth noting that due to the small data set and the randomness of separating into train/test sets, that it may appear that a few points can end up on the wrong side of the line. This is because they are in the test set, and this will result in a lower test accuracy. 6 | 7 | # Model 8 | 9 | We will aim to maximize the margin width, 2/||A||, or minimize ||A||. We allow for a soft margin by having an error term in the loss function which is the max(0, 1-pred*actual). 10 | 11 | ![Linear Separator](../images/01_introduction.png "Linear Separator") 12 | 13 | # Graph of Linear SVM 14 | 15 | Here is a plot of the linear SVM separator of I. setosa based on petal width and sepal length. 16 | 17 | ![Linear SVM Output](../images/02_linear_svm_loss.png "Linear SVM Output") 18 | 19 | The accuracy is below, plotted over each iteration. 20 | 21 | ![Linear SVM Accuracy](../images/02_linear_svm_accuracy.png "Linear SVM Accuracy") 22 | 23 | An important observation is that while we achieve the linear separator rather quickly (100% accuracy), the loss function continues to decrease. This is because we are trying to optimize for the maximal linear separator between the two classes. 24 | 25 | Ideally, we would have enough data to do a cross validation technique, or even to separate the data into train and test sets before optimization. 26 | -------------------------------------------------------------------------------- /04_Support_Vector_Machines/03_Reduction_to_Linear_Regression/03_support_vector_regression.py: -------------------------------------------------------------------------------- 1 | # SVM Regression 2 | #---------------------------------- 3 | # 4 | # This function shows how to use TensorFlow to 5 | # solve support vector regression. We are going 6 | # to find the line that has the maximum margin 7 | # which INCLUDES as many points as possible 8 | # 9 | # We will use the iris data, specifically: 10 | # y = Sepal Length 11 | # x = Pedal Width 12 | 13 | import matplotlib.pyplot as plt 14 | import numpy as np 15 | import tensorflow as tf 16 | from sklearn import datasets 17 | from tensorflow.python.framework import ops 18 | ops.reset_default_graph() 19 | 20 | # Create graph 21 | sess = tf.Session() 22 | 23 | # Load the data 24 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 25 | iris = datasets.load_iris() 26 | x_vals = np.array([x[3] for x in iris.data]) 27 | y_vals = np.array([y[0] for y in iris.data]) 28 | 29 | # Split data into train/test sets 30 | train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 31 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 32 | x_vals_train = x_vals[train_indices] 33 | x_vals_test = x_vals[test_indices] 34 | y_vals_train = y_vals[train_indices] 35 | y_vals_test = y_vals[test_indices] 36 | 37 | # Declare batch size 38 | batch_size = 50 39 | 40 | # Initialize placeholders 41 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 42 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 43 | 44 | # Create variables for linear regression 45 | A = tf.Variable(tf.random_normal(shape=[1,1])) 46 | b = tf.Variable(tf.random_normal(shape=[1,1])) 47 | 48 | # Declare model operations 49 | model_output = tf.add(tf.matmul(x_data, A), b) 50 | 51 | # Declare loss function 52 | # = max(0, abs(target - predicted) + epsilon) 53 | # 1/2 margin width parameter = epsilon 54 | epsilon = tf.constant([0.5]) 55 | # Margin term in loss 56 | loss = tf.reduce_mean(tf.maximum(0., tf.subtract(tf.abs(tf.subtract(model_output, y_target)), epsilon))) 57 | 58 | # Declare optimizer 59 | my_opt = tf.train.GradientDescentOptimizer(0.075) 60 | train_step = my_opt.minimize(loss) 61 | 62 | # Initialize variables 63 | init = tf.global_variables_initializer() 64 | sess.run(init) 65 | 66 | # Training loop 67 | train_loss = [] 68 | test_loss = [] 69 | for i in range(200): 70 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 71 | rand_x = np.transpose([x_vals_train[rand_index]]) 72 | rand_y = np.transpose([y_vals_train[rand_index]]) 73 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 74 | 75 | temp_train_loss = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_train]), y_target: np.transpose([y_vals_train])}) 76 | train_loss.append(temp_train_loss) 77 | 78 | temp_test_loss = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_test]), y_target: np.transpose([y_vals_test])}) 79 | test_loss.append(temp_test_loss) 80 | if (i+1)%50==0: 81 | print('-----------') 82 | print('Generation: ' + str(i+1)) 83 | print('A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 84 | print('Train Loss = ' + str(temp_train_loss)) 85 | print('Test Loss = ' + str(temp_test_loss)) 86 | 87 | # Extract Coefficients 88 | [[slope]] = sess.run(A) 89 | [[y_intercept]] = sess.run(b) 90 | width = sess.run(epsilon) 91 | 92 | # Get best fit line 93 | best_fit = [] 94 | best_fit_upper = [] 95 | best_fit_lower = [] 96 | for i in x_vals: 97 | best_fit.append(slope*i+y_intercept) 98 | best_fit_upper.append(slope*i+y_intercept+width) 99 | best_fit_lower.append(slope*i+y_intercept-width) 100 | 101 | # Plot fit with data 102 | plt.plot(x_vals, y_vals, 'o', label='Data Points') 103 | plt.plot(x_vals, best_fit, 'r-', label='SVM Regression Line', linewidth=3) 104 | plt.plot(x_vals, best_fit_upper, 'r--', linewidth=2) 105 | plt.plot(x_vals, best_fit_lower, 'r--', linewidth=2) 106 | plt.ylim([0, 10]) 107 | plt.legend(loc='lower right') 108 | plt.title('Sepal Length vs Petal Width') 109 | plt.xlabel('Petal Width') 110 | plt.ylabel('Sepal Length') 111 | plt.show() 112 | 113 | # Plot loss over time 114 | plt.plot(train_loss, 'k-', label='Train Set Loss') 115 | plt.plot(test_loss, 'r--', label='Test Set Loss') 116 | plt.title('L2 Loss per Generation') 117 | plt.xlabel('Generation') 118 | plt.ylabel('L2 Loss') 119 | plt.legend(loc='upper right') 120 | plt.show() 121 | -------------------------------------------------------------------------------- /04_Support_Vector_Machines/03_Reduction_to_Linear_Regression/readme.md: -------------------------------------------------------------------------------- 1 | # SVM Reduction to Linear Regression 2 | 3 | Instead of optimizing the maximal linear separator, we change the loss function to maximize the amount of data points we can make fit in our margin. This will give us a linear regression estimation. 4 | 5 | ![Linear SVM Reg Loss](../images/03_linear_svm_loss.png "Linear SVM Loss") 6 | 7 | ![Linear SVM Reg Output](../images/03_svm_regression_output.png "Linear SVM Fit") 8 | -------------------------------------------------------------------------------- /04_Support_Vector_Machines/04_Working_with_Kernels/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Kernels 2 | 3 | Linear SVMs are very powerful. But sometimes the data are not very linear. To this end, we can use the 'kernel trick' to map our data into a higher dimensional space, where it may be linearly separable. Doing this allows us to separate out non-linear classes. See the below example. 4 | 5 | If we attempt to separate the below circular-ring shaped classes with a standard linear SVM, we fail. 6 | 7 | ![Linear SVM Nonlinear Data](../images/04_nonlinear_data_linear_kernel.png "Linear SVM Fit") 8 | 9 | But if we separate it with a Gaussian-RBF kernel, we can find a linear separator in a higher dimension that works a lot better. 10 | 11 | ![Gaussian Kernel Nonlinear Data](../images/04_linear_svm_gaussian.png "Gaussian Kernel") 12 | -------------------------------------------------------------------------------- /04_Support_Vector_Machines/05_Implementing_Nonlinear_SVMs/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing NonLinear SVMs 2 | 3 | Here we show how to use the prior Gaussian RBF kernel to predict I.setosa from the Iris dataset. 4 | 5 | ![Nonlinear SVM](../images/05_non_linear_svms.png "Nonlinear SVM") 6 | -------------------------------------------------------------------------------- /04_Support_Vector_Machines/06_Implementing_Multiclass_SVMs/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Multiclass SVMs 2 | 3 | Here, we implement a 1-vs-all voting method for a multiclass SVM. We attempt to separate the three Iris flower classes with TensorFlow. 4 | 5 | ![Multiclass SVM](../images/06_multiclass_svm.png "Multiclass SVM") 6 | -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/01_introduction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/01_introduction.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/02_linear_svm_accuracy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/02_linear_svm_accuracy.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/02_linear_svm_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/02_linear_svm_loss.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/02_linear_svm_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/02_linear_svm_output.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/03_linear_svm_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/03_linear_svm_loss.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/03_svm_regression_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/03_svm_regression_output.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/04_linear_svm_gaussian.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/04_linear_svm_gaussian.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/04_nonlinear_data_linear_kernel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/04_nonlinear_data_linear_kernel.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/05_non_linear_svms.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/05_non_linear_svms.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/images/06_multiclass_svm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/04_Support_Vector_Machines/images/06_multiclass_svm.png -------------------------------------------------------------------------------- /04_Support_Vector_Machines/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 4: Support Vector Machines 2 | 3 | This chapter shows how to implement various SVM methods with TensorFlow. We first create a linear SVM and also show how it can be used for regression. We then introduce kernels (RBF Gaussian kernel) and show how to use it to split up non-linear data. We finish with a multi-dimensional implementation of non-linear SVMs to work with multiple classes. 4 | 5 | 1. [Introduction](01_Introduction#support-vector-machine-introduction) 6 | * We introduce the concept of SVMs and how we will go about implementing them in the TensorFlow framework. 7 | 2. [Working with Linear SVMs](02_Working_with_Linear_SVMs#working-with-linear-svms) 8 | * We create a linear SVM to separate I. setosa based on sepal length and pedal width in the Iris data set. 9 | 3. [Reduction to Linear Regression](03_Reduction_to_Linear_Regression#svm-reduction-to-linear-regression) 10 | * The heart of SVMs is separating classes with a line. We change tweek the algorithm slightly to perform SVM regression. 11 | 4. [Working with Kernels in TensorFlow](04_Working_with_Kernels#working-with-kernels) 12 | * In order to extend SVMs into non-linear data, we explain and show how to implement different kernels in TensorFlow. 13 | 5. [Implementing Non-Linear SVMs](05_Implementing_Nonlinear_SVMs#implementing-nonlinear-svms) 14 | * We use the Gaussian kernel (RBF) to separate non-linear classes. 15 | 6. [Implementing Multi-class SVMs](06_Implementing_Multiclass_SVMs#implementing-multiclass-svms) 16 | * SVMs are inherently binary predictors. We show how to extend them in a one-vs-all strategy in TensorFlow. 17 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/01_Introduction/readme.md: -------------------------------------------------------------------------------- 1 | # Nearest Neighbor Methods Introduction 2 | 3 | This chapter will focus on nearest neighbor methods and how to implement them in TensorFlow. We will start with an introduction to the method and show how to implement various forms, and the chapter will end with examples of address matching and image recognition. This is what we will cover: 4 | 5 | - Working with Nearest Neighbors 6 | 7 | - Working with Text-Based Distances 8 | 9 | - Computing Mixed Distance Functions 10 | 11 | - Using an Address Matching Example 12 | 13 | - Using Nearest Neighbors for Image Recognition 14 | 15 | ![Nearest Neighbor](../images/nearest_neighbor_intro.jpg "Nearest Neighbor") 16 | 17 | Nearest neighbor methods are based on a distance-based conceptual idea. We consider our training set as the model and make predictions on new points based on how close they are to points in the training set. A naïve way is to make the prediction as the closest training data point class. But since most datasets contain a degree of noise, a more common method would be to take a weighted average of a set of k nearest neighbors. This method is called k-nearest neighbors (k-NN). 18 | 19 | Given a training dataset (x1, x2, ..., xn), with corresponding targets (y1, y2, ..., yn), we can make a prediction on a point, z, by looking at a set of nearest neighbors. The actual method of prediction depends on whether or not we are doing regression (continuous y) or classification (discrete y). 20 | 21 | For discrete classification targets, the prediction may be given by a maximum voting scheme weighted by the distance to the prediction point: 22 | 23 | prediction(z) = max ( weighted sum of distances of points in each class ) 24 | 25 | - see jupyter notebook for the formula 26 | 27 | Here, our prediction is the maximum weighted value over all classes (j), where the weighted distance from the prediction point is usually given by the L1 or L2 distance functions. 28 | 29 | Continuous targets are very similar, but we usually just compute a weighted average of the target variable (y) by distance. 30 | 31 | There are many different specifications of distance metrics that we can choose. In this chapter, we will explore the L1 and L2 metrics as well as edit and textual distances. 32 | 33 | We also have to choose how to weight the distances. A straight forward way to weight the distances is by the distance itself. Points that are further away from our prediction should have less impact than nearer points. The most common way to weight is by the normalized inverse of the distance. We will implement this method in the next recipe. 34 | 35 | Note that k-NN is an aggregating method. For regression, we are performing a weighted average of neighbors. Because of this, predictions will be less extreme and less varied than the actual targets. The magnitude of this effect will be determined by k, the number of neighbors in the algorithm. 36 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/02_Working_with_Nearest_Neighbors/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Nearest Neighbors 2 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/03_Working_with_Text_Distances/03_text_distances.py: -------------------------------------------------------------------------------- 1 | # Text Distances 2 | #---------------------------------- 3 | # 4 | # This function illustrates how to use 5 | # the Levenstein distance (edit distance) 6 | # in TensorFlow. 7 | 8 | import tensorflow as tf 9 | 10 | sess = tf.Session() 11 | 12 | #---------------------------------- 13 | # First compute the edit distance between 'bear' and 'beers' 14 | hypothesis = list('bear') 15 | truth = list('beers') 16 | h1 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3]], 17 | hypothesis, 18 | [1,1,1]) 19 | 20 | t1 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,1], [0,0,3],[0,0,4]], 21 | truth, 22 | [1,1,1]) 23 | 24 | print(sess.run(tf.edit_distance(h1, t1, normalize=False))) 25 | 26 | #---------------------------------- 27 | # Compute the edit distance between ('bear','beer') and 'beers': 28 | hypothesis2 = list('bearbeer') 29 | truth2 = list('beersbeers') 30 | h2 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3], [0,1,0], [0,1,1], [0,1,2], [0,1,3]], 31 | hypothesis2, 32 | [1,2,4]) 33 | 34 | t2 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3], [0,0,4], [0,1,0], [0,1,1], [0,1,2], [0,1,3], [0,1,4]], 35 | truth2, 36 | [1,2,5]) 37 | 38 | print(sess.run(tf.edit_distance(h2, t2, normalize=True))) 39 | 40 | #---------------------------------- 41 | # Now compute distance between four words and 'beers' more efficiently with sparse tensors: 42 | hypothesis_words = ['bear','bar','tensor','flow'] 43 | truth_word = ['beers'] 44 | 45 | num_h_words = len(hypothesis_words) 46 | h_indices = [[xi, 0, yi] for xi,x in enumerate(hypothesis_words) for yi,y in enumerate(x)] 47 | h_chars = list(''.join(hypothesis_words)) 48 | 49 | h3 = tf.SparseTensor(h_indices, h_chars, [num_h_words,1,1]) 50 | 51 | truth_word_vec = truth_word*num_h_words 52 | t_indices = [[xi, 0, yi] for xi,x in enumerate(truth_word_vec) for yi,y in enumerate(x)] 53 | t_chars = list(''.join(truth_word_vec)) 54 | 55 | t3 = tf.SparseTensor(t_indices, t_chars, [num_h_words,1,1]) 56 | 57 | print(sess.run(tf.edit_distance(h3, t3, normalize=True))) 58 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/03_Working_with_Text_Distances/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Text Distances 2 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/04_Computing_with_Mixed_Distance_Functions/readme.md: -------------------------------------------------------------------------------- 1 | # Computing with Mixed Distance Functions 2 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/05_An_Address_Matching_Example/readme.md: -------------------------------------------------------------------------------- 1 | # An Address Matching Example 2 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/06_Nearest_Neighbors_for_Image_Recognition/06_image_recognition.py: -------------------------------------------------------------------------------- 1 | # MNIST Digit Prediction with k-Nearest Neighbors 2 | #----------------------------------------------- 3 | # 4 | # This script will load the MNIST data, and split 5 | # it into test/train and perform prediction with 6 | # nearest neighbors 7 | # 8 | # For each test integer, we will return the 9 | # closest image/integer. 10 | # 11 | # Integer images are represented as 28x8 matrices 12 | # of floating point numbers 13 | 14 | import random 15 | import numpy as np 16 | import tensorflow as tf 17 | import matplotlib.pyplot as plt 18 | from PIL import Image 19 | from tensorflow.examples.tutorials.mnist import input_data 20 | from tensorflow.python.framework import ops 21 | ops.reset_default_graph() 22 | 23 | # Create graph 24 | sess = tf.Session() 25 | 26 | # Load the data 27 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 28 | 29 | # Random sample 30 | np.random.seed(13) # set seed for reproducibility 31 | train_size = 1000 32 | test_size = 102 33 | rand_train_indices = np.random.choice(len(mnist.train.images), train_size, replace=False) 34 | rand_test_indices = np.random.choice(len(mnist.test.images), test_size, replace=False) 35 | x_vals_train = mnist.train.images[rand_train_indices] 36 | x_vals_test = mnist.test.images[rand_test_indices] 37 | y_vals_train = mnist.train.labels[rand_train_indices] 38 | y_vals_test = mnist.test.labels[rand_test_indices] 39 | 40 | # Declare k-value and batch size 41 | k = 4 42 | batch_size=6 43 | 44 | # Placeholders 45 | x_data_train = tf.placeholder(shape=[None, 784], dtype=tf.float32) 46 | x_data_test = tf.placeholder(shape=[None, 784], dtype=tf.float32) 47 | y_target_train = tf.placeholder(shape=[None, 10], dtype=tf.float32) 48 | y_target_test = tf.placeholder(shape=[None, 10], dtype=tf.float32) 49 | 50 | # Declare distance metric 51 | # L1 52 | distance = tf.reduce_sum(tf.abs(tf.subtract(x_data_train, tf.expand_dims(x_data_test,1))), axis=2) 53 | 54 | # L2 55 | #distance = tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(x_data_train, tf.expand_dims(x_data_test,1))), reduction_indices=1)) 56 | 57 | # Predict: Get min distance index (Nearest neighbor) 58 | top_k_xvals, top_k_indices = tf.nn.top_k(tf.negative(distance), k=k) 59 | prediction_indices = tf.gather(y_target_train, top_k_indices) 60 | # Predict the mode category 61 | count_of_predictions = tf.reduce_sum(prediction_indices, axis=1) 62 | prediction = tf.argmax(count_of_predictions) 63 | 64 | # Calculate how many loops over training data 65 | num_loops = int(np.ceil(len(x_vals_test)/batch_size)) 66 | 67 | test_output = [] 68 | actual_vals = [] 69 | for i in range(num_loops): 70 | min_index = i*batch_size 71 | max_index = min((i+1)*batch_size,len(x_vals_train)) 72 | x_batch = x_vals_test[min_index:max_index] 73 | y_batch = y_vals_test[min_index:max_index] 74 | predictions = sess.run(prediction, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, 75 | y_target_train: y_vals_train, y_target_test: y_batch}) 76 | test_output.extend(predictions) 77 | actual_vals.extend(np.argmax(y_batch, axis=1)) 78 | 79 | accuracy = sum([1./test_size for i in range(test_size) if test_output[i]==actual_vals[i]]) 80 | print('Accuracy on test set: ' + str(accuracy)) 81 | 82 | # Plot the last batch results: 83 | actuals = np.argmax(y_batch, axis=1) 84 | 85 | Nrows = 2 86 | Ncols = 3 87 | for i in range(len(actuals)): 88 | plt.subplot(Nrows, Ncols, i+1) 89 | plt.imshow(np.reshape(x_batch[i], [28,28]), cmap='Greys_r') 90 | plt.title('Actual: ' + str(actuals[i]) + ' Pred: ' + str(predictions[i]), 91 | fontsize=10) 92 | frame = plt.gca() 93 | frame.axes.get_xaxis().set_visible(False) 94 | frame.axes.get_yaxis().set_visible(False) 95 | 96 | plt.show() 97 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/06_Nearest_Neighbors_for_Image_Recognition/readme.md: -------------------------------------------------------------------------------- 1 | # Nearest Neighbors for Image Recognition 2 | -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/images/02_mse_vs_variance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/05_Nearest_Neighbor_Methods/images/02_mse_vs_variance.png -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/images/02_nn_histogram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/05_Nearest_Neighbor_Methods/images/02_nn_histogram.png -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/images/04_pred_vs_actual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/05_Nearest_Neighbor_Methods/images/04_pred_vs_actual.png -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/images/06_nn_image_recognition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/05_Nearest_Neighbor_Methods/images/06_nn_image_recognition.png -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/images/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/05_Nearest_Neighbor_Methods/images/image.png -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/images/nearest_neighbor_intro.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/05_Nearest_Neighbor_Methods/images/nearest_neighbor_intro.jpg -------------------------------------------------------------------------------- /05_Nearest_Neighbor_Methods/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 5: Nearest Neighbor Methods 2 | 3 | Nearest Neighbor methods are a very popular ML algorithm. We show how to implement k-Nearest Neighbors, weighted k-Nearest Neighbors, and k-Nearest Neighbors with mixed distance functions. In this chapter we also show how to use the Levenshtein distance (edit distance) in TensorFlow, and use it to calculate the distance between strings. We end this chapter with showing how to use k-Nearest Neighbors for categorical prediction with the MNIST handwritten digit recognition. 4 | 5 | 1. [Introduction](01_Introduction#nearest-neighbor-methods-introduction) 6 | * We introduce the concepts and methods needed for performing k-Nearest Neighbors in TensorFlow. 7 | 2. [Working with Nearest Neighbors](02_Working_with_Nearest_Neighbors#working-with-nearest-neighbors) 8 | * We create a nearest neighbor algorithm that tries to predict housing worth (regression). 9 | 3. [Working with Text Based Distances](03_Working_with_Text_Distances#working-with-text-distances) 10 | * In order to use a distance function on text, we show how to use edit distances in TensorFlow. 11 | 4. [Computing Mixing Distance Functions](04_Computing_with_Mixed_Distance_Functions#computing-with-mixed-distance-functions) 12 | * Here we implement scaling of the distance function by the standard deviation of the input feature for k-Nearest Neighbors. 13 | 5. [Using Address Matching](05_An_Address_Matching_Example#an-address-matching-example) 14 | * We use a mixed distance function to match addresses. We use numerical distance for zip codes, and string edit distance for street names. The street names are allowed to have typos. 15 | 6. [Using Nearest Neighbors for Image Recognition](06_Nearest_Neighbors_for_Image_Recognition#nearest-neighbors-for-image-recognition) 16 | * The MNIST digit image collection is a great data set for illustration of how to perform k-Nearest Neighbors for an image classification task. 17 | -------------------------------------------------------------------------------- /06_Neural_Networks/01_Introduction/readme.md: -------------------------------------------------------------------------------- 1 | # Neural Networks Introduction 2 | 3 | ------------------ 4 | 5 | In this chapter, we will introduce neural networks and how to implement them in TensorFlow. Most of the subsequent chapters will be based on neural networks, so learning how to use them in TensorFlow is very important. We will start by introducing basic concepts of neural networking and work up to multilayer networks. In the last section we will create a neural network that learns to play Tic Tac Toe. 6 | 7 | In this chapter, we'll cover the following recipes: 8 | 9 | - Implementing Operational Gates 10 | - Working with Gates and Activation Functions 11 | - Implementing a One Layer Neural Network 12 | - Implementing Different Layers 13 | - Using Multilayer Networks 14 | - Improving Predictions of Linear Models 15 | - Learning to Play Tic Tac Toe 16 | 17 | ---------------- 18 | 19 | Neural networks are currently breaking records in tasks such as image and speech recognition, reading handwriting, understanding text, image segmentation, dialogue systems, autonomous car driving, and so much more. While some of these aforementioned tasks will be covered in later chapters, it is important to introduce neural networks as an easy-to-implement machine learning algorithm, so that we can expand on it later. 20 | 21 | The concept of a neural network has been around for decades. However, it only recently gained traction because we now have the computational power to train large networks because of advances in processing power, algorithm efficiency, and data sizes. 22 | 23 | A neural network is basically a sequence of operations applied to a matrix of input data. These operations are usually collections of additions and multiplications followed by applications of non-linear functions. One example that we have already seen is logistic regression, the last section in Chapter 3, Linear Regression. Logistic regression is the sum of the partial slope-feature products followed by the application of the sigmoid function, which is non-linear. Neural networks generalize this a bit more by allowing any combination of operations and non-linear functions, which includes the applications of absolute value, maximum, minimum, and so on. 24 | 25 | The important trick with neural networks is called 'back propagation'. Back propagation is a procedure that allows us to update the model variables based on the learning rate and the output of the loss function. We used back propagation to update our model variables in the Chapter 3, Linear Regression and Chapter 4, and the Support Vector Machine chapter. 26 | 27 | Another important feature to take note of in neural networks is the non-linear activation function. Since most neural networks are just combinations of addition and multiplication operations, they will not be able to model non-linear data sets. To address this issue, we have used the non-linear activation functions in the neural networks. This will allow the neural network to adapt to most non-linear situations. 28 | 29 | It is important to remember that, like most of the algorithms we have seen so far, neural networks are sensitive to the hyper-parameters that we choose. In this chapter, we will see the impact of different learning rates, loss functions, and optimization procedures. 30 | 31 | There are more resources for learning about neural networks that are more in depth and detailed. Here are some following resources: 32 | 33 | - The seminal paper describing back propagation is Efficient Back Prop by Yann LeCun et. al. The PDF is located here: http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf 34 | 35 | - CS231, Convolutional Neural Networks for Visual Recognition, by Stanford University, class resources available here: http://cs231n.stanford.edu/ 36 | 37 | - CS224d, Deep Learning for Natural Language Processing, by Stanford University, class resources available here: http://cs224d.stanford.edu/ 38 | 39 | - Deep Learning, a book by the MIT Press. Goodfellow, et. al. 2016. Located: http://www.deeplearningbook.org 40 | 41 | - There is an online book called Neural Networks and Deep Learning by Michael Nielsen, located here: http://neuralnetworksanddeeplearning.com/ 42 | 43 | - For a more pragmatic approach and introduction to neural networks, Andrej Karpathy has written a great summary and JavaScript examples called A Hacker's Guide to Neural Networks. The write up is located here: http://karpathy.github.io/neuralnets/ 44 | 45 | - Another site that summarizes some good notes on deep learning is called Deep Learning for Beginners by Ian Goodfellow, Yoshua Bengio, and Aaron Courville. This web page can be found here: http://randomekek.github.io/deep/deeplearning.html 46 | -------------------------------------------------------------------------------- /06_Neural_Networks/02_Implementing_an_Operational_Gate/02_gates.py: -------------------------------------------------------------------------------- 1 | # Implementing Gates 2 | #---------------------------------- 3 | # 4 | # This function shows how to implement 5 | # various gates in TensorFlow 6 | # 7 | # One gate will be one operation with 8 | # a variable and a placeholder. 9 | # We will ask TensorFlow to change the 10 | # variable based on our loss function 11 | 12 | import tensorflow as tf 13 | from tensorflow.python.framework import ops 14 | ops.reset_default_graph() 15 | 16 | # Start Graph Session 17 | sess = tf.Session() 18 | 19 | #---------------------------------- 20 | # Create a multiplication gate: 21 | # f(x) = a * x 22 | # 23 | # a -- 24 | # | 25 | # |---- (multiply) --> output 26 | # x --| 27 | # 28 | 29 | a = tf.Variable(tf.constant(4.)) 30 | x_val = 5. 31 | x_data = tf.placeholder(dtype=tf.float32) 32 | 33 | multiplication = tf.multiply(a, x_data) 34 | 35 | # Declare the loss function as the difference between 36 | # the output and a target value, 50. 37 | loss = tf.square(tf.subtract(multiplication, 50.)) 38 | 39 | # Initialize variables 40 | init = tf.global_variables_initializer() 41 | sess.run(init) 42 | 43 | # Declare optimizer 44 | my_opt = tf.train.GradientDescentOptimizer(0.01) 45 | train_step = my_opt.minimize(loss) 46 | 47 | # Run loop across gate 48 | print('Optimizing a Multiplication Gate Output to 50.') 49 | for _ in range(10): 50 | sess.run(train_step, feed_dict={x_data: x_val}) 51 | a_val = sess.run(a) 52 | mult_output = sess.run(multiplication, feed_dict={x_data: x_val}) 53 | print(str(a_val) + ' * ' + str(x_val) + ' = ' + str(mult_output)) 54 | 55 | ''' 56 | Create a nested gate: 57 | f(x) = a * x + b 58 | 59 | a -- 60 | | 61 | |-- (multiply)-- 62 | x --| | 63 | |-- (add) --> output 64 | b --| 65 | 66 | ''' 67 | 68 | # Start a New Graph Session 69 | ops.reset_default_graph() 70 | sess = tf.Session() 71 | 72 | a = tf.Variable(tf.constant(1.)) 73 | b = tf.Variable(tf.constant(1.)) 74 | x_val = 5. 75 | x_data = tf.placeholder(dtype=tf.float32) 76 | 77 | two_gate = tf.add(tf.multiply(a, x_data), b) 78 | 79 | # Declare the loss function as the difference between 80 | # the output and a target value, 50. 81 | loss = tf.square(tf.subtract(two_gate, 50.)) 82 | 83 | # Initialize variables 84 | init = tf.global_variables_initializer() 85 | sess.run(init) 86 | 87 | # Declare optimizer 88 | my_opt = tf.train.GradientDescentOptimizer(0.01) 89 | train_step = my_opt.minimize(loss) 90 | 91 | # Run loop across gate 92 | print('\nOptimizing Two Gate Output to 50.') 93 | for _ in range(10): 94 | sess.run(train_step, feed_dict={x_data: x_val}) 95 | a_val, b_val = (sess.run(a), sess.run(b)) 96 | two_gate_output = sess.run(two_gate, feed_dict={x_data: x_val}) 97 | print(str(a_val) + ' * ' + str(x_val) + ' + ' + str(b_val) + ' = ' + str(two_gate_output)) -------------------------------------------------------------------------------- /06_Neural_Networks/02_Implementing_an_Operational_Gate/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing an Operational Gate 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /06_Neural_Networks/03_Working_with_Activation_Functions/03_activation_functions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Combining Gates and Activation Functions 3 | 4 | This function shows how to implement 5 | various gates with activation functions 6 | in TensorFlow 7 | 8 | This function is an extension of the 9 | prior gates, but with various activation 10 | functions. 11 | """ 12 | 13 | import tensorflow as tf 14 | import numpy as np 15 | import matplotlib.pyplot as plt 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Start Graph Session 20 | sess = tf.Session() 21 | tf.set_random_seed(5) 22 | np.random.seed(42) 23 | 24 | batch_size = 50 25 | 26 | a1 = tf.Variable(tf.random_normal(shape=[1, 1])) 27 | b1 = tf.Variable(tf.random_uniform(shape=[1, 1])) 28 | a2 = tf.Variable(tf.random_normal(shape=[1, 1])) 29 | b2 = tf.Variable(tf.random_uniform(shape=[1, 1])) 30 | x = np.random.normal(2, 0.1, 500) 31 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 32 | 33 | sigmoid_activation = tf.sigmoid(tf.add(tf.matmul(x_data, a1), b1)) 34 | 35 | relu_activation = tf.nn.relu(tf.add(tf.matmul(x_data, a2), b2)) 36 | 37 | # Declare the loss function as the difference between 38 | # the output and a target value, 0.75. 39 | loss1 = tf.reduce_mean(tf.square(tf.subtract(sigmoid_activation, 0.75))) 40 | loss2 = tf.reduce_mean(tf.square(tf.subtract(relu_activation, 0.75))) 41 | 42 | # Initialize variables 43 | init = tf.global_variables_initializer() 44 | sess.run(init) 45 | 46 | # Declare optimizer 47 | my_opt = tf.train.GradientDescentOptimizer(0.01) 48 | train_step_sigmoid = my_opt.minimize(loss1) 49 | train_step_relu = my_opt.minimize(loss2) 50 | 51 | # Run loop across gate 52 | print('\nOptimizing Sigmoid AND Relu Output to 0.75') 53 | loss_vec_sigmoid = [] 54 | loss_vec_relu = [] 55 | for i in range(500): 56 | rand_indices = np.random.choice(len(x), size=batch_size) 57 | x_vals = np.transpose([x[rand_indices]]) 58 | sess.run(train_step_sigmoid, feed_dict={x_data: x_vals}) 59 | sess.run(train_step_relu, feed_dict={x_data: x_vals}) 60 | 61 | loss_vec_sigmoid.append(sess.run(loss1, feed_dict={x_data: x_vals})) 62 | loss_vec_relu.append(sess.run(loss2, feed_dict={x_data: x_vals})) 63 | 64 | sigmoid_output = np.mean(sess.run(sigmoid_activation, feed_dict={x_data: x_vals})) 65 | relu_output = np.mean(sess.run(relu_activation, feed_dict={x_data: x_vals})) 66 | 67 | if i % 50 == 0: 68 | print('sigmoid = ' + str(np.mean(sigmoid_output)) + ' relu = ' + str(np.mean(relu_output))) 69 | 70 | # Plot the loss 71 | plt.plot(loss_vec_sigmoid, 'k-', label='Sigmoid Activation') 72 | plt.plot(loss_vec_relu, 'r--', label='Relu Activation') 73 | plt.ylim([0, 1.0]) 74 | plt.title('Loss per Generation') 75 | plt.xlabel('Generation') 76 | plt.ylabel('Loss') 77 | plt.legend(loc='upper right') 78 | plt.show() 79 | -------------------------------------------------------------------------------- /06_Neural_Networks/03_Working_with_Activation_Functions/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Activation Functions 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /06_Neural_Networks/04_Single_Hidden_Layer_Network/04_single_hidden_layer_network.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implementing a one-layer Neural Network 3 | 4 | We will illustrate how to create a one hidden layer NN 5 | 6 | We will use the iris data for this exercise 7 | 8 | We will build a one-hidden layer neural network 9 | to predict the fourth attribute, Petal Width from 10 | the other three (Sepal length, Sepal width, Petal length). 11 | """ 12 | 13 | import matplotlib.pyplot as plt 14 | import numpy as np 15 | import tensorflow as tf 16 | from sklearn import datasets 17 | from tensorflow.python.framework import ops 18 | ops.reset_default_graph() 19 | 20 | iris = datasets.load_iris() 21 | x_vals = np.array([x[0:3] for x in iris.data]) 22 | y_vals = np.array([x[3] for x in iris.data]) 23 | 24 | # Create graph session 25 | sess = tf.Session() 26 | 27 | # make results reproducible 28 | seed = 2 29 | tf.set_random_seed(seed) 30 | np.random.seed(seed) 31 | 32 | # Split data into train/test = 80%/20% 33 | train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 34 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 35 | x_vals_train = x_vals[train_indices] 36 | x_vals_test = x_vals[test_indices] 37 | y_vals_train = y_vals[train_indices] 38 | y_vals_test = y_vals[test_indices] 39 | 40 | 41 | # Normalize by column (min-max norm) 42 | def normalize_cols(m): 43 | col_max = m.max(axis=0) 44 | col_min = m.min(axis=0) 45 | return (m-col_min) / (col_max - col_min) 46 | 47 | x_vals_train = np.nan_to_num(normalize_cols(x_vals_train)) 48 | x_vals_test = np.nan_to_num(normalize_cols(x_vals_test)) 49 | 50 | # Declare batch size 51 | batch_size = 50 52 | 53 | # Initialize placeholders 54 | x_data = tf.placeholder(shape=[None, 3], dtype=tf.float32) 55 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 56 | 57 | # Create variables for both NN layers 58 | hidden_layer_nodes = 10 59 | A1 = tf.Variable(tf.random_normal(shape=[3, hidden_layer_nodes])) # inputs -> hidden nodes 60 | b1 = tf.Variable(tf.random_normal(shape=[hidden_layer_nodes])) # one biases for each hidden node 61 | A2 = tf.Variable(tf.random_normal(shape=[hidden_layer_nodes, 1])) # hidden inputs -> 1 output 62 | b2 = tf.Variable(tf.random_normal(shape=[1])) # 1 bias for the output 63 | 64 | # Declare model operations 65 | hidden_output = tf.nn.relu(tf.add(tf.matmul(x_data, A1), b1)) 66 | final_output = tf.nn.relu(tf.add(tf.matmul(hidden_output, A2), b2)) 67 | 68 | # Declare loss function (MSE) 69 | loss = tf.reduce_mean(tf.square(y_target - final_output)) 70 | 71 | # Declare optimizer 72 | my_opt = tf.train.GradientDescentOptimizer(0.005) 73 | train_step = my_opt.minimize(loss) 74 | 75 | # Initialize variables 76 | init = tf.global_variables_initializer() 77 | sess.run(init) 78 | 79 | # Training loop 80 | loss_vec = [] 81 | test_loss = [] 82 | for i in range(500): 83 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 84 | rand_x = x_vals_train[rand_index] 85 | rand_y = np.transpose([y_vals_train[rand_index]]) 86 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 87 | 88 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 89 | loss_vec.append(np.sqrt(temp_loss)) 90 | 91 | test_temp_loss = sess.run(loss, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])}) 92 | test_loss.append(np.sqrt(test_temp_loss)) 93 | if (i + 1) % 50 == 0: 94 | print('Generation: ' + str(i+1) + '. Loss = ' + str(temp_loss)) 95 | 96 | # Plot loss (MSE) over time 97 | plt.plot(loss_vec, 'k-', label='Train Loss') 98 | plt.plot(test_loss, 'r--', label='Test Loss') 99 | plt.title('Loss (MSE) per Generation') 100 | plt.legend(loc='upper right') 101 | plt.xlabel('Generation') 102 | plt.ylabel('Loss') 103 | plt.show() 104 | -------------------------------------------------------------------------------- /06_Neural_Networks/04_Single_Hidden_Layer_Network/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing a One Layer Neural Network 2 | 3 | We will use the [Iris data](http://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html) for this exercise. We will build a one-hidden layer fully connected neural network to predict one of the flower attributes from the other three. 4 | 5 | The four flower attributes are (1) sepal length, (2) sepal width, (3) pedal length, and (4) pedal width. We will use (1-3) to predict (4). The main purpose of this section is to illustrate how neural networks can implement regression just as easily as classification. Later on in this chapter, we will extend this model to have multiple hidden layers. 6 | 7 | # Model 8 | 9 | The model will have one hidden layer. If the hidden layer has 10 nodes, then the model will look like the following: 10 | 11 | ![One Hidden Layer Network](../images/04_nn_layout.png "One Hidden Layer Network") 12 | 13 | We will use the ReLU activation functions. 14 | 15 | For the loss function, we will use the average MSE across the batch. 16 | 17 | # Graph of Loss Function (Average Batch MSE) 18 | 19 | Running the script should result in a similar loss. 20 | 21 | ![Batch MSE](../images/04_nn_loss.png "Batch MSE") 22 | -------------------------------------------------------------------------------- /06_Neural_Networks/05_Implementing_Different_Layers/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Different Layers 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /06_Neural_Networks/06_Using_Multiple_Layers/readme.md: -------------------------------------------------------------------------------- 1 | # Using Multiple Layers 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /06_Neural_Networks/07_Improving_Linear_Regression/readme.md: -------------------------------------------------------------------------------- 1 | # Improving Linear Regression 2 | 3 | Placeholder for future purposes. 4 | -------------------------------------------------------------------------------- /06_Neural_Networks/08_Learning_Tic_Tac_Toe/base_tic_tac_toe_moves.csv: -------------------------------------------------------------------------------- 1 | 0,0,0,0,-1,0,0,0,0,0 2 | 0,-1,0,0,0,0,0,0,0,0 3 | 0,0,0,0,0,-1,0,0,0,6 4 | -1,0,0,0,0,0,0,0,0,4 5 | 0,0,0,0,0,0,1,-1,-1,3 6 | 0,-1,0,0,1,0,0,0,-1,0 7 | 0,-1,1,0,0,-1,0,0,0,7 8 | -1,0,0,0,-1,0,0,0,1,6 9 | 0,0,1,0,0,-1,-1,0,0,4 10 | 0,0,-1,0,0,0,0,-1,1,4 11 | 1,0,0,-1,0,0,0,-1,0,2 12 | 0,0,-1,0,1,0,-1,0,0,5 13 | -1,0,0,1,-1,-1,0,0,1,6 14 | -1,1,-1,0,1,0,0,1,0,8 15 | 0,0,0,-1,0,1,1,-1,-1,1 16 | -1,1,0,0,0,-1,0,-1,1,3 17 | 0,-1,1,0,1,-1,-1,0,0,8 18 | 0,0,-1,1,0,-1,0,-1,1,0 19 | 1,-1,0,0,-1,0,0,0,0,7 20 | 1,0,-1,0,-1,0,0,0,0,6 21 | 1,0,0,0,-1,0,-1,0,0,2 22 | 1,0,0,0,-1,-1,0,0,0,3 23 | 1,0,0,0,-1,0,0,0,-1,6 24 | 1,-1,0,-1,-1,0,0,1,0,5 25 | 1,-1,0,0,-1,0,-1,1,0,2 26 | 1,-1,-1,0,-1,0,0,1,0,6 27 | 1,-1,0,0,-1,-1,0,1,0,3 28 | 1,0,-1,-1,-1,0,1,0,0,8 29 | 1,-1,1,0,-1,0,-1,0,0,7 30 | 1,0,0,1,-1,-1,-1,0,0,2 31 | 1,0,0,-1,-1,0,1,0,-1,5 32 | -------------------------------------------------------------------------------- /06_Neural_Networks/08_Learning_Tic_Tac_Toe/readme.md: -------------------------------------------------------------------------------- 1 | # Learning to Play Tic-Tac-Toe 2 | 3 | ## Goal 4 | 5 | This example intends to feed examples of best moves for many different board combinations into a neural network in order to train the model to play Tic-Tac-Toe. 6 | 7 | The end of the script provides the user a chance to play against the trained model by asking for input moves and feeding such input moves into the model. 8 | 9 | ## Data Format 10 | 11 | All tic-tac-toe boards can be reduced down to a small number of boards, if we consider all geometric transformations on them. Such geometric transformations include: 12 | 13 | - Rotate 90 deg. 14 | - Rotate 180 deg. 15 | - Rotate 270 deg. 16 | - Vertical reflection. 17 | - Horizontal reflection. 18 | 19 | All possible boards can be generated from the base board with at most 2 transformations. 20 | 21 | The file base\_tic\_tac\_toe\_moves.csv contains rows, each of which represents a unique board representation with the desired best play tactic. 22 | 23 | We denote the board spaces as such, 'X' = 1, 'O'= -1, and an empty space will have a zero in it. The last column is the index of the best play response. A board will be indexed as follows: 24 | ``` 25 | 0 | 1 | 2 26 | --------- 27 | 3 | 4 | 5 28 | --------- 29 | 6 | 7 | 8 30 | ``` 31 | So for example, the board: 32 | ``` 33 | O | | 34 | --------- 35 | X | O | O 36 | --------- 37 | | | X 38 | ``` 39 | is equivalent to the row: [-1, 0, 0, 1, -1, -1, 0, 0, 1]. 40 | 41 | ![TicTacToeIndexing](../images/08_tictactoe_layout.png "TicTacToeIndexing") 42 | 43 | ## Neural Network Architecture 44 | 45 | We will keep it simple and have only one hidden layer that is fully connected. The hidden layer will be composed of 81 hidden nodes. If only because square numbers are appealing. See the below diagram for the NN we will construct. 46 | 47 | ![TicTacToe Architecture](../images/08_tic_tac_toe_architecture.png "TicTacToe Architecture") 48 | 49 | ## Important Functions 50 | 51 | There are a few important functions in the beginning of the code. 52 | 53 | 1. print_board(): takes a board vector and shows it as a tic-tac-toe board. 54 | 2. get_symmetry(): takes a board, the preferred response index, and a transformation. It then applies the transformation to the board and response to get a new vector. 55 | 56 | At the end of the code, we loop through an actual game. This allows the user to actually play the model they created. 57 | 58 | [See Code Here](tic_tac_toe_moves.py) 59 | 60 | ## Sample Game Output 61 | 62 | Here is a sample of the output of playing against the trained model. Human = X's and the model = O's. 63 | 64 | ``` 65 | Input index of your move (0-8): 4 66 | Model has moved 67 | O | | 68 | ___________ 69 | | X | 70 | ___________ 71 | | | 72 | 73 | Input index of your move (0-8): 6 74 | Model has moved 75 | O | | 76 | ___________ 77 | | X | 78 | ___________ 79 | X | O | 80 | 81 | Input index of your move (0-8): 2 82 | Model has moved 83 | O | | X 84 | ___________ 85 | | X | O 86 | ___________ 87 | X | O | 88 | Game Over! 89 | ``` 90 | 91 | ## Loss Output 92 | 93 | ![TicTacToe Loss](../images/08_tictactoe_loss.png "TicTacToe Loss") 94 | -------------------------------------------------------------------------------- /06_Neural_Networks/images/02_operational_gates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/02_operational_gates.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/03_activation1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/03_activation1.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/03_activation2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/03_activation2.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/04_nn_layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/04_nn_layout.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/04_nn_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/04_nn_loss.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/06_nn_multiple_layers_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/06_nn_multiple_layers_loss.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/07_lin_reg_acc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/07_lin_reg_acc.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/07_lin_reg_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/07_lin_reg_loss.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/08_tic_tac_toe_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/08_tic_tac_toe_architecture.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/08_tictactoe_layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/08_tictactoe_layout.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/08_tictactoe_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/08_tictactoe_loss.png -------------------------------------------------------------------------------- /06_Neural_Networks/images/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/06_Neural_Networks/images/image.png -------------------------------------------------------------------------------- /06_Neural_Networks/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 6: Neural Networks 2 | 3 | Neural Networks are very important in machine learning and growing in popularity due to the major breakthroughs in prior unsolved problems. We must start with introducing 'shallow' neural networks, which are very powerful and can help us improve our prior ML algorithm results. We start by introducing the very basic NN unit, the operational gate. We gradually add more and more to the neural network and end with training a model to play tic-tac-toe. 4 | 5 | 1. [Introduction](01_Introduction#neural-networks-introduction) 6 | * We introduce the concept of neural networks and how TensorFlow is built to easily handle these algorithms. 7 | 2. [Implementing Operational Gates](02_Implementing_an_Operational_Gate#implementing-an-operational-gate) 8 | * We implement an operational gate with one operation. Then we show how to extend this to multiple nested operations. 9 | 3. [Working with Gates and Activation Functions](03_Working_with_Activation_Functions#working-with-activation-functions) 10 | * Now we have to introduce activation functions on the gates. We show how different activation functions operate. 11 | 4. [Implementing a One Layer Neural Network](04_Single_Hidden_Layer_Network#implementing-a-one-layer-neural-network) 12 | * We have all the pieces to start implementing our first neural network. We do so here with regression on the Iris data set. 13 | 5. [Implementing Different Layers](05_Implementing_Different_Layers#implementing-different-layers) 14 | * This section introduces the convolution layer and the max-pool layer. We show how to chain these together in a 1D and 2D example with fully connected layers as well. 15 | 6. [Using Multi-layer Neural Networks](06_Using_Multiple_Layers#using-multiple-layers) 16 | * Here we show how to functionalize different layers and variables for a cleaner multi-layer neural network. 17 | 7. [Improving Predictions of Linear Models](07_Improving_Linear_Regression#improving-linear-regression) 18 | * We show how we can improve the convergence of our prior logistic regression with a set of hidden layers. 19 | 8. [Learning to Play Tic-Tac-Toe](08_Learning_Tic_Tac_Toe#learning-to-play-tic-tac-toe) 20 | * Given a set of tic-tac-toe boards and corresponding optimal moves, we train a neural network classification model to play. At the end of the script, we can attempt to play against the trained model. 21 | -------------------------------------------------------------------------------- /07_Natural_Language_Processing/01_Introduction/readme.md: -------------------------------------------------------------------------------- 1 | # Natural Language Processing Introduction 2 | 3 | Placeholder for future purposes. 4 | 5 | Up to this point, we have only considered machine learning algorithms that mostly operate on numerical inputs. If we want to use text, we must find a way to convert the text into numbers. There are many ways to do this and we will explore a few common ways this is achieved.\n", 6 | 7 | 8 | If we consider the sentence **“tensorflow makes machine learning easy”**, we could convert the words to numbers in the order that we observe them. This would make the sentence become “1 2 3 4 5”. Then when we see a new sentence, **“machine learning is easy”**, we can translate this as “3 4 0 5”. Denoting words we haven’t seen bore with an index of zero. With these two examples, we have limited our vocabulary to 6 numbers. With large texts we can choose how many words we want to keep, and usually keep the most frequent words, labeling everything else with the index of zero. 9 | 10 | 11 | If the word “learning” has a numerical value of 4, and the word “makes” has a numerical value of 2, then it would be natural to assume that “learning” is twice “makes”. Since we do not want this type of numerical relationship between words, we assume these numbers represent categories and not relational numbers. 12 | 13 | 14 | Another problem is that these two sentences are of different size. Each observation we make (sentences in this case) need to have the same size input to a model we wish to create. To get around this, we create each sentence into a sparse vector that has that value of one in a specific index if that word occurs in that index. 15 | -------------------------------------------------------------------------------- /07_Natural_Language_Processing/02_Working_with_Bag_of_Words/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Bag of Words 2 | 3 | To illustrate how to use bag of words with a text data set, we will use a spam-ham phone text database from the UCI machine learning data repository (https://archive.ics.uci.edu/ml/datasets/SMS+Spam+Collection). This is a collection of phone text messages that are spam or not-spam (ham). 4 | 5 | We will download this data, store it for future use, and then proceed with the bag of words method to predict if a text is spam or not. The model that will operate on the bag of words will be a logistic model with no hidden layer. We will use stochastic training, with batch size of one, and compute the accuracy on a held out test set at the end. -------------------------------------------------------------------------------- /07_Natural_Language_Processing/03_Implementing_tf_idf/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing TF-IDF 2 | ------------------------- 3 | 4 | TF-IDF is an acronym that stands for Text Frequency – Inverse Document Frequency. This term is essentially the product of text frequency and inverse document frequency for each word. 5 | 6 | 7 | In the prior recipe, we introduced the bag of words methodology, which assigned a value of one for every occurrence of a word in a sentence. This is probably not ideal as each category of sentence (spam and ham for the prior recipe example) most likely has the same frequency of “the”, “and” and other words, whereas words like “viagra" and “sale” probably should have increased importance in figuring out whether or not the text is spam. 8 | 9 | 10 | We first want to take into consideration the word frequency. Here we consider the frequency that a word occurs in an individual entry. The purpose of this part (TF), is to find terms that appear to be important in each entry. 11 | 12 | 13 | But words like “the” and “and” may appear very frequently in every entry. We want to down weight the importance of these words, so we can imagine that multiplying the above text frequency (TF) by the inverse of the whole document frequency might help find important words. But since a collection of texts (a corpus) may be quite large, it is common to take the logarithm of the inverse document frequency. This leaves us with the following formula for TF-IDF for each word in each document entry. 14 | 15 | $$ 16 | w_{tf-idf}=w_{tf} \cdot \frac{1}{log(w_{df})} 17 | $$ 18 | 19 | 20 | Where $w_{tf}$ is the word frequency by document, and $w_{df}$ is the total frequency of such word across all documents. We can imagine that high values of TF-IDF might indicate words that are very important to determining what a document is about. 21 | 22 | 23 | Creating the TF-IDF vectors requires us to load all the text into memory and count the occurrences of each word before we can start training our model. Because of this, it is not implemented fully in Tensorflow, so we will use Scikit-learn for creating our TF-IDF embedding, but use Tensorflow to fit the logistic model. 24 | -------------------------------------------------------------------------------- /07_Natural_Language_Processing/04_Working_With_Skip_Gram_Embeddings/readme.md: -------------------------------------------------------------------------------- 1 | # Working with Skip Gram Embeddings 2 | 3 | Prior to this recipe, we have not considered the order of words to be relevant in creating word embeddings. In early 2013, Tomas Mikolov and other researchers at Google authored a paper about creating word embeddings that address this issue (https://arxiv.org/abs/1301.3781), and they named their methods “word2vec”. 4 | 5 | The basic idea is to create word embeddings that capture a relational aspect of words. We seek to understand how various words are related to each other. Some examples of how these embeddings might behave are as follows. 6 | 7 | - “king” – “man” + “woman” = “queen” 8 | - “india pale ale” – “hops” + “malt” = “stout” 9 | 10 | We might achieve such numerical representation of words if we only consider their positional relationship to each other. If we could analyse a large enough source of coherent documents, we might find that the words “king”, “man”, and “queen” are mentioned closely to each other in our texts. If we also know that “man” and “woman” are related in a different way, then we might conclude that “man” is to “king” as “woman” is to “queen” and so on. 11 | 12 | To go about finding such an embedding, we will use a neural network that predicts surrounding words giving an input word. We could, just as easily, switched that and tried to predict a target word given a set of surrounding words, but we will start with the prior method. Both are variations of the word2vec procedure. But the prior method of predicting the surrounding words (the context) from a target word is called the skip-gram model. In the next recipe, we will implement the other method, predicting the target word from the context, which is called the continuous bag of words method (CBOW). 13 | 14 | See below figure for an illustration. 15 | 16 | ![Skipgram](../images/04_skipgram_model.png "Word2Vec-SkipGram Example") 17 | 18 | 19 | -------------------------------------------------------------------------------- /07_Natural_Language_Processing/05_Working_With_CBOW_Embeddings/readme.md: -------------------------------------------------------------------------------- 1 | # Working with CBOW Embeddings 2 | 3 | In this recipe we will implement the CBOW (continuous bag of words) method of word2vec. It is very similar to the skip-gram method, except we are predicting a single target word from a surrounding window of context words. 4 | 5 | In the prior example we treated each combination of window and target as a group of paired inputs and outputs, but with CBOW we will add the surrounding window embeddings together to get one embedding to predict the target word embedding. 6 | 7 | Most of the code will stay the same, except we will need to change how we create the embeddings and how we generate the data from the sentences. 8 | 9 | To make the code easier to read, we have moved all the major functions to a separate file, called ‘text_helpers.py’ in the same directory. This function holds the data loading, text normalization, dictionary creation, and batch generation functions. This functions are exactly as they appear in the prior recipe, “Working with Skip-gram Embeddings”, except where noted. 10 | 11 | See the following illustration of a CBOW example. 12 | 13 | ![CBOW](../images/05_cbow_model.png "Word2Vec-CBOW Example") 14 | -------------------------------------------------------------------------------- /07_Natural_Language_Processing/06_Using_Word2Vec_Embeddings/readme.md: -------------------------------------------------------------------------------- 1 | # Using Word2Vec Embeddings 2 | 3 | ## Summary 4 | 5 | Now that we have created and saved CBOW word embeddings, we need to use them to make sentiment prediction on the movie data set. In this recipe, we will learn how to load and use prior trained embeddings and use these embeddings to perform sentiment analysis by training a logistic linear model to predict a good or bad review. 6 | 7 | Sentiment analysis is a really hard task to do because human language makes it very hard to grasp the subtleties and nuances of the true meaning. Sarcasm, jokes, ambiguous references all make the task exponentially harder. We will create a simple logistic regression on the movie review data set to see if we can get any information out of the CBOW embeddings we created and saved in the prior recipe. Since the focus of this recipe is in the loading and usage of saved embeddings, we will not pursue more complicated models. 8 | 9 | ## Pre-requisites 10 | 11 | In order to load the CBOW embeddings, we must run the [CBOW tutorial](../05_Working_With_CBOW_Embeddings) before running this one. Navigate to the [CBOW tutorial](../05_Working_With_CBOW_Embeddings) folder and run: 12 | 13 | python3 05_Working_With_CBOW.py 14 | 15 | This will train and save the CBOW embeddings we need to perform predictions for this recipe. 16 | -------------------------------------------------------------------------------- /07_Natural_Language_Processing/07_Sentiment_Analysis_With_Doc2Vec/readme.md: -------------------------------------------------------------------------------- 1 | # Sentiment Analysis with Doc2Vec 2 | 3 | In the prior sections about word2vec methods, we have managed to capture positional relationships between words. What we have not done is capture the relationship of words to the document (or movie review) that they come from. One extension of word2vec that captures a document effect is called doc2vec. 4 | 5 | The basic idea of doc2vec is to introduce a document embedding, along with the word embeddings, that may help to capture the tone of the document. For example, just knowing that the words “movie” and “love” are nearby to each other may not help us determine the sentiment of the review. The review maybe talking about how they love the move or how they do not love the move. But if the review is long enough and more negative words are found in the document, maybe we can pick up on an overall tone that may help us predict the next words. 6 | 7 | Doc2vec simply adds an additional embedding matrix for the documents and uses a window of words plus the document index to predict the next word. All word windows in a document have the same document index. It is worthwhile to mention that it is important to think about how we will combine the document embedding and the word embeddings. We combine the word embeddings in the word window by taking the sum. And there are two main ways to combine these embeddings with the document embedding. 8 | 9 | Commonly, the document embedding is either (1) added to the word embeddings, or (2) concatenated to the end of the word embeddings. If we add the two embeddings we limit the document embedding size to be the same size as the word embedding size. If we concatenate, we lift that restriction, but increase the number of variables that the logistic regression must deal with. For illustrative purposes, we show how to deal with concatenation in this recipe. But in general, for smaller datasets, addition is the better choice. 10 | 11 | The first step will be to fit both the document and word embeddings on the whole corpus of movie reviews, then we perform a train-test split and train a logistic model and see if we can improve upon the accuracy of predicting the review sentiment. 12 | 13 | Example training-loss for logistic regression on doc2vec embeddings for movie-review sentiment dataset: 14 | 15 | ![Doc2Vec](../images/07_sentiment_doc2vec_loss.png "Doc2Vec Example Loss") 16 | -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/02_bag_of_words.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/02_bag_of_words.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/03_tfidf_acc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/03_tfidf_acc.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/03_tfidf_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/03_tfidf_loss.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/04_skipgram_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/04_skipgram_model.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/05_cbow_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/05_cbow_model.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/06_word2vec_acc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/06_word2vec_acc.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/06_word2vec_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/06_word2vec_loss.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/07_sentiment_doc2vec_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/07_sentiment_doc2vec_loss.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/images/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/07_Natural_Language_Processing/images/image.png -------------------------------------------------------------------------------- /07_Natural_Language_Processing/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 7: Natural Language Processing 2 | 3 | 1. [Introduction](01_Introduction#natural-language-processing-introduction) 4 | * We introduce methods for turning text into numerical vectors. We introduce the TensorFlow 'embedding' feature as well. 5 | 2. [Working with Bag-of-Words](02_Working_with_Bag_of_Words#working-with-bag-of-words) 6 | * Here we use TensorFlow to do a one-hot-encoding of words called bag-of-words. We use this method and logistic regression to predict if a text message is spam or ham. 7 | 3. [Implementing TF-IDF](03_Implementing_tf_idf#implementing-tf-idf) 8 | * We implement Text Frequency - Inverse Document Frequency (TFIDF) with a combination of Sci-kit Learn and TensorFlow. We perform logistic regression on TFIDF vectors to improve on our spam/ham text-message predictions. 9 | 4. [Working with Skip-Gram](04_Working_With_Skip_Gram_Embeddings#working-with-skip-gram-embeddings) 10 | * Our first implementation of Word2Vec called, "skip-gram" on a movie review database. 11 | 5. [Working with CBOW](05_Working_With_CBOW_Embeddings#working-with-cbow-embeddings) 12 | * Next, we implement a form of Word2Vec called, "CBOW" (Continuous Bag of Words) on a movie review database. We also introduce method to saving and loading word embeddings. 13 | 6. [Implementing Word2Vec Example](06_Using_Word2Vec_Embeddings#using-word2vec-embeddings) 14 | * In this example, we use the prior saved CBOW word embeddings to improve on our TF-IDF logistic regression of movie review sentiment. 15 | 7. [Performing Sentiment Analysis with Doc2Vec](07_Sentiment_Analysis_With_Doc2Vec#sentiment-analysis-with-doc2vec) 16 | * Here, we introduce a Doc2Vec method (concatenation of doc and word embeddings) to improve out logistic model of movie review sentiment. 17 | -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/01_Intro_to_CNN/readme.md: -------------------------------------------------------------------------------- 1 | ## Introduction to Convolutional Neural Networks 2 | 3 | Convolutional Neural Networks (CNNs) are responsible for the latest major breakthroughs in image recognition in the past few years. 4 | 5 | In mathematics, a convolution is a function which is applied over the output of another function. In our case, we will consider applying a matrix mutliplication (filter) across an image. See the below diagram for an example of how this may work. 6 | 7 | ![Convolutional Filter](../images/01_intro_cnn.png) 8 | 9 | CNNs generally follow a structure. The main convolutional setup is (input array) -> (convolutional filter layer) -> (Pooling) -> (Activation layer). The above diagram depicts how a convolutional layer may create one feature. Generally, filters are multidimensional and end up creating many features. It is also common to have a completely separate filter-feature creator of different sizes acting on the same layer. After this convolutional filter, it is common to apply a pooling layer. This pooling may be a max-pooling or an average pooling or another aggregation. One of the key concepts here is that the pooling layer has no parameters- while decreasing the layer size. See the below diagram for an example of max-pooling. 10 | 11 | ![Convolutional Filter](../images/01_intro_cnn2.png) 12 | 13 | After the max pooling, there is generally an activation layer. One of the more common activation layers is the ReLU (Rectified Linear Unit). See [Chapter 1, Section 6](../../01_Introduction/06_Implementing_Activation_Functions) for examples. 14 | -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/02_Intro_to_CNN_MNIST/readme.md: -------------------------------------------------------------------------------- 1 | Introduction to CNN with MNIST 2 | ============================== 3 | 4 | Here we illustrate how to use a simple CNN with three convolutional units to predict the MNIST handwritten digits. There is good reason why this dataset is used like the 'hello world' of image recognition, it is fairly compact while having a decent amount of training, test, and validation data. It only has one channel (black and white) and only ten possible outputs (0-9). 5 | 6 | When the script is done training the model, you should see similar output to the following graphs. 7 | 8 | ![Loss and Accuracy](../images/02_cnn1_loss_acc.png "Loss and Accuracy") 9 | 10 | Training and test loss (left) and test batch accuracy (right). 11 | 12 | 13 | ![Sample Test Batch](../images/02_cnn1_mnist_output.png "Sample of 6 Images") 14 | 15 | A random set of 6 digits with actuals and predicted labels. You can see a prediction failure in the lower right box. 16 | -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/03_CNN_CIFAR10/readme.md: -------------------------------------------------------------------------------- 1 | CIFAR-10 CNN 2 | ============ 3 | 4 | Here we will build an convolutional neural network to predict the CIFAR-10 data. 5 | 6 | The script provided will download and unzip the CIFAR-10 data. Then it will start training a CNN from scratch. You should see similar output at the end to the following two graphs. 7 | 8 | ![Loss and Accuracy](../images/03_cnn2_loss_acc.png) 9 | 10 | Here we see the training loss (left) and the test batch accuracy (right). 11 | -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/04_Retraining_Current_Architectures/04_download_cifar10.py: -------------------------------------------------------------------------------- 1 | # Download/Saving CIFAR-10 images in Inception format 2 | #--------------------------------------- 3 | # 4 | # In this script, we download the CIFAR-10 images and 5 | # transform/save them in the Inception Retraining Format 6 | # 7 | # The end purpose of the files is for re-training the 8 | # Google Inception tensorflow model to work on the CIFAR-10. 9 | 10 | import os 11 | import tarfile 12 | import _pickle as cPickle 13 | import numpy as np 14 | import urllib.request 15 | import scipy.misc 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | cifar_link = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' 20 | data_dir = 'temp' 21 | if not os.path.isdir(data_dir): 22 | os.makedirs(data_dir) 23 | 24 | # Download tar file 25 | target_file = os.path.join(data_dir, 'cifar-10-python.tar.gz') 26 | if not os.path.isfile(target_file): 27 | print('CIFAR-10 file not found. Downloading CIFAR data (Size = 163MB)') 28 | print('This may take a few minutes, please wait.') 29 | filename, headers = urllib.request.urlretrieve(cifar_link, target_file) 30 | 31 | # Extract into memory 32 | tar = tarfile.open(target_file) 33 | tar.extractall(path=data_dir) 34 | tar.close() 35 | objects = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] 36 | 37 | # Create train image folders 38 | train_folder = 'train_dir' 39 | if not os.path.isdir(os.path.join(data_dir, train_folder)): 40 | for i in range(10): 41 | folder = os.path.join(data_dir, train_folder, objects[i]) 42 | os.makedirs(folder) 43 | # Create test image folders 44 | test_folder = 'validation_dir' 45 | if not os.path.isdir(os.path.join(data_dir, test_folder)): 46 | for i in range(10): 47 | folder = os.path.join(data_dir, test_folder, objects[i]) 48 | os.makedirs(folder) 49 | 50 | # Extract images accordingly 51 | data_location = os.path.join(data_dir, 'cifar-10-batches-py') 52 | train_names = ['data_batch_' + str(x) for x in range(1,6)] 53 | test_names = ['test_batch'] 54 | 55 | 56 | def load_batch_from_file(file): 57 | file_conn = open(file, 'rb') 58 | image_dictionary = cPickle.load(file_conn, encoding='latin1') 59 | file_conn.close() 60 | return image_dictionary 61 | 62 | 63 | def save_images_from_dict(image_dict, folder='data_dir'): 64 | # image_dict.keys() = 'labels', 'filenames', 'data', 'batch_label' 65 | for ix, label in enumerate(image_dict['labels']): 66 | folder_path = os.path.join(data_dir, folder, objects[label]) 67 | filename = image_dict['filenames'][ix] 68 | #Transform image data 69 | image_array = image_dict['data'][ix] 70 | image_array.resize([3, 32, 32]) 71 | # Save image 72 | output_location = os.path.join(folder_path, filename) 73 | scipy.misc.imsave(output_location,image_array.transpose()) 74 | 75 | # Sort train images 76 | for file in train_names: 77 | print('Saving images from file: {}'.format(file)) 78 | file_location = os.path.join(data_dir, 'cifar-10-batches-py', file) 79 | image_dict = load_batch_from_file(file_location) 80 | save_images_from_dict(image_dict, folder=train_folder) 81 | 82 | # Sort test images 83 | for file in test_names: 84 | print('Saving images from file: {}'.format(file)) 85 | file_location = os.path.join(data_dir, 'cifar-10-batches-py', file) 86 | image_dict = load_batch_from_file(file_location) 87 | save_images_from_dict(image_dict, folder=test_folder) 88 | 89 | # Create labels file 90 | cifar_labels_file = os.path.join(data_dir,'cifar10_labels.txt') 91 | print('Writing labels file, {}'.format(cifar_labels_file)) 92 | with open(cifar_labels_file, 'w') as labels_file: 93 | for item in objects: 94 | labels_file.write("{}\n".format(item)) 95 | 96 | # After this is done, we proceed with the TensorFlow fine-tuning tutorial. 97 | 98 | # https://www.tensorflow.org/tutorials/image_retraining 99 | -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/04_Retraining_Current_Architectures/readme.md: -------------------------------------------------------------------------------- 1 | Retraining (fine-tuning) Current CNN Architectures 2 | ================================================== 3 | 4 | The purpose of the script provided in this section is to download the CIFAR-10 data, and sort it out in the proper folder structure for running it through the TensorFlow fine-tuning tutorial. The script should create the following folder structure. 5 | 6 | ``` 7 | -train_dir 8 | |--airplane 9 | |--automobile 10 | |--bird 11 | |--cat 12 | |--deer 13 | |--dog 14 | |--frog 15 | |--horse 16 | |--ship 17 | |--truck 18 | -validation_dir 19 | |--airplane 20 | |--automobile 21 | |--bird 22 | |--cat 23 | |--deer 24 | |--dog 25 | |--frog 26 | |--horse 27 | |--ship 28 | |--truck 29 | ``` 30 | 31 | After this is done, we proceed with the [TensorFlow fine-tuning tutorial](https://github.com/tensorflow/models/tree/master/inception). 32 | 33 | -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/05_Stylenet_NeuralStyle/readme.md: -------------------------------------------------------------------------------- 1 | Stylenet / Neural-Style 2 | ======================= 3 | 4 | The purpose of this script is to illustrate how to do stylenet in TensorFlow. We reference the following [paper](https://arxiv.org/abs/1508.06576) for this algorithm. 5 | 6 | ## Prerequisites 7 | * Download the VGG-verydeep-19.mat file [here](http://www.vlfeat.org/matconvnet/models/beta16/imagenet-vgg-verydeep-19.mat). 8 | * You must download two images, a [style image](../images/starry_night.jpg) and a [content image](../images/book_cover.jpg) for the algorithm to blend. (Image links are to the images used in the book.) 9 | 10 | The algorithm will output temporary images during training. 11 | 12 | ![Stylenet Example](../images/05_stylenet_ex.png) 13 | -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/06_Deepdream/book_cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/06_Deepdream/book_cover.jpg -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/06_Deepdream/readme.md: -------------------------------------------------------------------------------- 1 | Deepdream in TensorFlow 2 | ======================= 3 | 4 | Note: There is no new code in this script. It originates from the TensorFlow tutorial [located here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/tutorials/deepdream). However, this code is modified slightly to run on Python 3. The code is also commented very heavily to explain, line-by-line, what occurs in the deepdream demo. 5 | 6 | Here are some potential outputs. 7 | 8 | ![deepdream outputs](../images/06_deepdream_ex.png) 9 | > Deepdream results on four features with the book's cover image. 10 | 11 | Our interpretation so far: 12 | * Feature #50: feathers 13 | * Feature #110: Shells or scales 14 | * Feature #100: Swirly eyeballs 15 | * Feature #139: Flowers (this one is from TF tutorial) 16 | -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/01_intro_cnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/01_intro_cnn.png -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/01_intro_cnn2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/01_intro_cnn2.png -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/02_cnn1_loss_acc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/02_cnn1_loss_acc.png -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/02_cnn1_mnist_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/02_cnn1_mnist_output.png -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/03_cnn2_loss_acc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/03_cnn2_loss_acc.png -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/05_stylenet_ex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/05_stylenet_ex.png -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/06_deepdream_ex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/06_deepdream_ex.png -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/book_cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/book_cover.jpg -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/image.png -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/images/starry_night.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/08_Convolutional_Neural_Networks/images/starry_night.jpg -------------------------------------------------------------------------------- /08_Convolutional_Neural_Networks/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 8: Convolutional Neural Networks 2 | 3 | 1. [Introduction](01_Intro_to_CNN#introduction-to-convolutional-neural-networks) 4 | * We introduce convolutional neural networks (CNN), and how we can use them in TensorFlow. 5 | 2. [Implementing a Simple CNN.](02_Intro_to_CNN_MNIST#introduction-to-cnn-with-mnist) 6 | * Here, we show how to create a CNN architecture that performs well on the MNIST digit recognition task. 7 | 3. [Implementing an Advanced CNN.](03_CNN_CIFAR10#cifar-10-cnn) 8 | * In this example, we show how to replicate an architecture for the CIFAR-10 image recognition task. 9 | 4. [Retraining an Existing Architecture.](04_Retraining_Current_Architectures#retraining-fine-tuning-current-cnn-architectures) 10 | * We show how to download and setup the CIFAR-10 data for the [TensorFlow retraining/fine-tuning tutorial.](https://github.com/tensorflow/models/tree/master/inception) 11 | 5. [Using Stylenet/NeuralStyle.](05_Stylenet_NeuralStyle#stylenet--neural-style) 12 | * In this recipe, we show a basic implementation of using Stylenet or Neuralstyle. 13 | 6. [Implementing Deep Dream.](06_Deepdream#deepdream-in-tensorflow) 14 | * This script shows a line-by-line explanation of TensorFlow's deepdream tutorial. Taken from [Deepdream on TensorFlow](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/tutorials/deepdream). Note that the code here is converted to Python 3. 15 | -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/01_Introduction/readme.md: -------------------------------------------------------------------------------- 1 | # Introduction To RNNs in TensorFlow 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/02_Implementing_RNN_for_Spam_Prediction/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing an RNN for Spam Prediction 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/03_Implementing_LSTM/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing an LSTM Model 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/04_Stacking_Multiple_LSTM_Layers/readme.md: -------------------------------------------------------------------------------- 1 | # Stacking Multiple LSTM Layers 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/05_Creating_A_Sequence_To_Sequence_Model/readme.md: -------------------------------------------------------------------------------- 1 | # Creating a Sequence to Sequence Model with TensorFlow (Seq2Seq) 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/06_Training_A_Siamese_Similarity_Measure/readme.md: -------------------------------------------------------------------------------- 1 | # Training a Siamese Similarity Measure (RNNs) 2 | 3 | In this recipe, we will train a model to perform address matching, and consider addresses as string inputs. The model will use a siamese network to calculate the similarity between an input address and a set of canonical addresses. The canonical address that is the closest (highest similarity score) is chosen as the true address. 4 | 5 | # Benefits of Siamese Networks 6 | 7 | The benefits of using a siamese network is that we have at least doubled our data size (if not more) and the network will do exact matches by default, which means it will not have to learn the identity function between addresses. 8 | 9 | Here is a general architecture of a siamese neural network: 10 | 11 | ![General Siamese Architecture](../images/06_Similarity_RNN_Architecture.png "General Siamese Architecture") 12 | 13 | # Data 14 | 15 | The data we will use will be contrived data. We generate a list of addresses as the truth, then we introduce a set of addresses that have at least one typo in them. 16 | 17 | # Model Architecture 18 | 19 | For this specific siamese network, we use a RNN network as the input and then run the outputs through a fully connected layer. The similarity is then just the cosine function of the two vectors as seen below. 20 | 21 | ![Siamese RNN Diagram](../images/06_Similarity_RNN_Diagram.png "Siamese RNN Diagram") 22 | 23 | # Graph of Loss Function and Batch Accuracy 24 | 25 | Running the script should result in a similar loss. 26 | 27 | ![Accuracy and Loss](../images/06_Similarity_RNN.png "Accuracy and Loss") 28 | -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/01_RNN_Seq2Seq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/01_RNN_Seq2Seq.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/01_RNN_Single_Target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/01_RNN_Single_Target.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/02_RNN_Spam_Acc_Loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/02_RNN_Spam_Acc_Loss.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/03_LSTM_Loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/03_LSTM_Loss.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/04_MultipleLSTM_Loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/04_MultipleLSTM_Loss.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/04_MultipleRNN_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/04_MultipleRNN_Architecture.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/05_Seq2Seq_Loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/05_Seq2Seq_Loss.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/06_Similarity_RNN.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/06_Similarity_RNN.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/06_Similarity_RNN_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/06_Similarity_RNN_Architecture.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/images/06_Similarity_RNN_Diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/09_Recurrent_Neural_Networks/images/06_Similarity_RNN_Diagram.png -------------------------------------------------------------------------------- /09_Recurrent_Neural_Networks/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 9: Recurrent Neural Networks 2 | 3 | 1. [Introduction](01_Introduction#introduction-to-rnns-in-tensorflow) 4 | * We introduce Recurrent Neural Networks and how they are able to feed in a sequence and predict either a fixed target (categorical/numerical) or another sequence (sequence to sequence). 5 | 2. [Implementing an RNN Model for Spam Prediction](02_Implementing_RNN_for_Spam_Prediction#implementing-an-rnn-for-spam-prediction) 6 | * We create an RNN model to improve on our spam/ham SMS text predictions. 7 | 3. [Implementing an LSTM Model for Text Generation](03_Implementing_LSTM#implementing-an-lstm-model) 8 | * We show how to implement a LSTM (Long Short Term Memory) RNN for Shakespeare language generation. (Word level vocabulary) 9 | 4. [Stacking Multiple LSTM Layers](04_Stacking_Multiple_LSTM_Layers#stacking-multiple-lstm-layers) 10 | * We stack multiple LSTM layers to improve on our Shakespeare language generation. (Character level vocabulary) 11 | 5. [Creating a Sequence to Sequence Translation Model (Seq2Seq)](05_Creating_A_Sequence_To_Sequence_Model#creating-a-sequence-to-sequence-model-with-tensorflow-seq2seq) 12 | * We show how to use TensorFlow's sequence-to-sequence models to train an English-German translation model. 13 | 6. [Training a Siamese Similarity Measure](06_Training_A_Siamese_Similarity_Measure#training-a-siamese-similarity-measure-rnns) 14 | * Here, we implement a Siamese RNN to predict the similarity of addresses and use it for record matching. Using RNNs for record matching is very versatile, as we do not have a fixed set of target categories and can use the trained model to predict similarities across new addresses. 15 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/01_Implementing_Unit_Tests/readme.md: -------------------------------------------------------------------------------- 1 | # Implementing Unit Tests 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/02_Using_Multiple_Devices/02_using_multiple_devices.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Using Multiple Devices 3 | #---------------------------------- 4 | # 5 | # This function gives us the ways to use 6 | # multiple devices (executors) in TensorFlow. 7 | 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # To find out where placement occurs, set 'log_device_placement' 13 | sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) 14 | 15 | a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') 16 | b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') 17 | c = tf.matmul(a, b) 18 | 19 | # Runs the op. 20 | print(sess.run(c)) 21 | 22 | 23 | # If we load a graph and want device placement to be forgotten, 24 | # we set a parameter in our session: 25 | config = tf.ConfigProto() 26 | config.allow_soft_placement = True 27 | sess_soft = tf.Session(config=config) 28 | 29 | # GPUs 30 | #--------------------------------- 31 | # Note that the GPU must have a compute capability > 3.5 for TF to use. 32 | # http://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#compute-capability 33 | 34 | 35 | # Careful with GPU memory allocation, TF never releases it. TF starts with almost 36 | # all of the GPU memory allocated. We can slowly grow to that limit with an 37 | # option setting: 38 | 39 | config.gpu_options.allow_growth = True 40 | sess_grow = tf.Session(config=config) 41 | 42 | # Also, we can limit the size of GPU memory used, with the following option 43 | config.gpu_options.per_process_gpu_memory_fraction = 0.4 44 | sess_limited = tf.Session(config=config) 45 | 46 | 47 | # How to set placements on multiple devices. 48 | # Here, assume we have three devies CPU:0, GPU:0, and GPU:1 49 | if tf.test.is_built_with_cuda(): 50 | with tf.device('/cpu:0'): 51 | a = tf.constant([1.0, 3.0, 5.0], shape=[1, 3]) 52 | b = tf.constant([2.0, 4.0, 6.0], shape=[3, 1]) 53 | 54 | with tf.device('/gpu:1'): 55 | c = tf.matmul(a,b) 56 | c = tf.reshape(c, [-1]) 57 | 58 | with tf.device('/gpu:2'): 59 | d = tf.matmul(b,a) 60 | flat_d = tf.reshape(d, [-1]) 61 | 62 | combined = tf.multiply(c, flat_d) 63 | print(sess.run(combined)) 64 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/02_Using_Multiple_Devices/readme.md: -------------------------------------------------------------------------------- 1 | # Using Multiple Devices 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/03_Parallelizing_TensorFlow/03_parallelizing_tensorflow.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Parallelizing TensorFlow 3 | #---------------------------------- 4 | # 5 | # We will show how to use TensorFlow distributed 6 | 7 | import tensorflow as tf 8 | 9 | # We will setup a local cluster (on localhost) 10 | 11 | # Cluster for 2 local workers (tasks 0 and 1): 12 | cluster = tf.train.ClusterSpec({'local': ['localhost:2222', 'localhost:2223']}) 13 | # Server definition: 14 | server = tf.train.Server(cluster, job_name="local", task_index=0) 15 | server = tf.train.Server(cluster, job_name="local", task_index=1) 16 | # Finish and add 17 | # server.join() 18 | 19 | # Have each worker do a task 20 | # Worker 0 : create matrices 21 | # Worker 1 : calculate sum of all elements 22 | mat_dim = 25 23 | matrix_list = {} 24 | 25 | with tf.device('/job:local/task:0'): 26 | for i in range(0, 2): 27 | m_label = 'm_{}'.format(i) 28 | matrix_list[m_label] = tf.random_normal([mat_dim, mat_dim]) 29 | 30 | # Have each worker calculate the Cholesky Decomposition 31 | sum_outs = {} 32 | with tf.device('/job:local/task:1'): 33 | for i in range(0, 2): 34 | A = matrix_list['m_{}'.format(i)] 35 | sum_outs['m_{}'.format(i)] = tf.reduce_sum(A) 36 | 37 | # Sum all the cholesky decompositions 38 | summed_out = tf.add_n(list(sum_outs.values())) 39 | 40 | with tf.Session(server.target) as sess: 41 | result = sess.run(summed_out) 42 | print('Summed Values:{}'.format(result)) 43 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/03_Parallelizing_TensorFlow/readme.md: -------------------------------------------------------------------------------- 1 | # Parallelizing TensorFlow 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/04_Production_Tips/04_production_tips_for_tf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Tips for TensorFlow to Production 3 | #---------------------------------- 4 | # 5 | # Various Tips for Taking TensorFlow to Production 6 | 7 | ############################################ 8 | # 9 | # THIS SCRIPT IS NOT RUNNABLE. 10 | # -it only contains tips for production code 11 | # 12 | ############################################ 13 | 14 | # Also you can clear the default graph from memory 15 | import tensorflow as tf 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Saving Models 20 | # File types created from saving: 21 | # checkpoint file: Holds info on where the most recent models are 22 | # events file: Strictly for viewing graph in Tensorboard 23 | # pbtxt file: Textual protobufs file (uncompressed), used for debugging 24 | # chkp file: Holds data and model weights (large file) 25 | # meta chkp files: Model Graph and Meta-data (learning rate and operations) 26 | 27 | 28 | # Saving data pipeline structures (vocabulary, ) 29 | word_list = ['to', 'be', 'or', 'not', 'to', 'be'] 30 | vocab_list = list(set(word_list)) 31 | vocab2ix_dict = dict(zip(vocab_list, range(len(vocab_list)))) 32 | ix2vocab_dict = {val:key for key,val in vocab2ix_dict.items()} 33 | 34 | # Save vocabulary 35 | import json 36 | with open('vocab2ix_dict.json', 'w') as file_conn: 37 | json.dump(vocab2ix_dict, file_conn) 38 | 39 | # Load vocabulary 40 | with open('vocab2ix_dict.json', 'r') as file_conn: 41 | vocab2ix_dict = json.load(file_conn) 42 | 43 | # After model declaration, add a saving operations 44 | saver = tf.train.Saver() 45 | # Then during training, save every so often, referencing the training generation 46 | for i in range(generations): 47 | ... 48 | if i%save_every == 0: 49 | saver.save(sess, 'my_model', global_step=step) 50 | 51 | # Can also save only specific variables: 52 | saver = tf.train.Saver({"my_var": my_variable}) 53 | 54 | 55 | # other options for saver are 'keep checkpoint_every_n_hours' 56 | # also 'max_to_keep'= default 5. 57 | 58 | # Be sure to name operations, and variables for easy loading for referencing later 59 | conv_weights = tf.Variable(tf.random_normal(), name='conv_weights') 60 | loss = tf.reduce_mean(... , name='loss') 61 | 62 | # Instead of tyring argparse and main(), TensorFlow provides an 'app' function 63 | # to handle running and loading of arguments 64 | 65 | # At the beginning of the file, define the flags. 66 | tf.flags.DEFINE_string("worker_locations", "", "List of worker addresses.") 67 | tf.flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.') 68 | tf.flags.DEFINE_integer('generations', 1000, 'Number of training generations.') 69 | tf.flags.DEFINE_boolean('run_unit_tests', False, 'If true, run tests.') 70 | FLAGS = tf.flags.FLAGS 71 | 72 | # Need to define a 'main' function for the app to run 73 | def main(_): 74 | worker_ips = FLAGS.worker_locations.split(",") 75 | learning_rate = FLAGS.learning_rate 76 | generations = FLAGS.generations 77 | run_unit_tests = FLAGS.run_unit_tests 78 | 79 | # Run the TensorFlow app 80 | if __name__ == "__main__": 81 | # The following is looking for a "main()" function to run and will pass. 82 | tf.app.run() 83 | # Can modify this to be more custom: 84 | tf.app.run(main=my_main_function(), argv=my_arguments) 85 | 86 | 87 | # Use of TensorFlow's built in logging: 88 | # Five levels: DEBUG, INFO, WARN, ERROR, and FATAL 89 | tf.logging.set_verbosity(tf.logging.WARN) 90 | # WARN is the default value, but to see more information, you can set it to 91 | # INFO or DEBUG 92 | tf.logging.set_verbosity(tf.logging.DEBUG) 93 | # Note: 'DEBUG' is quite verbose. 94 | 95 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/04_Production_Tips/readme.md: -------------------------------------------------------------------------------- 1 | # Production Tips with TensorFlow 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/05_Production_Example/05_production_ex_eval.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # TensorFlow Production Example (Evaluating) 3 | #---------------------------------- 4 | # 5 | # We pull together everything and create an example 6 | # of best tensorflow production tips 7 | # 8 | # The example we will productionalize is the spam/ham RNN 9 | # from the RNN Chapter. 10 | 11 | import os 12 | import re 13 | import numpy as np 14 | import tensorflow as tf 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | tf.flags.DEFINE_string("storage_folder", "temp", "Where to store model and data.") 19 | tf.flags.DEFINE_bool('model_file', False, 'Model file location.') 20 | tf.flags.DEFINE_bool('run_unit_tests', False, 'If true, run tests.') 21 | FLAGS = tf.flags.FLAGS 22 | 23 | 24 | # Create a text cleaning function 25 | def clean_text(text_string): 26 | text_string = re.sub(r'([^\s\w]|_|[0-9])+', '', text_string) 27 | text_string = " ".join(text_string.split()) 28 | text_string = text_string.lower() 29 | return text_string 30 | 31 | 32 | # Load vocab processor 33 | def load_vocab(): 34 | vocab_path = os.path.join(FLAGS.storage_folder, "vocab") 35 | vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor.restore(vocab_path) 36 | return vocab_processor 37 | 38 | 39 | # Process input data: 40 | def process_data(input_data, vocab_processor): 41 | input_data = clean_text(input_data) 42 | input_data = input_data.split() 43 | processed_input = np.array(list(vocab_processor.transform(input_data))) 44 | return processed_input 45 | 46 | 47 | # Get input function 48 | def get_input_data(): 49 | """ 50 | For this function, we just prompt the user for a text message to evaluate 51 | But this function could also potentially read a file in as well. 52 | """ 53 | input_text = input("Please enter a text message to evaluate: ") 54 | vocab_processor = load_vocab() 55 | return process_data(input_text, vocab_processor) 56 | 57 | 58 | # Test clean_text function 59 | class clean_test(tf.test.TestCase): 60 | # Make sure cleaning function behaves correctly 61 | def clean_string_test(self): 62 | with self.test_session(): 63 | test_input = '--TensorFlow\'s so Great! Don\t you think so? ' 64 | test_expected = 'tensorflows so great don you think so' 65 | test_out = clean_text(test_input) 66 | self.assertEqual(test_expected, test_out) 67 | 68 | 69 | # Main function 70 | def main(args): 71 | # Get flags 72 | storage_folder = FLAGS.storage_folder 73 | 74 | # Get user input text 75 | x_data = get_input_data() 76 | 77 | # Load model 78 | graph = tf.Graph() 79 | with graph.as_default(): 80 | sess = tf.Session() 81 | with sess.as_default(): 82 | # Load the saved meta graph and restore variables 83 | saver = tf.train.import_meta_graph("{}.meta".format(os.path.join(storage_folder, "model.ckpt"))) 84 | saver.restore(sess, os.path.join(storage_folder, "model.ckpt")) 85 | 86 | # Get the placeholders from the graph by name 87 | x_data_ph = graph.get_operation_by_name("x_data_ph").outputs[0] 88 | dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0] 89 | probability_outputs = graph.get_operation_by_name("probability_outputs").outputs[0] 90 | 91 | # Make the prediction 92 | eval_feed_dict = {x_data_ph: x_data, dropout_keep_prob: 1.0} 93 | probability_prediction = sess.run(tf.reduce_mean(probability_outputs, 0), eval_feed_dict) 94 | 95 | # Print output (Or save to file or DB connection?) 96 | print('Probability of Spam: {:.4}'.format(probability_prediction[1])) 97 | 98 | 99 | # Run main module/tf App 100 | if __name__ == "__main__": 101 | if FLAGS.run_unit_tests: 102 | # Perform unit tests 103 | tf.test.main() 104 | else: 105 | # Run evaluation 106 | tf.app.run() -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/05_Production_Example/readme.md: -------------------------------------------------------------------------------- 1 | # A Production Example 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/06_Using_TensorFlow_Serving/06_Using_TensorFlow_Serving_Client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Using TensorFlow Serving (CLIENT) 3 | #---------------------------------- 4 | # 5 | # We show how to use "TensorFlow Serving", a model serving api from TensorFlow to serve a model. 6 | # 7 | # Pre-requisites: 8 | # - Visit https://www.tensorflow.org/serving/setup 9 | # and follow all the instructions on setting up TensorFlow Serving (including installing Bazel). 10 | # 11 | # The example we will query the TensorFlow-Serving-API we have running on port 9000 12 | 13 | import os 14 | import re 15 | import grpc 16 | import numpy as np 17 | import tensorflow as tf 18 | 19 | from tensorflow_serving.apis import predict_pb2 20 | from tensorflow_serving.apis import prediction_service_pb2 21 | 22 | tf.flags.DEFINE_string('server', '9000', 'PredictionService host') 23 | tf.flags.DEFINE_string('port', '0.0.0.0', 'PredictionService port') 24 | tf.flags.DEFINE_string('data_dir', 'temp', 'Folder where vocabulary is.') 25 | FLAGS = tf.flags.FLAGS 26 | 27 | 28 | # Def a functions to process texts into arrays of indices 29 | # Create a text cleaning function 30 | def clean_text(text_string): 31 | text_string = re.sub(r'([^\s\w]|_|[0-9])+', '', text_string) 32 | text_string = " ".join(text_string.split()) 33 | text_string = text_string.lower() 34 | return text_string 35 | 36 | 37 | # Load vocab processor 38 | def load_vocab(): 39 | vocab_path = os.path.join(FLAGS.data_dir, 'vocab') 40 | vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor.restore(vocab_path) 41 | return vocab_processor 42 | 43 | 44 | # Process input data: 45 | def process_data(input_data): 46 | vocab_processor = load_vocab() 47 | input_data = [clean_text(x) for x in input_data] 48 | processed_input = np.array(list(vocab_processor.transform(input_data))) 49 | return processed_input 50 | 51 | 52 | def get_results(data, server, port): 53 | channel = grpc.insecure_channel(':'.join([server, port])) 54 | stub = prediction_service_pb2.PredictionServiceStub(channel) 55 | processed_data = process_data(data) 56 | 57 | results = [] 58 | for input_x in processed_data: 59 | request = predict_pb2.PredictRequest() 60 | request.model_spec.name = 'spam_ham' 61 | request.model_spec.signature_name = 'predict_spam' # Change to predict spam 62 | request.inputs['texts'].CopyFrom(tf.contrib.util.make_tensor_proto(input_x, shape=[4, 20])) # 'texts' 63 | prediction_future = stub.Predict(request) 64 | prediction = prediction_future.result().outputs['scores'] 65 | # prediction = np.array(prediction_future.result().outputs['scores'].float_val) 66 | results.append(prediction) 67 | return results 68 | 69 | 70 | def main(data): 71 | if not FLAGS.server: 72 | print('please specify server host:port') 73 | return 74 | results = get_results(data, FLAGS.server, FLAGS.port) 75 | 76 | for input_text, output_pred in zip(data, results): 77 | print('Input text: {}, Prediction: {}'.format(input_text, output_pred)) 78 | 79 | 80 | if __name__ == '__main__': 81 | # Get sample data, here you may feel free to change this to a file, cloud-address, user input, etc... 82 | test_data = ['Please respond ASAP to claim your prize !', 83 | 'Hey, are you coming over for dinner tonight?', 84 | 'Text 444 now to see the top users in your area', 85 | 'drive safe, and thanks for visiting again!'] 86 | 87 | tf.app.run(argv=test_data) 88 | -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/images/file_structure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/10_Taking_TensorFlow_to_Production/images/file_structure.jpg -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/images/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/10_Taking_TensorFlow_to_Production/images/image.png -------------------------------------------------------------------------------- /10_Taking_TensorFlow_to_Production/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 10:Taking TensorFlow to Production 2 | 3 | 1. [Implementing Unit Tests](01_Implementing_Unit_Tests#implementing-unit-tests) 4 | * We show how to implement different types of unit tests on tensors (placeholders and variables). 5 | 2. [Using Multiple Executors (Devices)](02_Using_Multiple_Devices#using-multiple-devices) 6 | * How to use a machine with multiple devices. E.g., a machine with a CPU, and one or more GPUs. 7 | 3. [Parallelizing TensorFlow](03_Parallelizing_TensorFlow#parallelizing-tensorflow) 8 | * How to setup and use TensorFlow distributed on multiple machines. 9 | 4. [Tips for TensorFlow in Production](04_Production_Tips#production-tips-with-tensorflow) 10 | * Various tips for developing with TensorFlow 11 | 5. [An Example of Productionalizing TensorFlow](05_Production_Example#a-production-example) 12 | * We show how to do take the RNN model for predicting ham/spam (from Chapter 9, recipe #2) and put it in two production level files: training and evaluation. 13 | -------------------------------------------------------------------------------- /11_More_with_TensorFlow/01_Visualizing_Computational_Graphs/01_using_tensorboard.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Using Tensorboard 3 | #---------------------------------- 4 | # 5 | # We illustrate the various ways to use 6 | # Tensorboard 7 | 8 | import os 9 | import io 10 | import time 11 | import pathlib 12 | import numpy as np 13 | import matplotlib.pyplot as plt 14 | import tensorflow as tf 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | # Initialize a graph session 19 | sess = tf.Session() 20 | 21 | # Create tensorboard folder if not exists 22 | if not os.path.exists('tensorboard'): 23 | os.makedirs('tensorboard') 24 | print('Running a slowed down linear regression. ' 25 | 'Run the command: $tensorboard --logdir="tensorboard" ' 26 | ' Then navigate to http://127.0.0.1:6006') 27 | 28 | # You can also specify a port option with --port 6006 29 | 30 | # Create a visualizer object 31 | summary_writer = tf.summary.FileWriter('tensorboard', sess.graph) 32 | 33 | # Wait a few seconds for user to run tensorboard commands 34 | time.sleep(3) 35 | 36 | # Some parameters 37 | batch_size = 50 38 | generations = 100 39 | 40 | # Create sample input data 41 | x_data = np.arange(1000)/10. 42 | true_slope = 2. 43 | y_data = x_data * true_slope + np.random.normal(loc=0.0, scale=25, size=1000) 44 | 45 | # Split into train/test 46 | train_ix = np.random.choice(len(x_data), size=int(len(x_data)*0.9), replace=False) 47 | test_ix = np.setdiff1d(np.arange(1000), train_ix) 48 | x_data_train, y_data_train = x_data[train_ix], y_data[train_ix] 49 | x_data_test, y_data_test = x_data[test_ix], y_data[test_ix] 50 | 51 | # Declare placeholders 52 | x_graph_input = tf.placeholder(tf.float32, [None]) 53 | y_graph_input = tf.placeholder(tf.float32, [None]) 54 | 55 | # Declare model variables 56 | m = tf.Variable(tf.random_normal([1], dtype=tf.float32), name='Slope') 57 | 58 | # Declare model 59 | output = tf.multiply(m, x_graph_input, name='Batch_Multiplication') 60 | 61 | # Declare loss function (L1) 62 | residuals = output - y_graph_input 63 | l1_loss = tf.reduce_mean(tf.abs(residuals), name="L1_Loss") 64 | 65 | # Declare optimization function 66 | my_optim = tf.train.GradientDescentOptimizer(0.01) 67 | train_step = my_optim.minimize(l1_loss) 68 | 69 | # Visualize a scalar 70 | with tf.name_scope('Slope_Estimate'): 71 | tf.summary.scalar('Slope_Estimate', tf.squeeze(m)) 72 | 73 | # Visualize a histogram (errors) 74 | with tf.name_scope('Loss_and_Residuals'): 75 | tf.summary.histogram('Histogram_Errors', tf.squeeze(l1_loss)) 76 | tf.summary.histogram('Histogram_Residuals', tf.squeeze(residuals)) 77 | 78 | 79 | # Declare summary merging operation 80 | summary_op = tf.summary.merge_all() 81 | 82 | # Initialize Variables 83 | init = tf.global_variables_initializer() 84 | sess.run(init) 85 | 86 | for i in range(generations): 87 | batch_indices = np.random.choice(len(x_data_train), size=batch_size) 88 | x_batch = x_data_train[batch_indices] 89 | y_batch = y_data_train[batch_indices] 90 | _, train_loss, summary = sess.run([train_step, l1_loss, summary_op], 91 | feed_dict={x_graph_input: x_batch, 92 | y_graph_input: y_batch}) 93 | 94 | test_loss, test_resids = sess.run([l1_loss, residuals], feed_dict={x_graph_input: x_data_test, 95 | y_graph_input: y_data_test}) 96 | 97 | if (i + 1) % 10 == 0: 98 | print('Generation {} of {}. Train Loss: {:.3}, Test Loss: {:.3}.'.format(i+1, generations, train_loss, test_loss)) 99 | 100 | log_writer = tf.summary.FileWriter('tensorboard') 101 | log_writer.add_summary(summary, i) 102 | time.sleep(0.5) 103 | 104 | #Create a function to save a protobuf bytes version of the graph 105 | def gen_linear_plot(slope): 106 | linear_prediction = x_data * slope 107 | plt.plot(x_data, y_data, 'b.', label='data') 108 | plt.plot(x_data, linear_prediction, 'r-', linewidth=3, label='predicted line') 109 | plt.legend(loc='upper left') 110 | buf = io.BytesIO() 111 | plt.savefig(buf, format='png') 112 | buf.seek(0) 113 | return(buf) 114 | 115 | # Add image to tensorboard (plot the linear fit!) 116 | slope = sess.run(m) 117 | plot_buf = gen_linear_plot(slope[0]) 118 | # Convert PNG buffer to TF image 119 | image = tf.image.decode_png(plot_buf.getvalue(), channels=4) 120 | # Add the batch dimension 121 | image = tf.expand_dims(image, 0) 122 | # Add image summary 123 | image_summary_op = tf.summary.image("Linear_Plot", image) 124 | image_summary = sess.run(image_summary_op) 125 | log_writer.add_summary(image_summary, i) 126 | log_writer.close() 127 | -------------------------------------------------------------------------------- /11_More_with_TensorFlow/01_Visualizing_Computational_Graphs/readme.md: -------------------------------------------------------------------------------- 1 | # Visualizing Computational Graphs (w/Tensorboard) 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /11_More_with_TensorFlow/02_Working_with_a_Genetic_Algorithm/readme.md: -------------------------------------------------------------------------------- 1 | # Working with a Genetic Algorithm 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /11_More_with_TensorFlow/03_Clustering_Using_KMeans/readme.md: -------------------------------------------------------------------------------- 1 | # Clustering Using K-Means 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /11_More_with_TensorFlow/04_Solving_A_System_of_ODEs/04_solving_ode_system.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Solving a Sytem of ODEs 3 | #---------------------------------- 4 | # 5 | # In this script, we use TensorFlow to solve a sytem 6 | # of ODEs. 7 | # 8 | # The system of ODEs we will solve is the Lotka-Volterra 9 | # predator-prey system. 10 | 11 | 12 | # Declaring Operations 13 | import matplotlib.pyplot as plt 14 | import tensorflow as tf 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | # Open interactive graph session 19 | sess = tf.Session() 20 | 21 | # Discrete Lotka-Volterra predator/prey equations 22 | # 23 | # X(t+1) = X(t) + (aX(t) + bX(t)Y(t)) * t_delta # Prey 24 | # 25 | # Y(t+1) = Y(t) + (cY(t) + dX(t)Y(t)) * t_delta # Predator 26 | 27 | # Declare constants and variables 28 | x_initial = tf.constant(1.0) 29 | y_initial = tf.constant(1.0) 30 | X_t1 = tf.Variable(x_initial) 31 | Y_t1 = tf.Variable(y_initial) 32 | 33 | # Make the placeholders 34 | t_delta = tf.placeholder(tf.float32, shape=()) 35 | a = tf.placeholder(tf.float32, shape=()) 36 | b = tf.placeholder(tf.float32, shape=()) 37 | c = tf.placeholder(tf.float32, shape=()) 38 | d = tf.placeholder(tf.float32, shape=()) 39 | 40 | # Discretized ODE update 41 | X_t2 = X_t1 + (a * X_t1 + b * X_t1 * Y_t1) * t_delta 42 | Y_t2 = Y_t1 + (c * Y_t1 + d * X_t1 * Y_t1) * t_delta 43 | 44 | # Update to New Population 45 | step = tf.group( 46 | X_t1.assign(X_t2), 47 | Y_t1.assign(Y_t2)) 48 | 49 | init = tf.global_variables_initializer() 50 | sess.run(init) 51 | 52 | # Run the ODE 53 | prey_values = [] 54 | predator_values = [] 55 | for i in range(1000): 56 | # Step simulation (using constants for a known cyclic solution) 57 | step.run({a: (2./3.), b: (-4./3.), c: -1.0, d: 1.0, t_delta: 0.01}, session=sess) 58 | # Store each outcome 59 | temp_prey, temp_pred = sess.run([X_t1, Y_t1]) 60 | prey_values.append(temp_prey) 61 | predator_values.append(temp_pred) 62 | 63 | # Visualize the output 64 | plt.plot(prey_values) 65 | plt.plot(predator_values) 66 | plt.legend(['Prey', 'Predator'], loc='upper right') 67 | plt.show() 68 | -------------------------------------------------------------------------------- /11_More_with_TensorFlow/04_Solving_A_System_of_ODEs/readme.md: -------------------------------------------------------------------------------- 1 | # Solving a System of ODEs 2 | 3 | Placeholder for future purposes 4 | -------------------------------------------------------------------------------- /11_More_with_TensorFlow/05_Using_a_Random_Forest/05_Using_a_Random_Forest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Using a Random Forest 3 | --------------------- 4 | 5 | This script will illustrate how to use TensorFlow's Boosted Random Forest algorithm. 6 | 7 | 8 | For illustrative purposes we will show how to do this with the boston housing data. 9 | 10 | Attribute Information: 11 | 12 | 1. CRIM per capita crime rate by town 13 | 2. ZN proportion of residential land zoned for lots over 14 | 25,000 sq.ft. 15 | 3. INDUS proportion of non-retail business acres per town 16 | 4. CHAS Charles River dummy variable (= 1 if tract bounds 17 | river; 0 otherwise) 18 | 5. NOX nitric oxides concentration (parts per 10 million) 19 | 6. RM average number of rooms per dwelling 20 | 7. AGE proportion of owner-occupied units built prior to 1940 21 | 8. DIS weighted distances to five Boston employment centres 22 | 9. RAD index of accessibility to radial highways 23 | 10. TAX full-value property-tax rate per $10,000 24 | 11. PTRATIO pupil-teacher ratio by town 25 | 12. B 1000(Bk - 0.63)^2 where Bk is the proportion of blacks 26 | by town 27 | 13. LSTAT % lower status of the population 28 | 14. y_target Median value of owner-occupied homes in $1000's. 29 | """ 30 | 31 | import os 32 | import numpy as np 33 | import tensorflow as tf 34 | from keras.datasets import boston_housing 35 | from tensorflow.python.framework import ops 36 | ops.reset_default_graph() 37 | 38 | # For using the boosted trees classifier (binary classification) in TF: 39 | # Note: target labels have to be 0 and 1. 40 | boosted_classifier = tf.estimator.BoostedTreesClassifier 41 | 42 | # For using a boosted trees regression classifier (binary classification) in TF: 43 | regression_classifier = tf.estimator.BoostedTreesRegressor 44 | 45 | # Load data 46 | (x_train, y_train), (x_test, y_test) = boston_housing.load_data() 47 | 48 | # Set model parameters 49 | # Batch size 50 | batch_size = 32 51 | # Number of training steps 52 | train_steps = 500 53 | # Number of trees in our 'forest' 54 | n_trees = 100 55 | # Maximum depth of any tree in forest 56 | max_depth = 6 57 | 58 | # Data ETL 59 | binary_split_cols = ['CHAS', 'RAD'] 60 | col_names = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT'] 61 | X_dtrain = {col: x_train[:, ix] for ix, col in enumerate(col_names)} 62 | X_dtest = {col: x_test[:, ix] for ix, col in enumerate(col_names)} 63 | 64 | # Create feature columns! 65 | feature_cols = [] 66 | for ix, column in enumerate(x_train.T): 67 | col_name = col_names[ix] 68 | 69 | # Create binary split feature 70 | if col_name in binary_split_cols: 71 | # To create 2 buckets, need 1 boundary - the mean 72 | bucket_boundaries = [column.mean()] 73 | numeric_feature = tf.feature_column.numeric_column(col_name) 74 | final_feature = tf.feature_column.bucketized_column(source_column=numeric_feature, boundaries=bucket_boundaries) 75 | # Create bucketed feature 76 | else: 77 | # To create 5 buckets, need 4 boundaries 78 | bucket_boundaries = list(np.linspace(column.min() * 1.1, column.max() * 0.9, 4)) 79 | numeric_feature = tf.feature_column.numeric_column(col_name) 80 | final_feature = tf.feature_column.bucketized_column(source_column=numeric_feature, boundaries=bucket_boundaries) 81 | 82 | # Add feature to feature_col list 83 | feature_cols.append(final_feature) 84 | 85 | 86 | # Create an input function 87 | input_fun = tf.estimator.inputs.numpy_input_fn(X_dtrain, y=y_train, batch_size=batch_size, num_epochs=10, shuffle=True) 88 | 89 | # Training 90 | model = regression_classifier(feature_columns=feature_cols, 91 | n_trees=n_trees, 92 | max_depth=max_depth, 93 | learning_rate=0.25, 94 | n_batches_per_layer=batch_size) 95 | model.train(input_fn=input_fun, steps=train_steps) 96 | 97 | # Evaluation on test set 98 | # Do not shuffle when predicting 99 | p_input_fun = tf.estimator.inputs.numpy_input_fn(X_dtest, y=y_test, batch_size=batch_size, num_epochs=1, shuffle=False) 100 | # Get predictions 101 | predictions = list(model.predict(input_fn=p_input_fun)) 102 | final_preds = [pred['predictions'][0] for pred in predictions] 103 | 104 | # Get accuracy (mean absolute error, MAE) 105 | mae = np.mean([np.abs((actual - predicted) / predicted) for actual, predicted in zip(y_test, final_preds)]) 106 | print('Mean Abs Err on test set: {}'.format(acc)) 107 | -------------------------------------------------------------------------------- /11_More_with_TensorFlow/images/01_tensorboard1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/11_More_with_TensorFlow/images/01_tensorboard1.png -------------------------------------------------------------------------------- /11_More_with_TensorFlow/images/01_tensorboard2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/11_More_with_TensorFlow/images/01_tensorboard2.png -------------------------------------------------------------------------------- /11_More_with_TensorFlow/images/01_tensorboard3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/11_More_with_TensorFlow/images/01_tensorboard3.png -------------------------------------------------------------------------------- /11_More_with_TensorFlow/images/02_genetic_algorithm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/11_More_with_TensorFlow/images/02_genetic_algorithm.png -------------------------------------------------------------------------------- /11_More_with_TensorFlow/images/03_kmeans.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/11_More_with_TensorFlow/images/03_kmeans.png -------------------------------------------------------------------------------- /11_More_with_TensorFlow/images/04_ode_system.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/11_More_with_TensorFlow/images/04_ode_system.png -------------------------------------------------------------------------------- /11_More_with_TensorFlow/readme.md: -------------------------------------------------------------------------------- 1 | ## Ch 11: More with TensorFlow 2 | 3 | 1. [Visualizing Computational Graphs (with Tensorboard)](01_Visualizing_Computational_Graphs#visualizing-computational-graphs-wtensorboard) 4 | * An example of using histograms, scalar summaries, and creating images in Tensorboard. 5 | 2. [Working with a Genetic Algorithm](02_Working_with_a_Genetic_Algorithm#working-with-a-genetic-algorithm) 6 | * We create a genetic algorithm to optimize an individual (array of 50 numbers) toward the ground truth function. 7 | 3. [Clustering Using K-means](03_Clustering_Using_KMeans#clustering-using-k-means) 8 | * How to use TensorFlow to do k-means clustering. We use the Iris data set, set k=3, and use k-means to make predictions. 9 | 4. [Solving a System of ODEs](04_Solving_A_System_of_ODEs#solving-a-system-of-odes) 10 | * Here, we show how to use TensorFlow to solve a system of ODEs. The system of concern is the Lotka-Volterra predator-prey system. 11 | 5. [Using a Random Forest](05_Using_a_Random_Forest#using-a-random-forest) 12 | * We illustrate how to use TensorFlow's gradient boosted regression and classification trees. 13 | 6. [Using TensorFlow with Keras](06_Using_TensorFlow_with_Keras#using-tensorflow-with-keras) 14 | * Here we show how to use the Keras sequential model building for a fully connected neural network and a CNN model with callbacks. -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | ## For the TensorFlow Machine Learning Cookbook 4 | 5 | See [CODE_OF_CONDUCT.md](https://github.com/nfmcclure/tensorflow_cookbook/CODE_OF_CONDUCT.md). 6 | 7 | This code of conduct applies to all of the projects under this [Repository on GitHub](https://github.com/nfmcclure/tensorflow_cookbook) and any related community interactions (Stackoverflow, Google Plus, Email, Comments, Twitter, etc...). 8 | 9 | ## Diversity and Inclusion Statement 10 | 11 | Diversity and inclusion is a vital element to the success of open source and the tech community. Freedom of thought and the open exchange of ideas are key to an effective learning environment and central to the tech community (and all interactions at large). This kind of environment can only happen in a culture that recognizes the value of each person and fosters mutual respect. 12 | 13 | ## Code of Conduct 14 | 15 | Our community is dedicated to creating an inclusive open-source environment for everyone, regardless of race, ethnicity, nationality, religion, skin color, sex, sexual orientation, gender identity, national origin, age, health (physical or mental), genetic information, parental status, marital status, political affiliation, veteran status, socioeconomic status or background, neuro(a)typicality, appearance, body size, computing experience, or clothing. Consider that calling attention to differences can feel alienating. 16 | 17 | Our community does not tolerate harassment in any form towards any person. Harassment includes offensive verbal comments related to the protected classes above, sexual images in public spaces, deliberate intimidation, stalking, following, photography or audio/video recording against reasonable consent, sustained disruption of talks or other events, inappropriate physical contact, and unwelcome sexual attention (even without sexual contact). Harassment does not need to be recognized as unwanted or unwelcome by anyone other than the person being harassed. Be careful in the words that you choose. Remember that sexist, racist, and other exclusionary jokes can be offensive to those around you. Offensive jokes are not appropriate and will not be tolerated under any circumstances. 18 | 19 | Participants asked to stop any harassing behavior are expected to comply immediately. Community members violating this code will be addressed fairly and expediently. Violators may be sanctioned or removed from community spaces (Comments, issues, links, etc... removed). 20 | 21 | If you are being harassed, notice that someone else is being harassed, or have any other concerns, please immediately contact the project owner: `nfmcclure (at) gmail.com `. 22 | 23 | ## Credits 24 | 25 | Based on the [Contributor Covenant](https://github.com/Bantik/contributor_covenant) by [Coraline Ada Ehmke (Bantik)](https://github.com/Bantik). 26 | 27 | If you have suggestions to improve this code of conduct, please submit an issue or PR. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Nick McClure 4 | Copyright (c) 2016 Packt Publishing 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /images/book_covers.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/d998edfe7df11177df2ea17814be7bd6193788c9/images/book_covers.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==2.2.2 2 | six==1.11.0 3 | requests==2.20.0 4 | tensorflow_serving_api==1.9.0 5 | numpy==1.14.5 6 | scipy==1.1.0 7 | tensorflow==1.15.0 8 | grpcio==1.14.1 9 | jupyter_core==4.4.0 10 | Pillow==6.2.0 11 | GitPython==2.1.11 12 | grpc==0.3-19 13 | imageio==2.3.0 14 | jupyter==1.0.0 15 | nltk==3.4.5 16 | skimage==0.0 17 | scikit_learn==0.19.2 18 | -------------------------------------------------------------------------------- /test_script.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import sklearn 5 | import requests 6 | import jupyter 7 | --------------------------------------------------------------------------------