├── .gitattributes ├── .gitignore ├── Chapter 01 ├── activation_functions.py ├── data_gathering.py ├── matrices.py ├── operations.py ├── placeholders.py └── tensors.py ├── Chapter 02 ├── back_propagation.py ├── batch_stochastic_training.py ├── combining_everything_together.py ├── evaluating_models.py ├── layering_nested_operations.py ├── loss_functions.py ├── multiple_layers.py └── operations_on_a_graph.py ├── Chapter 03 ├── deming_regression.py ├── elasticnet_regression.py ├── lasso_and_ridge_regression.py ├── lin_reg_decomposition.py ├── lin_reg_inverse.py ├── lin_reg_l1_vs_l2.py ├── lin_reg_tensorflow_way.py └── logistic_regression.py ├── Chapter 04 ├── linear_svm.py ├── multiclass_svm.py ├── nonlinear_svm.py ├── support_vector_regression.py └── svm_kernels.py ├── Chapter 05 ├── address_matching.py ├── image_recognition.py ├── mixed_distance_functions_knn.py ├── nearest_neighbor.py └── text_distances.py ├── Chapter 06 ├── activation_functions.py ├── gates.py ├── implementing_different_layers.py ├── improving_linear_regression.py ├── single_hidden_layer_network.py ├── tic_tac_toe │ ├── base_tic_tac_toe_moves.csv │ └── tic_tac_toe_moves.py └── using_a_multiple_layer_network.py ├── Chapter 07 ├── bag_of_words.py ├── doc2vec.py ├── text_helpers.py ├── tf_idf.py ├── using_word2vec.py ├── word2vec_cbow.py └── word2vec_skipgram.py ├── Chapter 08 ├── cnn_cifar10.py ├── deepdream.py ├── download_cifar10.py ├── introductory_cnn.py └── stylenet.py ├── Chapter 09 ├── implementing_lstm.py ├── implementing_rnn.py ├── seq2seq_translation.py └── stacking_multiple_lstm.py ├── Chapter 10 ├── implementing_unit_tests.py ├── parallelizing_tensorflow.py ├── production_ex_eval.py ├── production_ex_train.py ├── production_tips_for_tf.py └── using_multiple_devices.py ├── Chapter 11 ├── genetic_algorithm.py ├── k_means.py ├── solving_ode_system.py └── using_tensorboard.py ├── LICENSE └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | .idea 49 | .vscode 50 | -------------------------------------------------------------------------------- /Chapter 01/activation_functions.py: -------------------------------------------------------------------------------- 1 | # Activation Functions 2 | #---------------------------------- 3 | # 4 | # This function introduces activation 5 | # functions in Tensorflow 6 | 7 | # Implementing Activation Functions 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import tensorflow as tf 11 | from tensorflow.python.framework import ops 12 | 13 | ops.reset_default_graph() 14 | 15 | # Open graph session 16 | sess = tf.Session() 17 | 18 | # X range 19 | x_vals = np.linspace(start=-10., stop=10., num=100) 20 | 21 | # ReLU activation 22 | print(sess.run(tf.nn.relu([-3., 3., 10.]))) 23 | y_relu = sess.run(tf.nn.relu(x_vals)) 24 | 25 | # ReLU-6 activation 26 | print(sess.run(tf.nn.relu6([-3., 3., 10.]))) 27 | y_relu6 = sess.run(tf.nn.relu6(x_vals)) 28 | 29 | # Sigmoid activation 30 | print(sess.run(tf.nn.sigmoid([-1., 0., 1.]))) 31 | y_sigmoid = sess.run(tf.nn.sigmoid(x_vals)) 32 | 33 | # Hyper Tangent activation 34 | print(sess.run(tf.nn.tanh([-1., 0., 1.]))) 35 | y_tanh = sess.run(tf.nn.tanh(x_vals)) 36 | 37 | # Softsign activation 38 | print(sess.run(tf.nn.softsign([-1., 0., 1.]))) 39 | y_softsign = sess.run(tf.nn.softsign(x_vals)) 40 | 41 | # Softplus activation 42 | print(sess.run(tf.nn.softplus([-1., 0., 1.]))) 43 | y_softplus = sess.run(tf.nn.softplus(x_vals)) 44 | 45 | # Exponential linear activation 46 | print(sess.run(tf.nn.elu([-1., 0., 1.]))) 47 | y_elu = sess.run(tf.nn.elu(x_vals)) 48 | 49 | # Plot the different functions 50 | plt.plot(x_vals, y_softplus, 'r--', label='Softplus', linewidth=2) 51 | plt.plot(x_vals, y_relu, 'b:', label='ReLU', linewidth=2) 52 | plt.plot(x_vals, y_relu6, 'g-.', label='ReLU6', linewidth=2) 53 | plt.plot(x_vals, y_elu, 'k-', label='ExpLU', linewidth=0.5) 54 | plt.ylim([-1.5,7]) 55 | plt.legend(loc='top left') 56 | plt.show() 57 | 58 | plt.plot(x_vals, y_sigmoid, 'r--', label='Sigmoid', linewidth=2) 59 | plt.plot(x_vals, y_tanh, 'b:', label='Tanh', linewidth=2) 60 | plt.plot(x_vals, y_softsign, 'g-.', label='Softsign', linewidth=2) 61 | plt.ylim([-2,2]) 62 | plt.legend(loc='top left') 63 | plt.show() 64 | -------------------------------------------------------------------------------- /Chapter 01/data_gathering.py: -------------------------------------------------------------------------------- 1 | # Data gathering 2 | # ---------------------------------- 3 | # 4 | # This function gives us the ways to access 5 | # the various data sets we will need 6 | 7 | # Data Gathering 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import tensorflow as tf 11 | from tensorflow.python.framework import ops 12 | 13 | ops.reset_default_graph() 14 | 15 | # Iris Data 16 | from sklearn import datasets 17 | 18 | iris = datasets.load_iris() 19 | print(len(iris.data)) 20 | print(len(iris.target)) 21 | print(iris.data[0]) 22 | print(set(iris.target)) 23 | 24 | # Low Birthrate Data 25 | import requests 26 | 27 | birthdata_url = 'https://www.umass.edu/statdata/statdata/data/lowbwt.dat' 28 | birth_file = requests.get(birthdata_url) 29 | birth_data = birth_file.text.split('\r\n')[5:] 30 | birth_header = [x for x in birth_data[0].split(' ') if len(x) >= 1] 31 | birth_data = [[float(x) for x in y.split(' ') if len(x) >= 1] for y in birth_data[1:] if len(y) >= 1] 32 | print(len(birth_data)) 33 | print(len(birth_data[0])) 34 | 35 | # Housing Price Data 36 | import requests 37 | 38 | housing_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data' 39 | housing_header = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 40 | 'MEDV'] 41 | housing_file = requests.get(housing_url) 42 | housing_data = [[float(x) for x in y.split(' ') if len(x) >= 1] for y in housing_file.text.split('\n') if len(y) >= 1] 43 | print(len(housing_data)) 44 | print(len(housing_data[0])) 45 | 46 | # MNIST Handwriting Data 47 | from tensorflow.examples.tutorials.mnist import input_data 48 | 49 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 50 | print(len(mnist.train.images)) 51 | print(len(mnist.test.images)) 52 | print(len(mnist.validation.images)) 53 | print(mnist.train.labels[1, :]) 54 | 55 | # Ham/Spam Text Data 56 | import requests 57 | import io 58 | from zipfile import ZipFile 59 | 60 | # Get/read zip file 61 | zip_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip' 62 | r = requests.get(zip_url) 63 | z = ZipFile(io.BytesIO(r.content)) 64 | file = z.read('SMSSpamCollection') 65 | # Format Data 66 | text_data = file.decode() 67 | text_data = text_data.encode('ascii', errors='ignore') 68 | text_data = text_data.decode().split('\n') 69 | text_data = [x.split('\t') for x in text_data if len(x) >= 1] 70 | [text_data_target, text_data_train] = [list(x) for x in zip(*text_data)] 71 | print(len(text_data_train)) 72 | print(set(text_data_target)) 73 | print(text_data_train[1]) 74 | 75 | # Movie Review Data 76 | import requests 77 | import io 78 | import tarfile 79 | 80 | movie_data_url = 'http://www.cs.cornell.edu/people/pabo/movie-review-data/rt-polaritydata.tar.gz' 81 | r = requests.get(movie_data_url) 82 | # Stream data into temp object 83 | stream_data = io.BytesIO(r.content) 84 | tmp = io.BytesIO() 85 | while True: 86 | s = stream_data.read(16384) 87 | if not s: 88 | break 89 | tmp.write(s) 90 | stream_data.close() 91 | tmp.seek(0) 92 | # Extract tar file 93 | tar_file = tarfile.open(fileobj=tmp, mode="r:gz") 94 | pos = tar_file.extractfile('rt-polaritydata/rt-polarity.pos') 95 | neg = tar_file.extractfile('rt-polaritydata/rt-polarity.neg') 96 | # Save pos/neg reviews 97 | pos_data = [] 98 | for line in pos: 99 | pos_data.append(line.decode('ISO-8859-1').encode('ascii', errors='ignore').decode()) 100 | neg_data = [] 101 | for line in neg: 102 | neg_data.append(line.decode('ISO-8859-1').encode('ascii', errors='ignore').decode()) 103 | tar_file.close() 104 | 105 | print(len(pos_data)) 106 | print(len(neg_data)) 107 | print(neg_data[0]) 108 | 109 | # The Works of Shakespeare Data 110 | import requests 111 | 112 | shakespeare_url = 'http://www.gutenberg.org/cache/epub/100/pg100.txt' 113 | # Get Shakespeare text 114 | response = requests.get(shakespeare_url) 115 | shakespeare_file = response.content 116 | # Decode binary into string 117 | shakespeare_text = shakespeare_file.decode('utf-8') 118 | # Drop first few descriptive paragraphs. 119 | shakespeare_text = shakespeare_text[7675:] 120 | print(len(shakespeare_text)) 121 | 122 | # English-German Sentence Translation Data 123 | import requests 124 | import io 125 | from zipfile import ZipFile 126 | 127 | sentence_url = 'http://www.manythings.org/anki/deu-eng.zip' 128 | r = requests.get(sentence_url) 129 | z = ZipFile(io.BytesIO(r.content)) 130 | file = z.read('deu.txt') 131 | # Format Data 132 | eng_ger_data = file.decode() 133 | eng_ger_data = eng_ger_data.encode('ascii', errors='ignore') 134 | eng_ger_data = eng_ger_data.decode().split('\n') 135 | eng_ger_data = [x.split('\t') for x in eng_ger_data if len(x) >= 1] 136 | [english_sentence, german_sentence] = [list(x) for x in zip(*eng_ger_data)] 137 | print(len(english_sentence)) 138 | print(len(german_sentence)) 139 | print(eng_ger_data[10]) 140 | -------------------------------------------------------------------------------- /Chapter 01/matrices.py: -------------------------------------------------------------------------------- 1 | # Matrices and Matrix Operations 2 | #---------------------------------- 3 | # 4 | # This function introduces various ways to create 5 | # matrices and how to use them in Tensorflow 6 | 7 | import numpy as np 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # Declaring matrices 13 | sess = tf.Session() 14 | 15 | # Declaring matrices 16 | 17 | # Identity matrix 18 | identity_matrix = tf.diag([1.0,1.0,1.0]) 19 | print('identity_matrix:') 20 | print(sess.run(identity_matrix)) 21 | 22 | # 2x3 random norm matrix 23 | A = tf.truncated_normal([2,3]) 24 | print 'A:' 25 | print(sess.run(A)) 26 | 27 | # 2x3 constant matrix 28 | B = tf.fill([2,3], 5.0) 29 | print 'B:' 30 | print(sess.run(B)) 31 | 32 | # 3x2 random uniform matrix 33 | C = tf.random_uniform([3,2]) 34 | print 'C:' 35 | print(sess.run(C)) 36 | print 'Rerun C:' 37 | print(sess.run(C)) # Note that we are reinitializing, hence the new random variabels 38 | 39 | # Create matrix from np array 40 | D = tf.convert_to_tensor(np.array([[1., 2., 3.], [-3., -7., -1.], [0., 5., -2.]])) 41 | print 'D:' 42 | print(sess.run(D)) 43 | 44 | # Matrix addition/subtraction 45 | print 'A+B:' 46 | print(sess.run(A+B)) 47 | print 'A-B:' 48 | print(sess.run(B-B)) 49 | 50 | # Matrix Multiplication 51 | print(sess.run(tf.matmul(B, identity_matrix))) 52 | 53 | # Matrix Transpose 54 | print(sess.run(tf.transpose(C))) # Again, new random variables 55 | 56 | # Matrix Determinant 57 | print(sess.run(tf.matrix_determinant(D))) 58 | 59 | # Matrix Inverse 60 | print(sess.run(tf.matrix_inverse(D))) 61 | 62 | # Cholesky Decomposition 63 | print(sess.run(tf.cholesky(identity_matrix))) 64 | 65 | # Eigenvalues and Eigenvectors 66 | print 'self_adjoint_eig:' 67 | print(sess.run(tf.self_adjoint_eig(D))) -------------------------------------------------------------------------------- /Chapter 01/operations.py: -------------------------------------------------------------------------------- 1 | # Operations 2 | #---------------------------------- 3 | # 4 | # This function introduces various operations 5 | # in Tensorflow 6 | 7 | # Declaring Operations 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import tensorflow as tf 11 | from tensorflow.python.framework import ops 12 | ops.reset_default_graph() 13 | 14 | # Open graph session 15 | sess = tf.Session() 16 | 17 | # div() vs truediv() vs floordiv() 18 | print(sess.run(tf.div(3,4))) 19 | print(sess.run(tf.truediv(3,4))) 20 | print(sess.run(tf.floordiv(3.0,4.0))) 21 | 22 | # Mod function 23 | print(sess.run(tf.mod(22.0,5.0))) 24 | 25 | # Cross Product 26 | print(sess.run(tf.cross([1.,0.,0.],[0.,1.,0.]))) 27 | 28 | # Trig functions 29 | print(sess.run(tf.sin(3.1416))) 30 | print(sess.run(tf.cos(3.1416))) 31 | # Tangemt 32 | print(sess.run(tf.div(tf.sin(3.1416/4.), tf.cos(3.1416/4.)))) 33 | 34 | # Custom operation 35 | test_nums = range(15) 36 | #from tensorflow.python.ops import math_ops 37 | #print(sess.run(tf.equal(test_num, 3))) 38 | def custom_polynomial(x_val): 39 | # Return 3x^2 - x + 10 40 | return(tf.subtract(3 * tf.square(x_val), x_val) + 10) 41 | 42 | print(sess.run(custom_polynomial(11))) 43 | # What should we get with list comprehension 44 | expected_output = [3*x*x-x+10 for x in test_nums] 45 | print(expected_output) 46 | 47 | # Tensorflow custom function output 48 | for num in test_nums: 49 | print(sess.run(custom_polynomial(num))) -------------------------------------------------------------------------------- /Chapter 01/placeholders.py: -------------------------------------------------------------------------------- 1 | # Placeholders 2 | #---------------------------------- 3 | # 4 | # This function introduces how to 5 | # use placeholders in Tensorflow 6 | 7 | import numpy as np 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # Using Placeholders 13 | sess = tf.Session() 14 | 15 | x = tf.placeholder(tf.float32, shape=(4, 4)) 16 | y = tf.identity(x) 17 | 18 | rand_array = np.random.rand(4, 4) 19 | 20 | merged = tf.summary.merge_all() 21 | writer = tf.summary.FileWriter("/tmp/variable_logs", sess.graph_def) 22 | 23 | print(sess.run(y, feed_dict={x: rand_array})) -------------------------------------------------------------------------------- /Chapter 01/tensors.py: -------------------------------------------------------------------------------- 1 | # Tensors 2 | 3 | import tensorflow as tf 4 | from tensorflow.python.framework import ops 5 | 6 | ops.reset_default_graph() 7 | 8 | # Introduce tensors in tf 9 | 10 | # Get graph handle 11 | sess = tf.Session() 12 | 13 | my_tensor = tf.zeros([1, 20]) 14 | 15 | # Declare a variable 16 | my_var = tf.Variable(tf.zeros([1, 20])) 17 | 18 | # Different kinds of variables 19 | row_dim = 2 20 | col_dim = 3 21 | 22 | # Zero initialized variable 23 | zero_var = tf.Variable(tf.zeros([row_dim, col_dim])) 24 | 25 | # One initialized variable 26 | ones_var = tf.Variable(tf.ones([row_dim, col_dim])) 27 | 28 | # shaped like other variable 29 | sess.run(zero_var.initializer) 30 | sess.run(ones_var.initializer) 31 | zero_similar = tf.Variable(tf.zeros_like(zero_var)) 32 | ones_similar = tf.Variable(tf.ones_like(ones_var)) 33 | 34 | sess.run(ones_similar.initializer) 35 | sess.run(zero_similar.initializer) 36 | 37 | # Fill shape with a constant 38 | fill_var = tf.Variable(tf.fill([row_dim, col_dim], -1)) 39 | 40 | # Create a variable from a constant 41 | const_var = tf.Variable(tf.constant([8, 6, 7, 5, 3, 0, 9])) 42 | # This can also be used to fill an array: 43 | const_fill_var = tf.Variable(tf.constant(-1, shape=[row_dim, col_dim])) 44 | 45 | # Sequence generation 46 | linear_var = tf.Variable(tf.linspace(start=0.0, stop=1.0, num=3)) # Generates [0.0, 0.5, 1.0] includes the end 47 | 48 | sequence_var = tf.Variable(tf.range(start=6, limit=15, delta=3)) # Generates [6, 9, 12] doesn't include the end 49 | 50 | # Random Numbers 51 | 52 | # Random Normal 53 | # rnorm_var = tf.random_normal([row_dim, col_dim], mean=0.0, stddev=1.0) 54 | 55 | # Initialize operation 56 | initialize_op = tf.global_variables_initializer() 57 | 58 | # Add summaries to tensorboard 59 | # merged = tf.merge_all_summaries() 60 | 61 | # Initialize graph writer: 62 | # writer = tf.train.SummaryWriter("/tmp/variable_logs", sess.graph_def) 63 | 64 | # Run initialization of variable 65 | sess.run(initialize_op) 66 | -------------------------------------------------------------------------------- /Chapter 02/back_propagation.py: -------------------------------------------------------------------------------- 1 | # Back Propagation 2 | # ---------------------------------- 3 | # 4 | # This python function shows how to implement back propagation 5 | # in regression and classification models. 6 | 7 | import matplotlib.pyplot as plt 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow.python.framework import ops 11 | 12 | ops.reset_default_graph() 13 | 14 | # Create graph 15 | sess = tf.Session() 16 | 17 | # Regression Example: 18 | # We will create sample data as follows: 19 | # x-data: 100 random samples from a normal ~ N(1, 0.1) 20 | # target: 100 values of the value 10. 21 | # We will fit the model: 22 | # x-data * A = target 23 | # Theoretically, A = 10. 24 | 25 | # Create data 26 | x_vals = np.random.normal(1, 0.1, 100) 27 | y_vals = np.repeat(10., 100) 28 | x_data = tf.placeholder(shape=[1], dtype=tf.float32) 29 | y_target = tf.placeholder(shape=[1], dtype=tf.float32) 30 | 31 | # Create variable (one model parameter = A) 32 | A = tf.Variable(tf.random_normal(shape=[1])) 33 | 34 | # Add operation to graph 35 | my_output = tf.multiply(x_data, A) 36 | 37 | # Add L2 loss operation to graph 38 | loss = tf.square(my_output - y_target) 39 | 40 | # Initialize variables 41 | init = tf.global_variables_initializer() 42 | sess.run(init) 43 | 44 | # Create Optimizer 45 | my_opt = tf.train.GradientDescentOptimizer(0.02) 46 | train_step = my_opt.minimize(loss) 47 | 48 | # Run Loop 49 | for i in range(100): 50 | rand_index = np.random.choice(100) 51 | rand_x = [x_vals[rand_index]] 52 | rand_y = [y_vals[rand_index]] 53 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 54 | if (i + 1) % 25 == 0: 55 | print('Step #' + str(i + 1) + ' A = ' + str(sess.run(A))) 56 | print('Loss = ' + str(sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}))) 57 | 58 | # Classification Example 59 | # We will create sample data as follows: 60 | # x-data: sample 50 random values from a normal = N(-1, 1) 61 | # + sample 50 random values from a normal = N(1, 1) 62 | # target: 50 values of 0 + 50 values of 1. 63 | # These are essentially 100 values of the corresponding output index 64 | # We will fit the binary classification model: 65 | # If sigmoid(x+A) < 0.5 -> 0 else 1 66 | # Theoretically, A should be -(mean1 + mean2)/2 67 | 68 | ops.reset_default_graph() 69 | 70 | # Create graph 71 | sess = tf.Session() 72 | 73 | # Create data 74 | x_vals = np.concatenate((np.random.normal(-1, 1, 50), np.random.normal(3, 1, 50))) 75 | y_vals = np.concatenate((np.repeat(0., 50), np.repeat(1., 50))) 76 | x_data = tf.placeholder(shape=[1], dtype=tf.float32) 77 | y_target = tf.placeholder(shape=[1], dtype=tf.float32) 78 | 79 | # Create variable (one model parameter = A) 80 | A = tf.Variable(tf.random_normal(mean=10, shape=[1])) 81 | 82 | # Add operation to graph 83 | # Want to create the operstion sigmoid(x + A) 84 | # Note, the sigmoid() part is in the loss function 85 | my_output = tf.add(x_data, A) 86 | 87 | # Now we have to add another dimension to each (batch size of 1) 88 | my_output_expanded = tf.expand_dims(my_output, 0) 89 | y_target_expanded = tf.expand_dims(y_target, 0) 90 | 91 | # Initialize variables 92 | init = tf.global_variables_initializer() 93 | sess.run(init) 94 | 95 | # Add classification loss (cross entropy) 96 | xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=my_output_expanded, logits=y_target_expanded) 97 | 98 | # Create Optimizer 99 | my_opt = tf.train.GradientDescentOptimizer(0.05) 100 | train_step = my_opt.minimize(xentropy) 101 | 102 | # Run loop 103 | for i in range(1400): 104 | rand_index = np.random.choice(100) 105 | rand_x = [x_vals[rand_index]] 106 | rand_y = [y_vals[rand_index]] 107 | 108 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 109 | if (i + 1) % 200 == 0: 110 | print('Step #' + str(i + 1) + ' A = ' + str(sess.run(A))) 111 | print('Loss = ' + str(sess.run(xentropy, feed_dict={x_data: rand_x, y_target: rand_y}))) 112 | 113 | # Evaluate Predictions 114 | predictions = [] 115 | for i in range(len(x_vals)): 116 | x_val = [x_vals[i]] 117 | prediction = sess.run(tf.round(tf.sigmoid(my_output)), feed_dict={x_data: x_val}) 118 | predictions.append(prediction[0]) 119 | 120 | accuracy = sum(x == y for x, y in zip(predictions, y_vals)) / 100. 121 | print('Ending Accuracy = ' + str(np.round(accuracy, 2))) 122 | -------------------------------------------------------------------------------- /Chapter 02/batch_stochastic_training.py: -------------------------------------------------------------------------------- 1 | # Batch and Stochastic Training 2 | #---------------------------------- 3 | # 4 | # This python function illustrates two different training methods: 5 | # batch and stochastic training. For each model, we will use 6 | # a regression model that predicts one model variable. 7 | 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import tensorflow as tf 11 | from tensorflow.python.framework import ops 12 | ops.reset_default_graph() 13 | 14 | # We will implement a regression example in stochastic and batch training 15 | 16 | # Stochastic Training: 17 | # Create graph 18 | sess = tf.Session() 19 | 20 | # Create data 21 | x_vals = np.random.normal(1, 0.1, 100) 22 | y_vals = np.repeat(10., 100) 23 | x_data = tf.placeholder(shape=[1], dtype=tf.float32) 24 | y_target = tf.placeholder(shape=[1], dtype=tf.float32) 25 | 26 | # Create variable (one model parameter = A) 27 | A = tf.Variable(tf.random_normal(shape=[1])) 28 | 29 | # Add operation to graph 30 | my_output = tf.multiply(x_data, A) 31 | 32 | # Add L2 loss operation to graph 33 | loss = tf.square(my_output - y_target) 34 | 35 | # Initialize variables 36 | init = tf.global_variables_initializer() 37 | sess.run(init) 38 | 39 | # Create Optimizer 40 | my_opt = tf.train.GradientDescentOptimizer(0.02) 41 | train_step = my_opt.minimize(loss) 42 | 43 | loss_stochastic = [] 44 | # Run Loop 45 | for i in range(100): 46 | rand_index = np.random.choice(100) 47 | rand_x = [x_vals[rand_index]] 48 | rand_y = [y_vals[rand_index]] 49 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 50 | if (i+1)%5==0: 51 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 52 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 53 | print('Loss = ' + str(temp_loss)) 54 | loss_stochastic.append(temp_loss) 55 | 56 | 57 | # Batch Training: 58 | # Re-initialize graph 59 | ops.reset_default_graph() 60 | sess = tf.Session() 61 | 62 | # Declare batch size 63 | batch_size = 20 64 | 65 | # Create data 66 | x_vals = np.random.normal(1, 0.1, 100) 67 | y_vals = np.repeat(10., 100) 68 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 69 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 70 | 71 | # Create variable (one model parameter = A) 72 | A = tf.Variable(tf.random_normal(shape=[1,1])) 73 | 74 | # Add operation to graph 75 | my_output = tf.matmul(x_data, A) 76 | 77 | # Add L2 loss operation to graph 78 | loss = tf.reduce_mean(tf.square(my_output - y_target)) 79 | 80 | # Initialize variables 81 | init = tf.global_variables_initializer() 82 | sess.run(init) 83 | 84 | # Create Optimizer 85 | my_opt = tf.train.GradientDescentOptimizer(0.02) 86 | train_step = my_opt.minimize(loss) 87 | 88 | loss_batch = [] 89 | # Run Loop 90 | for i in range(100): 91 | rand_index = np.random.choice(100, size=batch_size) 92 | rand_x = np.transpose([x_vals[rand_index]]) 93 | rand_y = np.transpose([y_vals[rand_index]]) 94 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 95 | if (i+1)%5==0: 96 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 97 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 98 | print('Loss = ' + str(temp_loss)) 99 | loss_batch.append(temp_loss) 100 | 101 | plt.plot(range(0, 100, 5), loss_stochastic, 'b-', label='Stochastic Loss') 102 | plt.plot(range(0, 100, 5), loss_batch, 'r--', label='Batch Loss, size=20') 103 | plt.legend(loc='upper right', prop={'size': 11}) 104 | plt.show() -------------------------------------------------------------------------------- /Chapter 02/combining_everything_together.py: -------------------------------------------------------------------------------- 1 | # Combining Everything Together 2 | #---------------------------------- 3 | # This file will perform binary classification on the 4 | # class if iris dataset. We will only predict if a flower is 5 | # I.setosa or not. 6 | # 7 | # We will create a simple binary classifier by creating a line 8 | # and running everything through a sigmoid to get a binary predictor. 9 | # The two features we will use are pedal length and pedal width. 10 | # 11 | # We will use batch training, but this can be easily 12 | # adapted to stochastic training. 13 | 14 | import matplotlib.pyplot as plt 15 | import numpy as np 16 | from sklearn import datasets 17 | import tensorflow as tf 18 | from tensorflow.python.framework import ops 19 | ops.reset_default_graph() 20 | 21 | # Load the iris data 22 | # iris.target = {0, 1, 2}, where '0' is setosa 23 | # iris.data ~ [sepal.width, sepal.length, pedal.width, pedal.length] 24 | iris = datasets.load_iris() 25 | binary_target = np.array([1. if x==0 else 0. for x in iris.target]) 26 | iris_2d = np.array([[x[2], x[3]] for x in iris.data]) 27 | 28 | # Declare batch size 29 | batch_size = 20 30 | 31 | # Create graph 32 | sess = tf.Session() 33 | 34 | # Declare placeholders 35 | x1_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 36 | x2_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 37 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 38 | 39 | # Create variables A and b (0 = x1 - A*x2 + b) 40 | A = tf.Variable(tf.random_normal(shape=[1, 1])) 41 | b = tf.Variable(tf.random_normal(shape=[1, 1])) 42 | 43 | # Add model to graph: 44 | # x1 - A*x2 + b 45 | my_mult = tf.matmul(x2_data, A) 46 | my_add = tf.add(my_mult, b) 47 | my_output = tf.subtract(x1_data, my_add) 48 | #my_output = tf.subtract(x_data[0], tf.add(tf.matmul(x_data[1], A), b)) 49 | 50 | # Add classification loss (cross entropy) 51 | xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=my_output, logits=y_target) 52 | 53 | # Create Optimizer 54 | my_opt = tf.train.GradientDescentOptimizer(0.05) 55 | train_step = my_opt.minimize(xentropy) 56 | 57 | # Initialize variables 58 | init = tf.global_variables_initializer() 59 | sess.run(init) 60 | 61 | # Run Loop 62 | for i in range(1000): 63 | rand_index = np.random.choice(len(iris_2d), size=batch_size) 64 | #rand_x = np.transpose([iris_2d[rand_index]]) 65 | rand_x = iris_2d[rand_index] 66 | rand_x1 = np.array([[x[0]] for x in rand_x]) 67 | rand_x2 = np.array([[x[1]] for x in rand_x]) 68 | #rand_y = np.transpose([binary_target[rand_index]]) 69 | rand_y = np.array([[y] for y in binary_target[rand_index]]) 70 | sess.run(train_step, feed_dict={x1_data: rand_x1, x2_data: rand_x2, y_target: rand_y}) 71 | if (i+1)%200==0: 72 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ', b = ' + str(sess.run(b))) 73 | 74 | 75 | # Visualize Results 76 | # Pull out slope/intercept 77 | [[slope]] = sess.run(A) 78 | [[intercept]] = sess.run(b) 79 | 80 | # Create fitted line 81 | x = np.linspace(0, 3, num=50) 82 | ablineValues = [] 83 | for i in x: 84 | ablineValues.append(slope*i+intercept) 85 | 86 | # Plot the fitted line over the data 87 | setosa_x = [a[1] for i,a in enumerate(iris_2d) if binary_target[i]==1] 88 | setosa_y = [a[0] for i,a in enumerate(iris_2d) if binary_target[i]==1] 89 | non_setosa_x = [a[1] for i,a in enumerate(iris_2d) if binary_target[i]==0] 90 | non_setosa_y = [a[0] for i,a in enumerate(iris_2d) if binary_target[i]==0] 91 | plt.plot(setosa_x, setosa_y, 'rx', ms=10, mew=2, label='setosa') 92 | plt.plot(non_setosa_x, non_setosa_y, 'ro', label='Non-setosa') 93 | plt.plot(x, ablineValues, 'b-') 94 | plt.xlim([0.0, 2.7]) 95 | plt.ylim([0.0, 7.1]) 96 | plt.suptitle('Linear Separator For I.setosa', fontsize=20) 97 | plt.xlabel('Petal Length') 98 | plt.ylabel('Petal Width') 99 | plt.legend(loc='lower right') 100 | plt.show() -------------------------------------------------------------------------------- /Chapter 02/evaluating_models.py: -------------------------------------------------------------------------------- 1 | # Evaluating models in Tensorflow 2 | # 3 | # This code will implement two models. The first 4 | # is a simple regression model, we will show how to 5 | # call the loss function, MSE during training, and 6 | # output it after for test and training sets. 7 | # 8 | # The second model will be a simple classification 9 | # model. We will also show how to print percent 10 | # classified correctly during training and after 11 | # for both the test and training sets. 12 | 13 | import matplotlib.pyplot as plt 14 | import numpy as np 15 | import tensorflow as tf 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Create graph 20 | sess = tf.Session() 21 | 22 | # Regression Example: 23 | # We will create sample data as follows: 24 | # x-data: 100 random samples from a normal ~ N(1, 0.1) 25 | # target: 100 values of the value 10. 26 | # We will fit the model: 27 | # x-data * A = target 28 | # Theoretically, A = 10. 29 | 30 | # Declare batch size 31 | batch_size = 25 32 | 33 | # Create data 34 | x_vals = np.random.normal(1, 0.1, 100) 35 | y_vals = np.repeat(10., 100) 36 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 37 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 38 | 39 | # Split data into train/test = 80%/20% 40 | train_indices = np.random.choice(len(x_vals), int(round(len(x_vals)*0.8)), replace=False) 41 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 42 | x_vals_train = x_vals[train_indices] 43 | x_vals_test = x_vals[test_indices] 44 | y_vals_train = y_vals[train_indices] 45 | y_vals_test = y_vals[test_indices] 46 | 47 | # Create variable (one model parameter = A) 48 | A = tf.Variable(tf.random_normal(shape=[1,1])) 49 | 50 | # Add operation to graph 51 | my_output = tf.matmul(x_data, A) 52 | 53 | # Add L2 loss operation to graph 54 | loss = tf.reduce_mean(tf.square(my_output - y_target)) 55 | 56 | # Initialize variables 57 | init = tf.global_variables_initializer() 58 | sess.run(init) 59 | 60 | # Create Optimizer 61 | my_opt = tf.train.GradientDescentOptimizer(0.02) 62 | train_step = my_opt.minimize(loss) 63 | 64 | # Run Loop 65 | for i in range(100): 66 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 67 | rand_x = np.transpose([x_vals_train[rand_index]]) 68 | rand_y = np.transpose([y_vals_train[rand_index]]) 69 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 70 | if (i+1)%25==0: 71 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 72 | print('Loss = ' + str(sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}))) 73 | 74 | # Evaluate accuracy (loss) on test set 75 | mse_test = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_test]), y_target: np.transpose([y_vals_test])}) 76 | mse_train = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_train]), y_target: np.transpose([y_vals_train])}) 77 | print('MSE on test:' + str(np.round(mse_test, 2))) 78 | print('MSE on train:' + str(np.round(mse_train, 2))) 79 | 80 | # Classification Example 81 | # We will create sample data as follows: 82 | # x-data: sample 50 random values from a normal = N(-1, 1) 83 | # + sample 50 random values from a normal = N(1, 1) 84 | # target: 50 values of 0 + 50 values of 1. 85 | # These are essentially 100 values of the corresponding output index 86 | # We will fit the binary classification model: 87 | # If sigmoid(x+A) < 0.5 -> 0 else 1 88 | # Theoretically, A should be -(mean1 + mean2)/2 89 | 90 | ops.reset_default_graph() 91 | 92 | # Create graph 93 | sess = tf.Session() 94 | 95 | # Declare batch size 96 | batch_size = 25 97 | 98 | # Create data 99 | x_vals = np.concatenate((np.random.normal(-1, 1, 50), np.random.normal(2, 1, 50))) 100 | y_vals = np.concatenate((np.repeat(0., 50), np.repeat(1., 50))) 101 | x_data = tf.placeholder(shape=[1, None], dtype=tf.float32) 102 | y_target = tf.placeholder(shape=[1, None], dtype=tf.float32) 103 | 104 | # Split data into train/test = 80%/20% 105 | train_indices = np.random.choice(len(x_vals), int(round(len(x_vals)*0.8)), replace=False) 106 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 107 | x_vals_train = x_vals[train_indices] 108 | x_vals_test = x_vals[test_indices] 109 | y_vals_train = y_vals[train_indices] 110 | y_vals_test = y_vals[test_indices] 111 | 112 | # Create variable (one model parameter = A) 113 | A = tf.Variable(tf.random_normal(mean=10, shape=[1])) 114 | 115 | # Add operation to graph 116 | # Want to create the operstion sigmoid(x + A) 117 | # Note, the sigmoid() part is in the loss function 118 | my_output = tf.add(x_data, A) 119 | 120 | # Initialize variables 121 | init = tf.global_variables_initializer() 122 | sess.run(init) 123 | 124 | # Add classification loss (cross entropy) 125 | xentropy = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=my_output, logits=y_target)) 126 | 127 | # Create Optimizer 128 | my_opt = tf.train.GradientDescentOptimizer(0.05) 129 | train_step = my_opt.minimize(xentropy) 130 | 131 | # Run loop 132 | for i in range(1800): 133 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 134 | rand_x = [x_vals_train[rand_index]] 135 | rand_y = [y_vals_train[rand_index]] 136 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 137 | if (i+1)%200==0: 138 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A))) 139 | print('Loss = ' + str(sess.run(xentropy, feed_dict={x_data: rand_x, y_target: rand_y}))) 140 | 141 | # Evaluate Predictions on test set 142 | y_prediction = tf.squeeze(tf.round(tf.nn.sigmoid(tf.add(x_data, A)))) 143 | correct_prediction = tf.equal(y_prediction, y_target) 144 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 145 | acc_value_test = sess.run(accuracy, feed_dict={x_data: [x_vals_test], y_target: [y_vals_test]}) 146 | acc_value_train = sess.run(accuracy, feed_dict={x_data: [x_vals_train], y_target: [y_vals_train]}) 147 | print('Accuracy on train set: ' + str(acc_value_train)) 148 | print('Accuracy on test set: ' + str(acc_value_test)) 149 | 150 | # Plot classification result 151 | A_result = -sess.run(A) 152 | bins = np.linspace(-5, 5, 50) 153 | plt.hist(x_vals[0:50], bins, alpha=0.5, label='N(-1,1)', color='white') 154 | plt.hist(x_vals[50:100], bins[0:50], alpha=0.5, label='N(2,1)', color='red') 155 | plt.plot((A_result, A_result), (0, 8), 'k--', linewidth=3, label='A = '+ str(np.round(A_result, 2))) 156 | plt.legend(loc='upper right') 157 | plt.title('Binary Classifier, Accuracy=' + str(np.round(acc_value, 2))) 158 | plt.show() -------------------------------------------------------------------------------- /Chapter 02/layering_nested_operations.py: -------------------------------------------------------------------------------- 1 | # Layering Nested Operations 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.python.framework import ops 6 | ops.reset_default_graph() 7 | 8 | # Create graph 9 | sess = tf.Session() 10 | 11 | # Create tensors 12 | 13 | # Create data to feed in 14 | my_array = np.array([[1., 3., 5., 7., 9.], 15 | [-2., 0., 2., 4., 6.], 16 | [-6., -3., 0., 3., 6.]]) 17 | x_vals = np.array([my_array, my_array + 1]) 18 | x_data = tf.placeholder(tf.float32, shape=(3, 5)) 19 | m1 = tf.constant([[1.],[0.],[-1.],[2.],[4.]]) 20 | m2 = tf.constant([[2.]]) 21 | a1 = tf.constant([[10.]]) 22 | 23 | # 1st Operation Layer = Multiplication 24 | prod1 = tf.matmul(x_data, m1) 25 | 26 | # 2nd Operation Layer = Multiplication 27 | prod2 = tf.matmul(prod1, m2) 28 | 29 | # 3rd Operation Layer = Addition 30 | add1 = tf.add(prod2, a1) 31 | 32 | for x_val in x_vals: 33 | print(sess.run(add1, feed_dict={x_data: x_val})) 34 | 35 | merged = tf.merge_all_summaries() 36 | my_writer = tf.train.SummaryWriter('/home/nick/OneDrive/Documents/tensor_flow_book/Code/2_Tensorflow_Way', sess.graph) -------------------------------------------------------------------------------- /Chapter 02/loss_functions.py: -------------------------------------------------------------------------------- 1 | # Loss Functions 2 | # ---------------------------------- 3 | # 4 | # This python script illustrates the different 5 | # loss functions for regression and classification. 6 | 7 | import matplotlib.pyplot as plt 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | 11 | ops.reset_default_graph() 12 | 13 | # Create graph 14 | sess = tf.Session() 15 | 16 | ###### Numerical Predictions ###### 17 | x_vals = tf.linspace(-1., 1., 500) 18 | target = tf.constant(0.) 19 | 20 | # L2 loss 21 | # L = (pred - actual)^2 22 | l2_y_vals = tf.square(target - x_vals) 23 | l2_y_out = sess.run(l2_y_vals) 24 | 25 | # L1 loss 26 | # L = abs(pred - actual) 27 | l1_y_vals = tf.abs(target - x_vals) 28 | l1_y_out = sess.run(l1_y_vals) 29 | 30 | # Pseudo-Huber loss 31 | # L = delta^2 * (sqrt(1 + ((pred - actual)/delta)^2) - 1) 32 | delta1 = tf.constant(0.25) 33 | phuber1_y_vals = tf.multiply(tf.square(delta1), tf.sqrt(1. + tf.square((target - x_vals) / delta1)) - 1.) 34 | phuber1_y_out = sess.run(phuber1_y_vals) 35 | 36 | delta2 = tf.constant(5.) 37 | phuber2_y_vals = tf.multiply(tf.square(delta2), tf.sqrt(1. + tf.square((target - x_vals) / delta2)) - 1.) 38 | phuber2_y_out = sess.run(phuber2_y_vals) 39 | 40 | # Plot the output: 41 | x_array = sess.run(x_vals) 42 | plt.plot(x_array, l2_y_out, 'b-', label='L2 Loss') 43 | plt.plot(x_array, l1_y_out, 'r--', label='L1 Loss') 44 | plt.plot(x_array, phuber1_y_out, 'k-.', label='P-Huber Loss (0.25)') 45 | plt.plot(x_array, phuber2_y_out, 'g:', label='P-Huber Loss (5.0)') 46 | plt.ylim(-0.2, 0.4) 47 | plt.legend(loc='lower right', prop={'size': 11}) 48 | plt.show() 49 | 50 | ###### Categorical Predictions ###### 51 | x_vals = tf.linspace(-3., 5., 500) 52 | target = tf.constant(1.) 53 | targets = tf.fill([500, ], 1.) 54 | 55 | # Hinge loss 56 | # Use for predicting binary (-1, 1) classes 57 | # L = max(0, 1 - (pred * actual)) 58 | hinge_y_vals = tf.maximum(0., 1. - tf.multiply(target, x_vals)) 59 | hinge_y_out = sess.run(hinge_y_vals) 60 | 61 | # Cross entropy loss 62 | # L = -actual * (log(pred)) - (1-actual)(log(1-pred)) 63 | xentropy_y_vals = - tf.multiply(target, tf.log(x_vals)) - tf.multiply((1. - target), tf.log(1. - x_vals)) 64 | xentropy_y_out = sess.run(xentropy_y_vals) 65 | 66 | # Sigmoid entropy loss 67 | # L = -actual * (log(sigmoid(pred))) - (1-actual)(log(1-sigmoid(pred))) 68 | # or 69 | # L = max(actual, 0) - actual * pred + log(1 + exp(-abs(actual))) 70 | xentropy_sigmoid_y_vals = tf.nn.sigmoid_cross_entropy_with_logits(labels=x_vals, logits=targets) 71 | xentropy_sigmoid_y_out = sess.run(xentropy_sigmoid_y_vals) 72 | 73 | # Weighted (softmax) cross entropy loss 74 | # L = -actual * (log(pred)) * weights - (1-actual)(log(1-pred)) 75 | # or 76 | # L = (1 - pred) * actual + (1 + (weights - 1) * pred) * log(1 + exp(-actual)) 77 | weight = tf.constant(0.5) 78 | xentropy_weighted_y_vals = tf.nn.weighted_cross_entropy_with_logits(x_vals, targets, weight) 79 | xentropy_weighted_y_out = sess.run(xentropy_weighted_y_vals) 80 | 81 | # Plot the output 82 | x_array = sess.run(x_vals) 83 | plt.plot(x_array, hinge_y_out, 'b-', label='Hinge Loss') 84 | plt.plot(x_array, xentropy_y_out, 'r--', label='Cross Entropy Loss') 85 | plt.plot(x_array, xentropy_sigmoid_y_out, 'k-.', label='Cross Entropy Sigmoid Loss') 86 | plt.plot(x_array, xentropy_weighted_y_out, 'g:', label='Weighted Cross Entropy Loss (x0.5)') 87 | plt.ylim(-1.5, 3) 88 | # plt.xlim(-1, 3) 89 | plt.legend(loc='lower right', prop={'size': 11}) 90 | plt.show() 91 | 92 | # Softmax entropy loss 93 | # L = -actual * (log(softmax(pred))) - (1-actual)(log(1-softmax(pred))) 94 | unscaled_logits = tf.constant([[1., -3., 10.]]) 95 | target_dist = tf.constant([[0.1, 0.02, 0.88]]) 96 | softmax_xentropy = tf.nn.softmax_cross_entropy_with_logits(unscaled_logits, target_dist) 97 | print(sess.run(softmax_xentropy)) 98 | 99 | # Sparse entropy loss 100 | # Use when classes and targets have to be mutually exclusive 101 | # L = sum( -actual * log(pred) ) 102 | unscaled_logits = tf.constant([[1., -3., 10.]]) 103 | sparse_target_dist = tf.constant([2]) 104 | sparse_xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(unscaled_logits, sparse_target_dist) 105 | print(sess.run(sparse_xentropy)) 106 | -------------------------------------------------------------------------------- /Chapter 02/multiple_layers.py: -------------------------------------------------------------------------------- 1 | # Layering Nested Operations 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.python.framework import ops 6 | ops.reset_default_graph() 7 | 8 | # Create graph 9 | sess = tf.Session() 10 | 11 | # Create tensors 12 | 13 | # Create a small random 'image' of size 4x4 14 | x_shape = [1, 4, 4, 1] 15 | x_val = np.random.uniform(size=x_shape) 16 | 17 | x_data = tf.placeholder(tf.float32, shape=x_shape) 18 | 19 | # Create a layer that takes a spatial moving window average 20 | # Our window will be 2x2 with a stride of 2 for height and width 21 | # The filter value will be 0.25 because we want the average of the 2x2 window 22 | my_filter = tf.constant(0.25, shape=[2, 2, 1, 1]) 23 | my_strides = [1, 2, 2, 1] 24 | mov_avg_layer= tf.nn.conv2d(x_data, my_filter, my_strides, 25 | padding='SAME', name='Moving_Avg_Window') 26 | 27 | # Define a custom layer which will be sigmoid(Ax+b) where 28 | # x is a 2x2 matrix and A and b are 2x2 matrices 29 | def custom_layer(input_matrix): 30 | input_matrix_sqeezed = tf.squeeze(input_matrix) 31 | A = tf.constant([[1., 2.], [-1., 3.]]) 32 | b = tf.constant(1., shape=[2, 2]) 33 | temp1 = tf.matmul(A, input_matrix_sqeezed) 34 | temp = tf.add(temp1, b) # Ax + b 35 | return(tf.sigmoid(temp)) 36 | 37 | # Add custom layer to graph 38 | with tf.name_scope('Custom_Layer') as scope: 39 | custom_layer1 = custom_layer(mov_avg_layer) 40 | 41 | # The output should be an array that is 2x2, but size (1,2,2,1) 42 | #print(sess.run(mov_avg_layer, feed_dict={x_data: x_val})) 43 | 44 | # After custom operation, size is now 2x2 (squeezed out size 1 dims) 45 | print(sess.run(custom_layer1, feed_dict={x_data: x_val})) 46 | 47 | merged = tf.merge_all_summaries() 48 | my_writer = tf.train.SummaryWriter('/home/nick/OneDrive/Documents/tensor_flow_book/Code/2_Tensorflow_Way', sess.graph) -------------------------------------------------------------------------------- /Chapter 02/operations_on_a_graph.py: -------------------------------------------------------------------------------- 1 | # Operations on a Computational Graph 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.python.framework import ops 6 | ops.reset_default_graph() 7 | 8 | # Create graph 9 | sess = tf.Session() 10 | 11 | # Create tensors 12 | 13 | # Create data to feed in 14 | x_vals = np.array([1., 3., 5., 7., 9.]) 15 | x_data = tf.placeholder(tf.float32) 16 | m = tf.constant(3.) 17 | 18 | # Multiplication 19 | prod = tf.multiply(x_data, m) 20 | for x_val in x_vals: 21 | print(sess.run(prod, feed_dict={x_data: x_val})) 22 | 23 | merged = tf.merge_all_summaries() 24 | my_writer = tf.train.SummaryWriter('/home/nick/OneDrive/Documents/tensor_flow_book/Code/2_Tensorflow_Way', sess.graph) -------------------------------------------------------------------------------- /Chapter 03/deming_regression.py: -------------------------------------------------------------------------------- 1 | # Deming Regression 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve linear Deming regression. 6 | # y = Ax + b 7 | # 8 | # We will use the iris data, specifically: 9 | # y = Sepal Length 10 | # x = Petal Width 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from sklearn import datasets 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Create graph 20 | sess = tf.Session() 21 | 22 | # Load the data 23 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 24 | iris = datasets.load_iris() 25 | x_vals = np.array([x[3] for x in iris.data]) 26 | y_vals = np.array([y[0] for y in iris.data]) 27 | 28 | # Declare batch size 29 | batch_size = 50 30 | 31 | # Initialize placeholders 32 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 33 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 34 | 35 | # Create variables for linear regression 36 | A = tf.Variable(tf.random_normal(shape=[1,1])) 37 | b = tf.Variable(tf.random_normal(shape=[1,1])) 38 | 39 | # Declare model operations 40 | model_output = tf.add(tf.matmul(x_data, A), b) 41 | 42 | # Declare Demming loss function 43 | demming_numerator = tf.abs(tf.subtract(y_target, tf.add(tf.matmul(x_data, A), b))) 44 | demming_denominator = tf.sqrt(tf.add(tf.square(A),1)) 45 | loss = tf.reduce_mean(tf.truediv(demming_numerator, demming_denominator)) 46 | 47 | # Initialize variables 48 | init = tf.global_variables_initializer() 49 | sess.run(init) 50 | 51 | # Declare optimizer 52 | my_opt = tf.train.GradientDescentOptimizer(0.1) 53 | train_step = my_opt.minimize(loss) 54 | 55 | # Training loop 56 | loss_vec = [] 57 | for i in range(250): 58 | rand_index = np.random.choice(len(x_vals), size=batch_size) 59 | rand_x = np.transpose([x_vals[rand_index]]) 60 | rand_y = np.transpose([y_vals[rand_index]]) 61 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 62 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 63 | loss_vec.append(temp_loss) 64 | if (i+1)%50==0: 65 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 66 | print('Loss = ' + str(temp_loss)) 67 | 68 | # Get the optimal coefficients 69 | [slope] = sess.run(A) 70 | [y_intercept] = sess.run(b) 71 | 72 | # Get best fit line 73 | best_fit = [] 74 | for i in x_vals: 75 | best_fit.append(slope*i+y_intercept) 76 | 77 | # Plot the result 78 | plt.plot(x_vals, y_vals, 'o', label='Data Points') 79 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 80 | plt.legend(loc='upper left') 81 | plt.title('Sepal Length vs Pedal Width') 82 | plt.xlabel('Pedal Width') 83 | plt.ylabel('Sepal Length') 84 | plt.show() 85 | 86 | # Plot loss over time 87 | plt.plot(loss_vec, 'k-') 88 | plt.title('L2 Loss per Generation') 89 | plt.xlabel('Generation') 90 | plt.ylabel('L2 Loss') 91 | plt.show() 92 | -------------------------------------------------------------------------------- /Chapter 03/elasticnet_regression.py: -------------------------------------------------------------------------------- 1 | # Elastic Net Regression 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve elastic net regression. 6 | # y = Ax + b 7 | # 8 | # We will use the iris data, specifically: 9 | # y = Sepal Length 10 | # x = Pedal Length, Petal Width, Sepal Width 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from sklearn import datasets 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Create graph 20 | sess = tf.Session() 21 | 22 | # Load the data 23 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 24 | iris = datasets.load_iris() 25 | x_vals = np.array([[x[1], x[2], x[3]] for x in iris.data]) 26 | y_vals = np.array([y[0] for y in iris.data]) 27 | 28 | # Declare batch size 29 | batch_size = 50 30 | 31 | # Initialize placeholders 32 | x_data = tf.placeholder(shape=[None, 3], dtype=tf.float32) 33 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 34 | 35 | # Create variables for linear regression 36 | A = tf.Variable(tf.random_normal(shape=[3,1])) 37 | b = tf.Variable(tf.random_normal(shape=[1,1])) 38 | 39 | # Declare model operations 40 | model_output = tf.add(tf.matmul(x_data, A), b) 41 | 42 | # Declare the elastic net loss function 43 | elastic_param1 = tf.constant(1.) 44 | elastic_param2 = tf.constant(1.) 45 | l1_a_loss = tf.reduce_mean(tf.abs(A)) 46 | l2_a_loss = tf.reduce_mean(tf.square(A)) 47 | e1_term = tf.multiply(elastic_param1, l1_a_loss) 48 | e2_term = tf.multiply(elastic_param2, l2_a_loss) 49 | loss = tf.expand_dims(tf.add(tf.add(tf.reduce_mean(tf.square(y_target - model_output)), e1_term), e2_term), 0) 50 | 51 | # Initialize variables 52 | init = tf.global_variables_initializer() 53 | sess.run(init) 54 | 55 | # Declare optimizer 56 | my_opt = tf.train.GradientDescentOptimizer(0.001) 57 | train_step = my_opt.minimize(loss) 58 | 59 | # Training loop 60 | loss_vec = [] 61 | for i in range(1000): 62 | rand_index = np.random.choice(len(x_vals), size=batch_size) 63 | rand_x = x_vals[rand_index] 64 | rand_y = np.transpose([y_vals[rand_index]]) 65 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 66 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 67 | loss_vec.append(temp_loss[0]) 68 | if (i+1)%250==0: 69 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 70 | print('Loss = ' + str(temp_loss)) 71 | 72 | # Get the optimal coefficients 73 | [[sw_coef], [pl_coef], [pw_ceof]] = sess.run(A) 74 | [y_intercept] = sess.run(b) 75 | 76 | # Plot loss over time 77 | plt.plot(loss_vec, 'k-') 78 | plt.title('Loss per Generation') 79 | plt.xlabel('Generation') 80 | plt.ylabel('Loss') 81 | plt.show() 82 | -------------------------------------------------------------------------------- /Chapter 03/lasso_and_ridge_regression.py: -------------------------------------------------------------------------------- 1 | # Lasso and Ridge Regression 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve lasso or ridge regression. 6 | # y = Ax + b 7 | # 8 | # We will use the iris data, specifically: 9 | # y = Sepal Length 10 | # x = Petal Width 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from sklearn import datasets 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Create graph 20 | sess = tf.Session() 21 | 22 | # Load the data 23 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 24 | iris = datasets.load_iris() 25 | x_vals = np.array([x[3] for x in iris.data]) 26 | y_vals = np.array([y[0] for y in iris.data]) 27 | 28 | # Declare batch size 29 | batch_size = 50 30 | 31 | # Initialize placeholders 32 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 33 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 34 | 35 | # Create variables for linear regression 36 | A = tf.Variable(tf.random_normal(shape=[1,1])) 37 | b = tf.Variable(tf.random_normal(shape=[1,1])) 38 | 39 | # Declare model operations 40 | model_output = tf.add(tf.matmul(x_data, A), b) 41 | 42 | # Declare Lasso loss function 43 | # Lasso Loss = L2_Loss + heavyside_step, 44 | # Where heavyside_step ~ 0 if A < constant, otherwise ~ 99 45 | #lasso_param = tf.constant(0.9) 46 | #heavyside_step = tf.truediv(1., tf.add(1., tf.exp(tf.multiply(-100., tf.subtract(A, lasso_param))))) 47 | #regularization_param = tf.multiply(heavyside_step, 99.) 48 | #loss = tf.add(tf.reduce_mean(tf.square(y_target - model_output)), regularization_param) 49 | 50 | # Declare the Ridge loss function 51 | # Ridge loss = L2_loss + L2 norm of slope 52 | ridge_param = tf.constant(1.) 53 | ridge_loss = tf.reduce_mean(tf.square(A)) 54 | loss = tf.expand_dims(tf.add(tf.reduce_mean(tf.square(y_target - model_output)), tf.multiply(ridge_param, ridge_loss)), 0) 55 | 56 | # Initialize variables 57 | init = tf.global_variables_initializer() 58 | sess.run(init) 59 | 60 | # Declare optimizer 61 | my_opt = tf.train.GradientDescentOptimizer(0.001) 62 | train_step = my_opt.minimize(loss) 63 | 64 | # Training loop 65 | loss_vec = [] 66 | for i in range(1500): 67 | rand_index = np.random.choice(len(x_vals), size=batch_size) 68 | rand_x = np.transpose([x_vals[rand_index]]) 69 | rand_y = np.transpose([y_vals[rand_index]]) 70 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 71 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 72 | loss_vec.append(temp_loss[0]) 73 | if (i+1)%300==0: 74 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 75 | print('Loss = ' + str(temp_loss)) 76 | 77 | # Get the optimal coefficients 78 | [slope] = sess.run(A) 79 | [y_intercept] = sess.run(b) 80 | 81 | # Get best fit line 82 | best_fit = [] 83 | for i in x_vals: 84 | best_fit.append(slope*i+y_intercept) 85 | 86 | # Plot the result 87 | plt.plot(x_vals, y_vals, 'o', label='Data Points') 88 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 89 | plt.legend(loc='upper left') 90 | plt.title('Sepal Length vs Pedal Width') 91 | plt.xlabel('Pedal Width') 92 | plt.ylabel('Sepal Length') 93 | plt.show() 94 | 95 | # Plot loss over time 96 | plt.plot(loss_vec, 'k-') 97 | plt.title('L2 Loss per Generation') 98 | plt.xlabel('Generation') 99 | plt.ylabel('L2 Loss') 100 | plt.show() 101 | -------------------------------------------------------------------------------- /Chapter 03/lin_reg_decomposition.py: -------------------------------------------------------------------------------- 1 | # Linear Regression: Decomposition Method 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve linear regression via the matrix inverse. 6 | # 7 | # Given Ax=b, and a Cholesky decomposition such that 8 | # A = L*L' then we can get solve for x via 9 | # 1) L*y=t(A)*b 10 | # 2) L'*x=y 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | # Create graph 19 | sess = tf.Session() 20 | 21 | # Create the data 22 | x_vals = np.linspace(0, 10, 100) 23 | y_vals = x_vals + np.random.normal(0, 1, 100) 24 | 25 | # Create design matrix 26 | x_vals_column = np.transpose(np.matrix(x_vals)) 27 | ones_column = np.transpose(np.matrix(np.repeat(1, 100))) 28 | A = np.column_stack((x_vals_column, ones_column)) 29 | 30 | # Create b matrix 31 | b = np.transpose(np.matrix(y_vals)) 32 | 33 | # Create tensors 34 | A_tensor = tf.constant(A) 35 | b_tensor = tf.constant(b) 36 | 37 | # Find Cholesky Decomposition 38 | tA_A = tf.matmul(tf.transpose(A_tensor), A_tensor) 39 | L = tf.cholesky(tA_A) 40 | 41 | # Solve L*y=t(A)*b 42 | tA_b = tf.matmul(tf.transpose(A_tensor), b) 43 | sol1 = tf.matrix_solve(L, tA_b) 44 | 45 | # Solve L' * y = sol1 46 | sol2 = tf.matrix_solve(tf.transpose(L), sol1) 47 | 48 | solution_eval = sess.run(sol2) 49 | 50 | # Extract coefficients 51 | slope = solution_eval[0][0] 52 | y_intercept = solution_eval[1][0] 53 | 54 | print('slope: ' + str(slope)) 55 | print('y_intercept: ' + str(y_intercept)) 56 | 57 | # Get best fit line 58 | best_fit = [] 59 | for i in x_vals: 60 | best_fit.append(slope*i+y_intercept) 61 | 62 | # Plot the results 63 | plt.plot(x_vals, y_vals, 'o', label='Data') 64 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 65 | plt.legend(loc='upper left') 66 | plt.show() -------------------------------------------------------------------------------- /Chapter 03/lin_reg_inverse.py: -------------------------------------------------------------------------------- 1 | # Linear Regression: Inverse Matrix Method 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve linear regression via the matrix inverse. 6 | # 7 | # Given Ax=b, solving for x: 8 | # x = (t(A) * A)^(-1) * t(A) * b 9 | # where t(A) is the transpose of A 10 | 11 | import matplotlib.pyplot as plt 12 | import numpy as np 13 | import tensorflow as tf 14 | from tensorflow.python.framework import ops 15 | ops.reset_default_graph() 16 | 17 | # Create graph 18 | sess = tf.Session() 19 | 20 | # Create the data 21 | x_vals = np.linspace(0, 10, 100) 22 | y_vals = x_vals + np.random.normal(0, 1, 100) 23 | 24 | # Create design matrix 25 | x_vals_column = np.transpose(np.matrix(x_vals)) 26 | ones_column = np.transpose(np.matrix(np.repeat(1, 100))) 27 | A = np.column_stack((x_vals_column, ones_column)) 28 | 29 | # Create b matrix 30 | b = np.transpose(np.matrix(y_vals)) 31 | 32 | # Create tensors 33 | A_tensor = tf.constant(A) 34 | b_tensor = tf.constant(b) 35 | 36 | # Matrix inverse solution 37 | tA_A = tf.matmul(tf.transpose(A_tensor), A_tensor) 38 | tA_A_inv = tf.matrix_inverse(tA_A) 39 | product = tf.matmul(tA_A_inv, tf.transpose(A_tensor)) 40 | solution = tf.matmul(product, b_tensor) 41 | 42 | solution_eval = sess.run(solution) 43 | 44 | # Extract coefficients 45 | slope = solution_eval[0][0] 46 | y_intercept = solution_eval[1][0] 47 | 48 | print('slope: ' + str(slope)) 49 | print('y_intercept: ' + str(y_intercept)) 50 | 51 | # Get best fit line 52 | best_fit = [] 53 | for i in x_vals: 54 | best_fit.append(slope*i+y_intercept) 55 | 56 | # Plot the results 57 | plt.plot(x_vals, y_vals, 'o', label='Data') 58 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 59 | plt.legend(loc='upper left') 60 | plt.show() -------------------------------------------------------------------------------- /Chapter 03/lin_reg_l1_vs_l2.py: -------------------------------------------------------------------------------- 1 | # Linear Regression: L1 vs L2 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve linear regression via the matrix inverse. 6 | 7 | import matplotlib.pyplot as plt 8 | import numpy as np 9 | import tensorflow as tf 10 | from sklearn import datasets 11 | from tensorflow.python.framework import ops 12 | ops.reset_default_graph() 13 | 14 | # Create graph 15 | sess = tf.Session() 16 | 17 | # Load the data 18 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 19 | iris = datasets.load_iris() 20 | x_vals = np.array([x[3] for x in iris.data]) 21 | y_vals = np.array([y[0] for y in iris.data]) 22 | 23 | # Declare batch size and number of iterations 24 | batch_size = 25 25 | learning_rate = 0.4 # Will not converge with learning rate at 0.4 26 | iterations = 50 27 | 28 | # Initialize placeholders 29 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 30 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 31 | 32 | # Create variables for linear regression 33 | A = tf.Variable(tf.random_normal(shape=[1,1])) 34 | b = tf.Variable(tf.random_normal(shape=[1,1])) 35 | 36 | # Declare model operations 37 | model_output = tf.add(tf.matmul(x_data, A), b) 38 | 39 | # Declare loss functions 40 | loss_l1 = tf.reduce_mean(tf.abs(y_target - model_output)) 41 | 42 | # Initialize variables 43 | init = tf.global_variables_initializer() 44 | sess.run(init) 45 | 46 | # Declare optimizers 47 | my_opt_l1 = tf.train.GradientDescentOptimizer(learning_rate) 48 | train_step_l1 = my_opt_l1.minimize(loss_l1) 49 | 50 | # Training loop 51 | loss_vec_l1 = [] 52 | for i in range(iterations): 53 | rand_index = np.random.choice(len(x_vals), size=batch_size) 54 | rand_x = np.transpose([x_vals[rand_index]]) 55 | rand_y = np.transpose([y_vals[rand_index]]) 56 | sess.run(train_step_l1, feed_dict={x_data: rand_x, y_target: rand_y}) 57 | temp_loss_l1 = sess.run(loss_l1, feed_dict={x_data: rand_x, y_target: rand_y}) 58 | loss_vec_l1.append(temp_loss_l1) 59 | if (i+1)%25==0: 60 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 61 | 62 | 63 | # L2 Loss 64 | # Reinitialize graph 65 | ops.reset_default_graph() 66 | 67 | # Create graph 68 | sess = tf.Session() 69 | 70 | # Initialize placeholders 71 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 72 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 73 | 74 | # Create variables for linear regression 75 | A = tf.Variable(tf.random_normal(shape=[1,1])) 76 | b = tf.Variable(tf.random_normal(shape=[1,1])) 77 | 78 | # Declare model operations 79 | model_output = tf.add(tf.matmul(x_data, A), b) 80 | 81 | # Declare loss functions 82 | loss_l2 = tf.reduce_mean(tf.square(y_target - model_output)) 83 | 84 | # Initialize variables 85 | init = tf.global_variables_initializer() 86 | sess.run(init) 87 | 88 | # Declare optimizers 89 | my_opt_l2 = tf.train.GradientDescentOptimizer(learning_rate) 90 | train_step_l2 = my_opt_l2.minimize(loss_l2) 91 | 92 | loss_vec_l2 = [] 93 | for i in range(iterations): 94 | rand_index = np.random.choice(len(x_vals), size=batch_size) 95 | rand_x = np.transpose([x_vals[rand_index]]) 96 | rand_y = np.transpose([y_vals[rand_index]]) 97 | sess.run(train_step_l2, feed_dict={x_data: rand_x, y_target: rand_y}) 98 | temp_loss_l2 = sess.run(loss_l2, feed_dict={x_data: rand_x, y_target: rand_y}) 99 | loss_vec_l2.append(temp_loss_l2) 100 | if (i+1)%25==0: 101 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 102 | 103 | 104 | # Plot loss over time 105 | plt.plot(loss_vec_l1, 'k-', label='L1 Loss') 106 | plt.plot(loss_vec_l2, 'r--', label='L2 Loss') 107 | plt.title('L1 and L2 Loss per Generation') 108 | plt.xlabel('Generation') 109 | plt.ylabel('L1 Loss') 110 | plt.legend(loc='upper right') 111 | plt.show() -------------------------------------------------------------------------------- /Chapter 03/lin_reg_tensorflow_way.py: -------------------------------------------------------------------------------- 1 | # Linear Regression: Tensorflow Way 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve linear regression. 6 | # y = Ax + b 7 | # 8 | # We will use the iris data, specifically: 9 | # y = Sepal Length 10 | # x = Petal Width 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from sklearn import datasets 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Create graph 20 | sess = tf.Session() 21 | 22 | # Load the data 23 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 24 | iris = datasets.load_iris() 25 | x_vals = np.array([x[3] for x in iris.data]) 26 | y_vals = np.array([y[0] for y in iris.data]) 27 | 28 | # Declare batch size 29 | batch_size = 25 30 | 31 | # Initialize placeholders 32 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 33 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 34 | 35 | # Create variables for linear regression 36 | A = tf.Variable(tf.random_normal(shape=[1,1])) 37 | b = tf.Variable(tf.random_normal(shape=[1,1])) 38 | 39 | # Declare model operations 40 | model_output = tf.add(tf.matmul(x_data, A), b) 41 | 42 | # Declare loss function (L2 loss) 43 | loss = tf.reduce_mean(tf.square(y_target - model_output)) 44 | 45 | # Initialize variables 46 | init = tf.global_variables_initializer() 47 | sess.run(init) 48 | 49 | # Declare optimizer 50 | my_opt = tf.train.GradientDescentOptimizer(0.05) 51 | train_step = my_opt.minimize(loss) 52 | 53 | # Training loop 54 | loss_vec = [] 55 | for i in range(100): 56 | rand_index = np.random.choice(len(x_vals), size=batch_size) 57 | rand_x = np.transpose([x_vals[rand_index]]) 58 | rand_y = np.transpose([y_vals[rand_index]]) 59 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 60 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 61 | loss_vec.append(temp_loss) 62 | if (i+1)%25==0: 63 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 64 | print('Loss = ' + str(temp_loss)) 65 | 66 | # Get the optimal coefficients 67 | [slope] = sess.run(A) 68 | [y_intercept] = sess.run(b) 69 | 70 | # Get best fit line 71 | best_fit = [] 72 | for i in x_vals: 73 | best_fit.append(slope*i+y_intercept) 74 | 75 | # Plot the result 76 | plt.plot(x_vals, y_vals, 'o', label='Data Points') 77 | plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3) 78 | plt.legend(loc='upper left') 79 | plt.title('Sepal Length vs Pedal Width') 80 | plt.xlabel('Pedal Width') 81 | plt.ylabel('Sepal Length') 82 | plt.show() 83 | 84 | # Plot loss over time 85 | plt.plot(loss_vec, 'k-') 86 | plt.title('L2 Loss per Generation') 87 | plt.xlabel('Generation') 88 | plt.ylabel('L2 Loss') 89 | plt.show() 90 | -------------------------------------------------------------------------------- /Chapter 03/logistic_regression.py: -------------------------------------------------------------------------------- 1 | # Logistic Regression 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve logistic regression. 6 | # y = sigmoid(Ax + b) 7 | # 8 | # We will use the low birth weight data, specifically: 9 | # y = 0 or 1 = low birth weight 10 | # x = demographic and medical history data 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | import requests 16 | from sklearn import datasets 17 | from sklearn.preprocessing import normalize 18 | from tensorflow.python.framework import ops 19 | ops.reset_default_graph() 20 | 21 | # Create graph 22 | sess = tf.Session() 23 | 24 | birthdata_url = 'https://www.umass.edu/statdata/statdata/data/lowbwt.dat' 25 | birth_file = requests.get(birthdata_url) 26 | birth_data = birth_file.text.split('\r\n')[5:] 27 | birth_header = [x for x in birth_data[0].split(' ') if len(x)>=1] 28 | birth_data = [[float(x) for x in y.split(' ') if len(x)>=1] for y in birth_data[1:] if len(y)>=1] 29 | # Pull out target variable 30 | y_vals = np.array([x[1] for x in birth_data]) 31 | # Pull out predictor variables (not id, not target, and not birthweight) 32 | x_vals = np.array([x[2:9] for x in birth_data]) 33 | 34 | # Split data into train/test = 80%/20% 35 | train_indices = np.random.choice(len(x_vals), int(round(len(x_vals)*0.8)), replace=False) 36 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 37 | x_vals_train = x_vals[train_indices] 38 | x_vals_test = x_vals[test_indices] 39 | y_vals_train = y_vals[train_indices] 40 | y_vals_test = y_vals[test_indices] 41 | 42 | # Normalize by column (min-max norm) 43 | def normalize_cols(m): 44 | col_max = m.max(axis=0) 45 | col_min = m.min(axis=0) 46 | return (m-col_min) / (col_max - col_min) 47 | 48 | x_vals_train = np.nan_to_num(normalize_cols(x_vals_train)) 49 | x_vals_test = np.nan_to_num(normalize_cols(x_vals_test)) 50 | 51 | # Declare batch size 52 | batch_size = 25 53 | 54 | # Initialize placeholders 55 | x_data = tf.placeholder(shape=[None, 7], dtype=tf.float32) 56 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 57 | 58 | # Create variables for linear regression 59 | A = tf.Variable(tf.random_normal(shape=[7,1])) 60 | b = tf.Variable(tf.random_normal(shape=[1,1])) 61 | 62 | # Declare model operations 63 | model_output = tf.add(tf.matmul(x_data, A), b) 64 | 65 | # Declare loss function (Cross Entropy loss) 66 | loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=model_output, logits=y_target)) 67 | 68 | # Initialize variables 69 | init = tf.global_variables_initializer() 70 | sess.run(init) 71 | 72 | # Declare optimizer 73 | my_opt = tf.train.GradientDescentOptimizer(0.01) 74 | train_step = my_opt.minimize(loss) 75 | 76 | # Actual Prediction 77 | prediction = tf.round(tf.sigmoid(model_output)) 78 | predictions_correct = tf.cast(tf.equal(prediction, y_target), tf.float32) 79 | accuracy = tf.reduce_mean(predictions_correct) 80 | 81 | # Training loop 82 | loss_vec = [] 83 | train_acc = [] 84 | test_acc = [] 85 | for i in range(1500): 86 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 87 | rand_x = x_vals_train[rand_index] 88 | rand_y = np.transpose([y_vals_train[rand_index]]) 89 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 90 | 91 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 92 | loss_vec.append(temp_loss) 93 | temp_acc_train = sess.run(accuracy, feed_dict={x_data: x_vals_train, y_target: np.transpose([y_vals_train])}) 94 | train_acc.append(temp_acc_train) 95 | temp_acc_test = sess.run(accuracy, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])}) 96 | test_acc.append(temp_acc_test) 97 | if (i+1)%300==0: 98 | print('Loss = ' + str(temp_loss)) 99 | 100 | # Plot loss over time 101 | plt.plot(loss_vec, 'k-') 102 | plt.title('Cross Entropy Loss per Generation') 103 | plt.xlabel('Generation') 104 | plt.ylabel('Cross Entropy Loss') 105 | plt.show() 106 | 107 | # Plot train and test accuracy 108 | plt.plot(train_acc, 'k-', label='Train Set Accuracy') 109 | plt.plot(test_acc, 'r--', label='Test Set Accuracy') 110 | plt.title('Train and Test Accuracy') 111 | plt.xlabel('Generation') 112 | plt.ylabel('Accuracy') 113 | plt.legend(loc='lower right') 114 | plt.show() -------------------------------------------------------------------------------- /Chapter 04/linear_svm.py: -------------------------------------------------------------------------------- 1 | # Linear Support Vector Machine: Soft Margin 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # create a soft margin SVM 6 | # 7 | # We will use the iris data, specifically: 8 | # x1 = Sepal Length 9 | # x2 = Petal Width 10 | # Class 1 : I. setosa 11 | # Class -1: not I. setosa 12 | # 13 | # We know here that x and y are linearly seperable 14 | # for I. setosa classification. 15 | 16 | import matplotlib.pyplot as plt 17 | import numpy as np 18 | import tensorflow as tf 19 | from sklearn import datasets 20 | from tensorflow.python.framework import ops 21 | ops.reset_default_graph() 22 | 23 | # Create graph 24 | sess = tf.Session() 25 | 26 | # Load the data 27 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 28 | iris = datasets.load_iris() 29 | x_vals = np.array([[x[0], x[3]] for x in iris.data]) 30 | y_vals = np.array([1 if y==0 else -1 for y in iris.target]) 31 | 32 | # Split data into train/test sets 33 | train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 34 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 35 | x_vals_train = x_vals[train_indices] 36 | x_vals_test = x_vals[test_indices] 37 | y_vals_train = y_vals[train_indices] 38 | y_vals_test = y_vals[test_indices] 39 | 40 | # Declare batch size 41 | batch_size = 100 42 | 43 | # Initialize placeholders 44 | x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32) 45 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 46 | 47 | # Create variables for linear regression 48 | A = tf.Variable(tf.random_normal(shape=[2,1])) 49 | b = tf.Variable(tf.random_normal(shape=[1,1])) 50 | 51 | # Declare model operations 52 | model_output = tf.subtract(tf.matmul(x_data, A), b) 53 | 54 | # Declare vector L2 'norm' function squared 55 | l2_norm = tf.reduce_sum(tf.square(A)) 56 | 57 | # Declare loss function 58 | # = max(0, 1-pred*actual) + alpha * L2_norm(A)^2 59 | # L2 regularization parameter, alpha 60 | alpha = tf.constant([0.01]) 61 | # Margin term in loss 62 | classification_term = tf.reduce_mean(tf.maximum(0., tf.subtract(1., tf.multiply(model_output, y_target)))) 63 | # Put terms together 64 | loss = tf.add(classification_term, tf.multiply(alpha, l2_norm)) 65 | 66 | # Declare prediction function 67 | prediction = tf.sign(model_output) 68 | accuracy = tf.reduce_mean(tf.cast(tf.equal(prediction, y_target), tf.float32)) 69 | 70 | # Declare optimizer 71 | my_opt = tf.train.GradientDescentOptimizer(0.01) 72 | train_step = my_opt.minimize(loss) 73 | 74 | # Initialize variables 75 | init = tf.global_variables_initializer() 76 | sess.run(init) 77 | 78 | # Training loop 79 | loss_vec = [] 80 | train_accuracy = [] 81 | test_accuracy = [] 82 | for i in range(500): 83 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 84 | rand_x = x_vals_train[rand_index] 85 | rand_y = np.transpose([y_vals_train[rand_index]]) 86 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 87 | 88 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 89 | loss_vec.append(temp_loss) 90 | 91 | train_acc_temp = sess.run(accuracy, feed_dict={x_data: x_vals_train, y_target: np.transpose([y_vals_train])}) 92 | train_accuracy.append(train_acc_temp) 93 | 94 | test_acc_temp = sess.run(accuracy, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])}) 95 | test_accuracy.append(test_acc_temp) 96 | 97 | if (i+1)%100==0: 98 | print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 99 | print('Loss = ' + str(temp_loss)) 100 | 101 | # Extract coefficients 102 | [[a1], [a2]] = sess.run(A) 103 | [[b]] = sess.run(b) 104 | slope = -a2/a1 105 | y_intercept = b/a1 106 | 107 | # Extract x1 and x2 vals 108 | x1_vals = [d[1] for d in x_vals] 109 | 110 | # Get best fit line 111 | best_fit = [] 112 | for i in x1_vals: 113 | best_fit.append(slope*i+y_intercept) 114 | 115 | # Separate I. setosa 116 | setosa_x = [d[1] for i,d in enumerate(x_vals) if y_vals[i]==1] 117 | setosa_y = [d[0] for i,d in enumerate(x_vals) if y_vals[i]==1] 118 | not_setosa_x = [d[1] for i,d in enumerate(x_vals) if y_vals[i]==-1] 119 | not_setosa_y = [d[0] for i,d in enumerate(x_vals) if y_vals[i]==-1] 120 | 121 | # Plot data and line 122 | plt.plot(setosa_x, setosa_y, 'o', label='I. setosa') 123 | plt.plot(not_setosa_x, not_setosa_y, 'x', label='Non-setosa') 124 | plt.plot(x1_vals, best_fit, 'r-', label='Linear Separator', linewidth=3) 125 | plt.ylim([0, 10]) 126 | plt.legend(loc='lower right') 127 | plt.title('Sepal Length vs Pedal Width') 128 | plt.xlabel('Pedal Width') 129 | plt.ylabel('Sepal Length') 130 | plt.show() 131 | 132 | # Plot train/test accuracies 133 | plt.plot(train_accuracy, 'k-', label='Training Accuracy') 134 | plt.plot(test_accuracy, 'r--', label='Test Accuracy') 135 | plt.title('Train and Test Set Accuracies') 136 | plt.xlabel('Generation') 137 | plt.ylabel('Accuracy') 138 | plt.legend(loc='lower right') 139 | plt.show() 140 | 141 | # Plot loss over time 142 | plt.plot(loss_vec, 'k-') 143 | plt.title('Loss per Generation') 144 | plt.xlabel('Generation') 145 | plt.ylabel('Loss') 146 | plt.show() -------------------------------------------------------------------------------- /Chapter 04/multiclass_svm.py: -------------------------------------------------------------------------------- 1 | # Multi-class (Nonlinear) SVM Example 2 | # ---------------------------------- 3 | # 4 | # This function wll illustrate how to 5 | # implement the gaussian kernel with 6 | # multiple classes on the iris dataset. 7 | # 8 | # Gaussian Kernel: 9 | # K(x1, x2) = exp(-gamma * abs(x1 - x2)^2) 10 | # 11 | # X : (Sepal Length, Petal Width) 12 | # Y: (I. setosa, I. virginica, I. versicolor) (3 classes) 13 | # 14 | # Basic idea: introduce an extra dimension to do 15 | # one vs all classification. 16 | # 17 | # The prediction of a point will be the category with 18 | # the largest margin or distance to boundary. 19 | 20 | import matplotlib.pyplot as plt 21 | import numpy as np 22 | import tensorflow as tf 23 | from sklearn import datasets 24 | from tensorflow.python.framework import ops 25 | 26 | ops.reset_default_graph() 27 | 28 | # Create graph 29 | sess = tf.Session() 30 | 31 | # Load the data 32 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 33 | iris = datasets.load_iris() 34 | x_vals = np.array([[x[0], x[3]] for x in iris.data]) 35 | y_vals1 = np.array([1 if y == 0 else -1 for y in iris.target]) 36 | y_vals2 = np.array([1 if y == 1 else -1 for y in iris.target]) 37 | y_vals3 = np.array([1 if y == 2 else -1 for y in iris.target]) 38 | y_vals = np.array([y_vals1, y_vals2, y_vals3]) 39 | class1_x = [x[0] for i, x in enumerate(x_vals) if iris.target[i] == 0] 40 | class1_y = [x[1] for i, x in enumerate(x_vals) if iris.target[i] == 0] 41 | class2_x = [x[0] for i, x in enumerate(x_vals) if iris.target[i] == 1] 42 | class2_y = [x[1] for i, x in enumerate(x_vals) if iris.target[i] == 1] 43 | class3_x = [x[0] for i, x in enumerate(x_vals) if iris.target[i] == 2] 44 | class3_y = [x[1] for i, x in enumerate(x_vals) if iris.target[i] == 2] 45 | 46 | # Declare batch size 47 | batch_size = 50 48 | 49 | # Initialize placeholders 50 | x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32) 51 | y_target = tf.placeholder(shape=[3, None], dtype=tf.float32) 52 | prediction_grid = tf.placeholder(shape=[None, 2], dtype=tf.float32) 53 | 54 | # Create variables for svm 55 | b = tf.Variable(tf.random_normal(shape=[3, batch_size])) 56 | 57 | # Gaussian (RBF) kernel 58 | gamma = tf.constant(-10.0) 59 | dist = tf.reduce_sum(tf.square(x_data), 1) 60 | dist = tf.reshape(dist, [-1, 1]) 61 | sq_dists = tf.add(tf.subtract(dist, tf.multiply(2., tf.matmul(x_data, tf.transpose(x_data)))), tf.transpose(dist)) 62 | my_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists))) 63 | 64 | 65 | # Declare function to do reshape/batch multiplication 66 | def reshape_matmul(mat): 67 | v1 = tf.expand_dims(mat, 1) 68 | v2 = tf.reshape(v1, [3, batch_size, 1]) 69 | return (tf.matmul(v2, v1)) 70 | 71 | 72 | # Compute SVM Model 73 | model_output = tf.matmul(b, my_kernel) 74 | first_term = tf.reduce_sum(b) 75 | b_vec_cross = tf.matmul(tf.transpose(b), b) 76 | y_target_cross = reshape_matmul(y_target) 77 | 78 | second_term = tf.reduce_sum(tf.multiply(my_kernel, tf.multiply(b_vec_cross, y_target_cross)), [1, 2]) 79 | loss = tf.reduce_sum(tf.negative(tf.subtract(first_term, second_term))) 80 | 81 | # Gaussian (RBF) prediction kernel 82 | rA = tf.reshape(tf.reduce_sum(tf.square(x_data), 1), [-1, 1]) 83 | rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1), [-1, 1]) 84 | pred_sq_dist = tf.add(tf.subtract(rA, tf.multiply(2., tf.matmul(x_data, tf.transpose(prediction_grid)))), 85 | tf.transpose(rB)) 86 | pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist))) 87 | 88 | prediction_output = tf.matmul(tf.multiply(y_target, b), pred_kernel) 89 | prediction = tf.argmax(prediction_output - tf.expand_dims(tf.reduce_mean(prediction_output, 1), 1), 0) 90 | accuracy = tf.reduce_mean(tf.cast(tf.equal(prediction, tf.argmax(y_target, 0)), tf.float32)) 91 | 92 | # Declare optimizer 93 | my_opt = tf.train.GradientDescentOptimizer(0.01) 94 | train_step = my_opt.minimize(loss) 95 | 96 | # Initialize variables 97 | init = tf.global_variables_initializer() 98 | sess.run(init) 99 | 100 | # Training loop 101 | loss_vec = [] 102 | batch_accuracy = [] 103 | for i in range(100): 104 | rand_index = np.random.choice(len(x_vals), size=batch_size) 105 | rand_x = x_vals[rand_index] 106 | rand_y = y_vals[:, rand_index] 107 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 108 | 109 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 110 | loss_vec.append(temp_loss) 111 | 112 | acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x, 113 | y_target: rand_y, 114 | prediction_grid: rand_x}) 115 | batch_accuracy.append(acc_temp) 116 | 117 | if (i + 1) % 25 == 0: 118 | print('Step #' + str(i + 1)) 119 | print('Loss = ' + str(temp_loss)) 120 | 121 | # Create a mesh to plot points in 122 | x_min, x_max = x_vals[:, 0].min() - 1, x_vals[:, 0].max() + 1 123 | y_min, y_max = x_vals[:, 1].min() - 1, x_vals[:, 1].max() + 1 124 | xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02), 125 | np.arange(y_min, y_max, 0.02)) 126 | grid_points = np.c_[xx.ravel(), yy.ravel()] 127 | grid_predictions = sess.run(prediction, feed_dict={x_data: rand_x, 128 | y_target: rand_y, 129 | prediction_grid: grid_points}) 130 | grid_predictions = grid_predictions.reshape(xx.shape) 131 | 132 | # Plot points and grid 133 | plt.contourf(xx, yy, grid_predictions, cmap=plt.cm.Paired, alpha=0.8) 134 | plt.plot(class1_x, class1_y, 'ro', label='I. setosa') 135 | plt.plot(class2_x, class2_y, 'kx', label='I. versicolor') 136 | plt.plot(class3_x, class3_y, 'gv', label='I. virginica') 137 | plt.title('Gaussian SVM Results on Iris Data') 138 | plt.xlabel('Pedal Length') 139 | plt.ylabel('Sepal Width') 140 | plt.legend(loc='lower right') 141 | plt.ylim([-0.5, 3.0]) 142 | plt.xlim([3.5, 8.5]) 143 | plt.show() 144 | 145 | # Plot batch accuracy 146 | plt.plot(batch_accuracy, 'k-', label='Accuracy') 147 | plt.title('Batch Accuracy') 148 | plt.xlabel('Generation') 149 | plt.ylabel('Accuracy') 150 | plt.legend(loc='lower right') 151 | plt.show() 152 | 153 | # Plot loss over time 154 | plt.plot(loss_vec, 'k-') 155 | plt.title('Loss per Generation') 156 | plt.xlabel('Generation') 157 | plt.ylabel('Loss') 158 | plt.show() 159 | -------------------------------------------------------------------------------- /Chapter 04/nonlinear_svm.py: -------------------------------------------------------------------------------- 1 | # Nonlinear SVM Example 2 | #---------------------------------- 3 | # 4 | # This function wll illustrate how to 5 | # implement the gaussian kernel on 6 | # the iris dataset. 7 | # 8 | # Gaussian Kernel: 9 | # K(x1, x2) = exp(-gamma * abs(x1 - x2)^2) 10 | 11 | import matplotlib.pyplot as plt 12 | import numpy as np 13 | import tensorflow as tf 14 | from sklearn import datasets 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | # Create graph 19 | sess = tf.Session() 20 | 21 | # Load the data 22 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 23 | iris = datasets.load_iris() 24 | x_vals = np.array([[x[0], x[3]] for x in iris.data]) 25 | y_vals = np.array([1 if y==0 else -1 for y in iris.target]) 26 | class1_x = [x[0] for i,x in enumerate(x_vals) if y_vals[i]==1] 27 | class1_y = [x[1] for i,x in enumerate(x_vals) if y_vals[i]==1] 28 | class2_x = [x[0] for i,x in enumerate(x_vals) if y_vals[i]==-1] 29 | class2_y = [x[1] for i,x in enumerate(x_vals) if y_vals[i]==-1] 30 | 31 | # Declare batch size 32 | batch_size = 150 33 | 34 | # Initialize placeholders 35 | x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32) 36 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 37 | prediction_grid = tf.placeholder(shape=[None, 2], dtype=tf.float32) 38 | 39 | # Create variables for svm 40 | b = tf.Variable(tf.random_normal(shape=[1,batch_size])) 41 | 42 | # Gaussian (RBF) kernel 43 | gamma = tf.constant(-25.0) 44 | dist = tf.reduce_sum(tf.square(x_data), 1) 45 | dist = tf.reshape(dist, [-1,1]) 46 | sq_dists = tf.add(tf.subtract(dist, tf.multiply(2., tf.matmul(x_data, tf.transpose(x_data)))), tf.transpose(dist)) 47 | my_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists))) 48 | 49 | # Compute SVM Model 50 | model_output = tf.matmul(b, my_kernel) 51 | first_term = tf.reduce_sum(b) 52 | b_vec_cross = tf.matmul(tf.transpose(b), b) 53 | y_target_cross = tf.matmul(y_target, tf.transpose(y_target)) 54 | second_term = tf.reduce_sum(tf.multiply(my_kernel, tf.multiply(b_vec_cross, y_target_cross))) 55 | loss = tf.negative(tf.subtract(first_term, second_term)) 56 | 57 | # Gaussian (RBF) prediction kernel 58 | rA = tf.reshape(tf.reduce_sum(tf.square(x_data), 1),[-1,1]) 59 | rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1),[-1,1]) 60 | pred_sq_dist = tf.add(tf.subtract(rA, tf.multiply(2., tf.matmul(x_data, tf.transpose(prediction_grid)))), tf.transpose(rB)) 61 | pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist))) 62 | 63 | prediction_output = tf.matmul(tf.multiply(tf.transpose(y_target),b), pred_kernel) 64 | prediction = tf.sign(prediction_output-tf.reduce_mean(prediction_output)) 65 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.squeeze(prediction), tf.squeeze(y_target)), tf.float32)) 66 | 67 | # Declare optimizer 68 | my_opt = tf.train.GradientDescentOptimizer(0.01) 69 | train_step = my_opt.minimize(loss) 70 | 71 | # Initialize variables 72 | init = tf.global_variables_initializer() 73 | sess.run(init) 74 | 75 | # Training loop 76 | loss_vec = [] 77 | batch_accuracy = [] 78 | for i in range(300): 79 | rand_index = np.random.choice(len(x_vals), size=batch_size) 80 | rand_x = x_vals[rand_index] 81 | rand_y = np.transpose([y_vals[rand_index]]) 82 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 83 | 84 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 85 | loss_vec.append(temp_loss) 86 | 87 | acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x, 88 | y_target: rand_y, 89 | prediction_grid:rand_x}) 90 | batch_accuracy.append(acc_temp) 91 | 92 | if (i+1)%75==0: 93 | print('Step #' + str(i+1)) 94 | print('Loss = ' + str(temp_loss)) 95 | 96 | # Create a mesh to plot points in 97 | x_min, x_max = x_vals[:, 0].min() - 1, x_vals[:, 0].max() + 1 98 | y_min, y_max = x_vals[:, 1].min() - 1, x_vals[:, 1].max() + 1 99 | xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02), 100 | np.arange(y_min, y_max, 0.02)) 101 | grid_points = np.c_[xx.ravel(), yy.ravel()] 102 | [grid_predictions] = sess.run(prediction, feed_dict={x_data: rand_x, 103 | y_target: rand_y, 104 | prediction_grid: grid_points}) 105 | grid_predictions = grid_predictions.reshape(xx.shape) 106 | 107 | # Plot points and grid 108 | plt.contourf(xx, yy, grid_predictions, cmap=plt.cm.Paired, alpha=0.8) 109 | plt.plot(class1_x, class1_y, 'ro', label='I. setosa') 110 | plt.plot(class2_x, class2_y, 'kx', label='Non setosa') 111 | plt.title('Gaussian SVM Results on Iris Data') 112 | plt.xlabel('Pedal Length') 113 | plt.ylabel('Sepal Width') 114 | plt.legend(loc='lower right') 115 | plt.ylim([-0.5, 3.0]) 116 | plt.xlim([3.5, 8.5]) 117 | plt.show() 118 | 119 | # Plot batch accuracy 120 | plt.plot(batch_accuracy, 'k-', label='Accuracy') 121 | plt.title('Batch Accuracy') 122 | plt.xlabel('Generation') 123 | plt.ylabel('Accuracy') 124 | plt.legend(loc='lower right') 125 | plt.show() 126 | 127 | # Plot loss over time 128 | plt.plot(loss_vec, 'k-') 129 | plt.title('Loss per Generation') 130 | plt.xlabel('Generation') 131 | plt.ylabel('Loss') 132 | plt.show() 133 | 134 | # sess.run(prediction_output, feed_dict={x_data: rand_x, y_target: rand_y, prediction_grid: grid_points}) 135 | # sess.run(pred_kernel, feed_dict={x_data: rand_x, y_target: rand_y, prediction_grid: grid_points}) 136 | # sess.run(model_output, feed_dict={x_data:rand_x, y_target: rand_y}) 137 | # sess.run(second_term, feed_dict={x_data:rand_x, y_target: rand_y}) -------------------------------------------------------------------------------- /Chapter 04/support_vector_regression.py: -------------------------------------------------------------------------------- 1 | # SVM Regression 2 | #---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve support vector regression. We are going 6 | # to find the line that has the maximum margin 7 | # which INCLUDES as many points as possible 8 | # 9 | # We will use the iris data, specifically: 10 | # y = Sepal Length 11 | # x = Pedal Width 12 | 13 | import matplotlib.pyplot as plt 14 | import numpy as np 15 | import tensorflow as tf 16 | from sklearn import datasets 17 | from tensorflow.python.framework import ops 18 | ops.reset_default_graph() 19 | 20 | # Create graph 21 | sess = tf.Session() 22 | 23 | # Load the data 24 | # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)] 25 | iris = datasets.load_iris() 26 | x_vals = np.array([x[3] for x in iris.data]) 27 | y_vals = np.array([y[0] for y in iris.data]) 28 | 29 | # Split data into train/test sets 30 | train_indices = np.random.choice(len(x_vals), int(round(len(x_vals)*0.8)), replace=False) 31 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 32 | x_vals_train = x_vals[train_indices] 33 | x_vals_test = x_vals[test_indices] 34 | y_vals_train = y_vals[train_indices] 35 | y_vals_test = y_vals[test_indices] 36 | 37 | # Declare batch size 38 | batch_size = 50 39 | 40 | # Initialize placeholders 41 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 42 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 43 | 44 | # Create variables for linear regression 45 | A = tf.Variable(tf.random_normal(shape=[1,1])) 46 | b = tf.Variable(tf.random_normal(shape=[1,1])) 47 | 48 | # Declare model operations 49 | model_output = tf.add(tf.matmul(x_data, A), b) 50 | 51 | # Declare loss function 52 | # = max(0, abs(target - predicted) + epsilon) 53 | # 1/2 margin width parameter = epsilon 54 | epsilon = tf.constant([0.5]) 55 | # Margin term in loss 56 | loss = tf.reduce_mean(tf.maximum(0., tf.subtract(tf.abs(tf.subtract(model_output, y_target)), epsilon))) 57 | 58 | # Declare optimizer 59 | my_opt = tf.train.GradientDescentOptimizer(0.075) 60 | train_step = my_opt.minimize(loss) 61 | 62 | # Initialize variables 63 | init = tf.global_variables_initializer() 64 | sess.run(init) 65 | 66 | # Training loop 67 | train_loss = [] 68 | test_loss = [] 69 | for i in range(200): 70 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 71 | rand_x = np.transpose([x_vals_train[rand_index]]) 72 | rand_y = np.transpose([y_vals_train[rand_index]]) 73 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 74 | 75 | temp_train_loss = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_train]), y_target: np.transpose([y_vals_train])}) 76 | train_loss.append(temp_train_loss) 77 | 78 | temp_test_loss = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_test]), y_target: np.transpose([y_vals_test])}) 79 | test_loss.append(temp_test_loss) 80 | if (i+1)%50==0: 81 | print('-----------') 82 | print('Generation: ' + str(i+1)) 83 | print('A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b))) 84 | print('Train Loss = ' + str(temp_train_loss)) 85 | print('Test Loss = ' + str(temp_test_loss)) 86 | 87 | # Extract Coefficients 88 | [[slope]] = sess.run(A) 89 | [[y_intercept]] = sess.run(b) 90 | [width] = sess.run(epsilon) 91 | 92 | # Get best fit line 93 | best_fit = [] 94 | best_fit_upper = [] 95 | best_fit_lower = [] 96 | for i in x_vals: 97 | best_fit.append(slope*i+y_intercept) 98 | best_fit_upper.append(slope*i+y_intercept+width) 99 | best_fit_lower.append(slope*i+y_intercept-width) 100 | 101 | # Plot fit with data 102 | plt.plot(x_vals, y_vals, 'o', label='Data Points') 103 | plt.plot(x_vals, best_fit, 'r-', label='SVM Regression Line', linewidth=3) 104 | plt.plot(x_vals, best_fit_upper, 'r--', linewidth=2) 105 | plt.plot(x_vals, best_fit_lower, 'r--', linewidth=2) 106 | plt.ylim([0, 10]) 107 | plt.legend(loc='lower right') 108 | plt.title('Sepal Length vs Pedal Width') 109 | plt.xlabel('Pedal Width') 110 | plt.ylabel('Sepal Length') 111 | plt.show() 112 | 113 | # Plot loss over time 114 | plt.plot(train_loss, 'k-', label='Train Set Loss') 115 | plt.plot(test_loss, 'r--', label='Test Set Loss') 116 | plt.title('L2 Loss per Generation') 117 | plt.xlabel('Generation') 118 | plt.ylabel('L2 Loss') 119 | plt.legend(loc='upper right') 120 | plt.show() 121 | -------------------------------------------------------------------------------- /Chapter 04/svm_kernels.py: -------------------------------------------------------------------------------- 1 | # Illustration of Various Kernels 2 | #---------------------------------- 3 | # 4 | # This function wll illustrate how to 5 | # implement various kernels in Tensorflow. 6 | # 7 | # Linear Kernel: 8 | # K(x1, x2) = t(x1) * x2 9 | # 10 | # Gaussian Kernel (RBF): 11 | # K(x1, x2) = exp(-gamma * abs(x1 - x2)^2) 12 | 13 | import matplotlib.pyplot as plt 14 | import numpy as np 15 | import tensorflow as tf 16 | from sklearn import datasets 17 | from tensorflow.python.framework import ops 18 | ops.reset_default_graph() 19 | 20 | # Create graph 21 | sess = tf.Session() 22 | 23 | # Generate non-lnear data 24 | (x_vals, y_vals) = datasets.make_circles(n_samples=350, factor=.5, noise=.1) 25 | y_vals = np.array([1 if y==1 else -1 for y in y_vals]) 26 | class1_x = [x[0] for i,x in enumerate(x_vals) if y_vals[i]==1] 27 | class1_y = [x[1] for i,x in enumerate(x_vals) if y_vals[i]==1] 28 | class2_x = [x[0] for i,x in enumerate(x_vals) if y_vals[i]==-1] 29 | class2_y = [x[1] for i,x in enumerate(x_vals) if y_vals[i]==-1] 30 | 31 | # Declare batch size 32 | batch_size = 350 33 | 34 | # Initialize placeholders 35 | x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32) 36 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 37 | prediction_grid = tf.placeholder(shape=[None, 2], dtype=tf.float32) 38 | 39 | # Create variables for svm 40 | b = tf.Variable(tf.random_normal(shape=[1,batch_size])) 41 | 42 | # Apply kernel 43 | # Linear Kernel 44 | # my_kernel = tf.matmul(x_data, tf.transpose(x_data)) 45 | 46 | # Gaussian (RBF) kernel 47 | gamma = tf.constant(-50.0) 48 | dist = tf.reduce_sum(tf.square(x_data), 1) 49 | dist = tf.reshape(dist, [-1,1]) 50 | sq_dists = tf.add(tf.subtract(dist, tf.multiply(2., tf.matmul(x_data, tf.transpose(x_data)))), tf.transpose(dist)) 51 | my_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists))) 52 | 53 | # Compute SVM Model 54 | model_output = tf.matmul(b, my_kernel) 55 | first_term = tf.reduce_sum(b) 56 | b_vec_cross = tf.matmul(tf.transpose(b), b) 57 | y_target_cross = tf.matmul(y_target, tf.transpose(y_target)) 58 | second_term = tf.reduce_sum(tf.multiply(my_kernel, tf.multiply(b_vec_cross, y_target_cross))) 59 | loss = tf.negative(tf.subtract(first_term, second_term)) 60 | 61 | # Create Prediction Kernel 62 | # Linear prediction kernel 63 | # my_kernel = tf.matmul(x_data, tf.transpose(prediction_grid)) 64 | 65 | # Gaussian (RBF) prediction kernel 66 | rA = tf.reshape(tf.reduce_sum(tf.square(x_data), 1),[-1,1]) 67 | rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1),[-1,1]) 68 | pred_sq_dist = tf.add(tf.subtract(rA, tf.multiply(2., tf.matmul(x_data, tf.transpose(prediction_grid)))), tf.transpose(rB)) 69 | pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist))) 70 | 71 | prediction_output = tf.matmul(tf.multiply(tf.transpose(y_target),b), pred_kernel) 72 | prediction = tf.sign(prediction_output-tf.reduce_mean(prediction_output)) 73 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.squeeze(prediction), tf.squeeze(y_target)), tf.float32)) 74 | 75 | # Declare optimizer 76 | my_opt = tf.train.GradientDescentOptimizer(0.002) 77 | train_step = my_opt.minimize(loss) 78 | 79 | # Initialize variables 80 | init = tf.global_variables_initializer() 81 | sess.run(init) 82 | 83 | # Training loop 84 | loss_vec = [] 85 | batch_accuracy = [] 86 | for i in range(1000): 87 | rand_index = np.random.choice(len(x_vals), size=batch_size) 88 | rand_x = x_vals[rand_index] 89 | rand_y = np.transpose([y_vals[rand_index]]) 90 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 91 | 92 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 93 | loss_vec.append(temp_loss) 94 | 95 | acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x, 96 | y_target: rand_y, 97 | prediction_grid:rand_x}) 98 | batch_accuracy.append(acc_temp) 99 | 100 | if (i+1)%250==0: 101 | print('Step #' + str(i+1)) 102 | print('Loss = ' + str(temp_loss)) 103 | 104 | # Create a mesh to plot points in 105 | x_min, x_max = x_vals[:, 0].min() - 1, x_vals[:, 0].max() + 1 106 | y_min, y_max = x_vals[:, 1].min() - 1, x_vals[:, 1].max() + 1 107 | xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.02), 108 | np.arange(y_min, y_max, 0.02)) 109 | grid_points = np.c_[xx.ravel(), yy.ravel()] 110 | [grid_predictions] = sess.run(prediction, feed_dict={x_data: rand_x, 111 | y_target: rand_y, 112 | prediction_grid: grid_points}) 113 | grid_predictions = grid_predictions.reshape(xx.shape) 114 | 115 | # Plot points and grid 116 | plt.contourf(xx, yy, grid_predictions, cmap=plt.cm.Paired, alpha=0.8) 117 | plt.plot(class1_x, class1_y, 'ro', label='Class 1') 118 | plt.plot(class2_x, class2_y, 'kx', label='Class -1') 119 | plt.title('Gaussian SVM Results') 120 | plt.xlabel('x') 121 | plt.ylabel('y') 122 | plt.legend(loc='lower right') 123 | plt.ylim([-1.5, 1.5]) 124 | plt.xlim([-1.5, 1.5]) 125 | plt.show() 126 | 127 | # Plot batch accuracy 128 | plt.plot(batch_accuracy, 'k-', label='Accuracy') 129 | plt.title('Batch Accuracy') 130 | plt.xlabel('Generation') 131 | plt.ylabel('Accuracy') 132 | plt.legend(loc='lower right') 133 | plt.show() 134 | 135 | # Plot loss over time 136 | plt.plot(loss_vec, 'k-') 137 | plt.title('Loss per Generation') 138 | plt.xlabel('Generation') 139 | plt.ylabel('Loss') 140 | plt.show() 141 | 142 | # sess.run(prediction_output, feed_dict={x_data: rand_x, y_target: rand_y, prediction_grid: grid_points}) 143 | # sess.run(pred_kernel, feed_dict={x_data: rand_x, y_target: rand_y, prediction_grid: grid_points}) 144 | # sess.run(model_output, feed_dict={x_data:rand_x, y_target: rand_y}) 145 | # sess.run(second_term, feed_dict={x_data:rand_x, y_target: rand_y}) -------------------------------------------------------------------------------- /Chapter 05/address_matching.py: -------------------------------------------------------------------------------- 1 | # Address Matching with k-Nearest Neighbors 2 | #---------------------------------- 3 | # 4 | # This function illustrates a way to perform 5 | # address matching between two data sets. 6 | # 7 | # For each test address, we will return the 8 | # closest reference address to it. 9 | # 10 | # We will consider two distance functions: 11 | # 1) Edit distance for street number/name and 12 | # 2) Euclidian distance (L2) for the zip codes 13 | 14 | import random 15 | import string 16 | import numpy as np 17 | import tensorflow as tf 18 | from tensorflow.python.framework import ops 19 | ops.reset_default_graph() 20 | 21 | # First we generate the data sets we will need 22 | # n = Size of created data sets 23 | n = 10 24 | street_names = ['abbey', 'baker', 'canal', 'donner', 'elm'] 25 | street_types = ['rd', 'st', 'ln', 'pass', 'ave'] 26 | rand_zips = [random.randint(65000,65999) for i in range(5)] 27 | 28 | # Function to randomly create one typo in a string w/ a probability 29 | def create_typo(s, prob=0.75): 30 | if random.uniform(0,1) < prob: 31 | rand_ind = random.choice(range(len(s))) 32 | s_list = list(s) 33 | s_list[rand_ind]=random.choice(string.ascii_lowercase) 34 | s = ''.join(s_list) 35 | return(s) 36 | 37 | # Generate the reference dataset 38 | numbers = [random.randint(1, 9999) for i in range(n)] 39 | streets = [random.choice(street_names) for i in range(n)] 40 | street_suffs = [random.choice(street_types) for i in range(n)] 41 | zips = [random.choice(rand_zips) for i in range(n)] 42 | full_streets = [str(x) + ' ' + y + ' ' + z for x,y,z in zip(numbers, streets, street_suffs)] 43 | reference_data = [list(x) for x in zip(full_streets,zips)] 44 | 45 | # Generate test dataset with some typos 46 | typo_streets = [create_typo(x) for x in streets] 47 | typo_full_streets = [str(x) + ' ' + y + ' ' + z for x,y,z in zip(numbers, typo_streets, street_suffs)] 48 | test_data = [list(x) for x in zip(typo_full_streets,zips)] 49 | 50 | # Now we can perform address matching 51 | # Create graph 52 | sess = tf.Session() 53 | 54 | # Placeholders 55 | test_address = tf.sparse_placeholder( dtype=tf.string) 56 | test_zip = tf.placeholder(shape=[None, 1], dtype=tf.float32) 57 | ref_address = tf.sparse_placeholder(dtype=tf.string) 58 | ref_zip = tf.placeholder(shape=[None, n], dtype=tf.float32) 59 | 60 | # Declare Zip code distance for a test zip and reference set 61 | zip_dist = tf.square(tf.subtract(ref_zip, test_zip)) 62 | 63 | # Declare Edit distance for address 64 | address_dist = tf.edit_distance(test_address, ref_address, normalize=True) 65 | 66 | # Create similarity scores 67 | zip_max = tf.gather(tf.squeeze(zip_dist), tf.argmax(zip_dist, 1)) 68 | zip_min = tf.gather(tf.squeeze(zip_dist), tf.argmin(zip_dist, 1)) 69 | zip_sim = tf.div(tf.subtract(zip_max, zip_dist), tf.subtract(zip_max, zip_min)) 70 | address_sim = tf.subtract(1., address_dist) 71 | 72 | # Combine distance functions 73 | address_weight = 0.5 74 | zip_weight = 1. - address_weight 75 | weighted_sim = tf.add(tf.transpose(tf.multiply(address_weight, address_sim)), tf.multiply(zip_weight, zip_sim)) 76 | 77 | # Predict: Get max similarity entry 78 | top_match_index = tf.argmax(weighted_sim, 1) 79 | 80 | 81 | # Function to Create a character-sparse tensor from strings 82 | def sparse_from_word_vec(word_vec): 83 | num_words = len(word_vec) 84 | indices = [[xi, 0, yi] for xi,x in enumerate(word_vec) for yi,y in enumerate(x)] 85 | chars = list(''.join(word_vec)) 86 | return(tf.SparseTensorValue(indices, chars, [num_words,1,1])) 87 | 88 | # Loop through test indices 89 | reference_addresses = [x[0] for x in reference_data] 90 | reference_zips = np.array([[x[1] for x in reference_data]]) 91 | 92 | # Create sparse address reference set 93 | sparse_ref_set = sparse_from_word_vec(reference_addresses) 94 | 95 | for i in range(n): 96 | test_address_entry = test_data[i][0] 97 | test_zip_entry = [[test_data[i][1]]] 98 | 99 | # Create sparse address vectors 100 | test_address_repeated = [test_address_entry] * n 101 | sparse_test_set = sparse_from_word_vec(test_address_repeated) 102 | 103 | feeddict={test_address: sparse_test_set, 104 | test_zip: test_zip_entry, 105 | ref_address: sparse_ref_set, 106 | ref_zip: reference_zips} 107 | best_match = sess.run(top_match_index, feed_dict=feeddict) 108 | best_street = reference_addresses[best_match[0]] 109 | [best_zip] = reference_zips[0][best_match] 110 | [[test_zip_]] = test_zip_entry 111 | print('Address: ' + str(test_address_entry) + ', ' + str(test_zip_)) 112 | print('Match : ' + str(best_street) + ', ' + str(best_zip)) -------------------------------------------------------------------------------- /Chapter 05/image_recognition.py: -------------------------------------------------------------------------------- 1 | # MNIST Digit Prediction with k-Nearest Neighbors 2 | #----------------------------------------------- 3 | # 4 | # This script will load the MNIST data, and split 5 | # it into test/train and perform prediction with 6 | # nearest neighbors 7 | # 8 | # For each test integer, we will return the 9 | # closest image/integer. 10 | # 11 | # Integer images are represented as 28x8 matrices 12 | # of floating point numbers 13 | 14 | import random 15 | import numpy as np 16 | import tensorflow as tf 17 | import matplotlib.pyplot as plt 18 | from PIL import Image 19 | from tensorflow.examples.tutorials.mnist import input_data 20 | from tensorflow.python.framework import ops 21 | ops.reset_default_graph() 22 | 23 | # Create graph 24 | sess = tf.Session() 25 | 26 | # Load the data 27 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 28 | 29 | # Random sample 30 | train_size = 1000 31 | test_size = 102 32 | rand_train_indices = np.random.choice(len(mnist.train.images), train_size, replace=False) 33 | rand_test_indices = np.random.choice(len(mnist.test.images), test_size, replace=False) 34 | x_vals_train = mnist.train.images[rand_train_indices] 35 | x_vals_test = mnist.test.images[rand_test_indices] 36 | y_vals_train = mnist.train.labels[rand_train_indices] 37 | y_vals_test = mnist.test.labels[rand_test_indices] 38 | 39 | # Declare k-value and batch size 40 | k = 4 41 | batch_size=6 42 | 43 | # Placeholders 44 | x_data_train = tf.placeholder(shape=[None, 784], dtype=tf.float32) 45 | x_data_test = tf.placeholder(shape=[None, 784], dtype=tf.float32) 46 | y_target_train = tf.placeholder(shape=[None, 10], dtype=tf.float32) 47 | y_target_test = tf.placeholder(shape=[None, 10], dtype=tf.float32) 48 | 49 | # Declare distance metric 50 | # L1 51 | distance = tf.reduce_sum(tf.abs(tf.subtract(x_data_train, tf.expand_dims(x_data_test,1))), reduction_indices=2) 52 | 53 | # L2 54 | #distance = tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(x_data_train, tf.expand_dims(x_data_test,1))), reduction_indices=1)) 55 | 56 | # Predict: Get min distance index (Nearest neighbor) 57 | top_k_xvals, top_k_indices = tf.nn.top_k(tf.negative(distance), k=k) 58 | prediction_indices = tf.gather(y_target_train, top_k_indices) 59 | # Predict the mode category 60 | count_of_predictions = tf.reduce_sum(prediction_indices, reduction_indices=1) 61 | prediction = tf.argmax(count_of_predictions, dimension=1) 62 | 63 | # Calculate how many loops over training data 64 | num_loops = int(np.ceil(len(x_vals_test)/batch_size)) 65 | 66 | test_output = [] 67 | actual_vals = [] 68 | for i in range(num_loops): 69 | min_index = i*batch_size 70 | max_index = min((i+1)*batch_size,len(x_vals_train)) 71 | x_batch = x_vals_test[min_index:max_index] 72 | y_batch = y_vals_test[min_index:max_index] 73 | predictions = sess.run(prediction, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, 74 | y_target_train: y_vals_train, y_target_test: y_batch}) 75 | test_output.extend(predictions) 76 | actual_vals.extend(np.argmax(y_batch, axis=1)) 77 | 78 | accuracy = sum([1./test_size for i in range(test_size) if test_output[i]==actual_vals[i]]) 79 | print('Accuracy on test set: ' + str(accuracy)) 80 | 81 | # Plot the last batch results: 82 | actuals = np.argmax(y_batch, axis=1) 83 | 84 | Nrows = 2 85 | Ncols = 3 86 | for i in range(len(actuals)): 87 | plt.subplot(Nrows, Ncols, i+1) 88 | plt.imshow(np.reshape(x_batch[i], [28,28]), cmap='Greys_r') 89 | plt.title('Actual: ' + str(actuals[i]) + ' Pred: ' + str(predictions[i]), 90 | fontsize=10) 91 | frame = plt.gca() 92 | frame.axes.get_xaxis().set_visible(False) 93 | frame.axes.get_yaxis().set_visible(False) -------------------------------------------------------------------------------- /Chapter 05/mixed_distance_functions_knn.py: -------------------------------------------------------------------------------- 1 | # Mixed Distance Functions for k-Nearest Neighbor 2 | # ---------------------------------- 3 | # 4 | # This function shows how to use different distance 5 | # metrics on different features for kNN. 6 | # 7 | # Data: 8 | # ----------x-values----------- 9 | # CRIM : per capita crime rate by town 10 | # ZN : prop. of res. land zones 11 | # INDUS : prop. of non-retail business acres 12 | # CHAS : Charles river dummy variable 13 | # NOX : nitrix oxides concentration / 10 M 14 | # RM : Avg. # of rooms per building 15 | # AGE : prop. of buildings built prior to 1940 16 | # DIS : Weighted distances to employment centers 17 | # RAD : Index of radian highway access 18 | # TAX : Full tax rate value per $10k 19 | # PTRATIO: Pupil/Teacher ratio by town 20 | # B : 1000*(Bk-0.63)^2, Bk=prop. of blacks 21 | # LSTAT : % lower status of pop 22 | # ------------y-value----------- 23 | # MEDV : Median Value of homes in $1,000's 24 | 25 | 26 | import matplotlib.pyplot as plt 27 | import numpy as np 28 | import tensorflow as tf 29 | import requests 30 | from tensorflow.python.framework import ops 31 | 32 | ops.reset_default_graph() 33 | 34 | # Create graph 35 | sess = tf.Session() 36 | 37 | # Load the data 38 | housing_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data' 39 | housing_header = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 40 | 'MEDV'] 41 | cols_used = ['CRIM', 'INDUS', 'NOX', 'RM', 'AGE', 'DIS', 'TAX', 'PTRATIO', 'B', 'LSTAT'] 42 | num_features = len(cols_used) 43 | housing_file = requests.get(housing_url) 44 | housing_data = [[float(x) for x in y.split(' ') if len(x) >= 1] for y in housing_file.text.split('\n') if len(y) >= 1] 45 | 46 | y_vals = np.transpose([np.array([y[13] for y in housing_data])]) 47 | x_vals = np.array([[x for i, x in enumerate(y) if housing_header[i] in cols_used] for y in housing_data]) 48 | 49 | ## Min-Max Scaling 50 | x_vals = (x_vals - x_vals.min(0)) / x_vals.ptp(0) 51 | 52 | ## Create distance metric weight matrix weighted by standard deviation 53 | weight_diagonal = x_vals.std(0) 54 | weight_matrix = tf.cast(tf.diag(weight_diagonal), dtype=tf.float32) 55 | 56 | # Split the data into train and test sets 57 | train_indices = np.random.choice(len(x_vals), int(round(len(x_vals) * 0.8)), replace=False) 58 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 59 | x_vals_train = x_vals[train_indices] 60 | x_vals_test = x_vals[test_indices] 61 | y_vals_train = y_vals[train_indices] 62 | y_vals_test = y_vals[test_indices] 63 | 64 | # Declare k-value and batch size 65 | k = 4 66 | batch_size = len(x_vals_test) 67 | 68 | # Placeholders 69 | x_data_train = tf.placeholder(shape=[None, num_features], dtype=tf.float32) 70 | x_data_test = tf.placeholder(shape=[None, num_features], dtype=tf.float32) 71 | y_target_train = tf.placeholder(shape=[None, 1], dtype=tf.float32) 72 | y_target_test = tf.placeholder(shape=[None, 1], dtype=tf.float32) 73 | 74 | # Declare weighted distance metric 75 | # Weighted - L2 = sqrt((x-y)^T * A * (x-y)) 76 | subtraction_term = tf.subtract(x_data_train, tf.expand_dims(x_data_test, 1)) 77 | first_product = tf.matmul(subtraction_term, tf.tile(tf.expand_dims(weight_matrix, 0), [batch_size, 1, 1])) 78 | second_product = tf.matmul(first_product, tf.transpose(subtraction_term, perm=[0, 2, 1])) 79 | distance = tf.sqrt(tf.matrix_diag_part(second_product)) 80 | 81 | # Predict: Get min distance index (Nearest neighbor) 82 | top_k_xvals, top_k_indices = tf.nn.top_k(tf.negative(distance), k=k) 83 | x_sums = tf.expand_dims(tf.reduce_sum(top_k_xvals, 1), 1) 84 | x_sums_repeated = tf.matmul(x_sums, tf.ones([1, k], tf.float32)) 85 | x_val_weights = tf.expand_dims(tf.div(top_k_xvals, x_sums_repeated), 1) 86 | 87 | top_k_yvals = tf.gather(y_target_train, top_k_indices) 88 | prediction = tf.squeeze(tf.matmul(x_val_weights, top_k_yvals), squeeze_dims=[1]) 89 | 90 | # Calculate MSE 91 | mse = tf.div(tf.reduce_sum(tf.square(tf.subtract(prediction, y_target_test))), batch_size) 92 | 93 | # Calculate how many loops over training data 94 | num_loops = int(np.ceil(len(x_vals_test) / batch_size)) 95 | 96 | for i in range(num_loops): 97 | min_index = i * batch_size 98 | max_index = min((i + 1) * batch_size, len(x_vals_train)) 99 | x_batch = x_vals_test[min_index:max_index] 100 | y_batch = y_vals_test[min_index:max_index] 101 | predictions = sess.run(prediction, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, 102 | y_target_train: y_vals_train, y_target_test: y_batch}) 103 | batch_mse = sess.run(mse, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, 104 | y_target_train: y_vals_train, y_target_test: y_batch}) 105 | 106 | print('Batch #' + str(i + 1) + ' MSE: ' + str(np.round(batch_mse, 3))) 107 | 108 | # Plot prediction and actual distribution 109 | bins = np.linspace(5, 50, 45) 110 | 111 | plt.hist(predictions, bins, alpha=0.5, label='Prediction') 112 | plt.hist(y_batch, bins, alpha=0.5, label='Actual') 113 | plt.title('Histogram of Predicted and Actual Values') 114 | plt.xlabel('Med Home Value in $1,000s') 115 | plt.ylabel('Frequency') 116 | plt.legend(loc='upper right') 117 | plt.show() 118 | -------------------------------------------------------------------------------- /Chapter 05/nearest_neighbor.py: -------------------------------------------------------------------------------- 1 | # k-Nearest Neighbor 2 | #---------------------------------- 3 | # 4 | # This function illustrates how to use 5 | # k-nearest neighbors in tensorflow 6 | # 7 | # We will use the 1970s Boston housing dataset 8 | # which is available through the UCI 9 | # ML data repository. 10 | # 11 | # Data: 12 | #----------x-values----------- 13 | # CRIM : per capita crime rate by town 14 | # ZN : prop. of res. land zones 15 | # INDUS : prop. of non-retail business acres 16 | # CHAS : Charles river dummy variable 17 | # NOX : nitrix oxides concentration / 10 M 18 | # RM : Avg. # of rooms per building 19 | # AGE : prop. of buildings built prior to 1940 20 | # DIS : Weighted distances to employment centers 21 | # RAD : Index of radian highway access 22 | # TAX : Full tax rate value per $10k 23 | # PTRATIO: Pupil/Teacher ratio by town 24 | # B : 1000*(Bk-0.63)^2, Bk=prop. of blacks 25 | # LSTAT : % lower status of pop 26 | #------------y-value----------- 27 | # MEDV : Median Value of homes in $1,000's 28 | 29 | import matplotlib.pyplot as plt 30 | import numpy as np 31 | import tensorflow as tf 32 | import requests 33 | from tensorflow.python.framework import ops 34 | ops.reset_default_graph() 35 | 36 | # Create graph 37 | sess = tf.Session() 38 | 39 | # Load the data 40 | housing_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data' 41 | housing_header = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV'] 42 | cols_used = ['CRIM', 'INDUS', 'NOX', 'RM', 'AGE', 'DIS', 'TAX', 'PTRATIO', 'B', 'LSTAT'] 43 | num_features = len(cols_used) 44 | housing_file = requests.get(housing_url) 45 | housing_data = [[float(x) for x in y.split(' ') if len(x)>=1] for y in housing_file.text.split('\n') if len(y)>=1] 46 | 47 | y_vals = np.transpose([np.array([y[13] for y in housing_data])]) 48 | x_vals = np.array([[x for i,x in enumerate(y) if housing_header[i] in cols_used] for y in housing_data]) 49 | 50 | ## Min-Max Scaling 51 | x_vals = (x_vals - x_vals.min(0)) / x_vals.ptp(0) 52 | 53 | # Split the data into train and test sets 54 | train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 55 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 56 | x_vals_train = x_vals[train_indices] 57 | x_vals_test = x_vals[test_indices] 58 | y_vals_train = y_vals[train_indices] 59 | y_vals_test = y_vals[test_indices] 60 | 61 | # Declare k-value and batch size 62 | k = 4 63 | batch_size=len(x_vals_test) 64 | 65 | # Placeholders 66 | x_data_train = tf.placeholder(shape=[None, num_features], dtype=tf.float32) 67 | x_data_test = tf.placeholder(shape=[None, num_features], dtype=tf.float32) 68 | y_target_train = tf.placeholder(shape=[None, 1], dtype=tf.float32) 69 | y_target_test = tf.placeholder(shape=[None, 1], dtype=tf.float32) 70 | 71 | # Declare distance metric 72 | # L1 73 | distance = tf.reduce_sum(tf.abs(tf.subtract(x_data_train, tf.expand_dims(x_data_test,1))), reduction_indices=2) 74 | 75 | # L2 76 | #distance = tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(x_data_train, tf.expand_dims(x_data_test,1))), reduction_indices=1)) 77 | 78 | # Predict: Get min distance index (Nearest neighbor) 79 | top_k_xvals, top_k_indices = tf.nn.top_k(tf.negative(distance), k=k) 80 | x_sums = tf.expand_dims(tf.reduce_sum(top_k_xvals, 1),1) 81 | x_sums_repeated = tf.matmul(x_sums,tf.ones([1, k], tf.float32)) 82 | x_val_weights = tf.expand_dims(tf.div(top_k_xvals,x_sums_repeated), 1) 83 | 84 | top_k_yvals = tf.gather(y_target_train, top_k_indices) 85 | prediction = tf.squeeze(tf.matmul(x_val_weights,top_k_yvals), squeeze_dims=[1]) 86 | 87 | # Calculate MSE 88 | mse = tf.div(tf.reduce_sum(tf.square(tf.subtract(prediction, y_target_test))), batch_size) 89 | 90 | # Calculate how many loops over training data 91 | num_loops = int(np.ceil(len(x_vals_test)/batch_size)) 92 | 93 | for i in range(num_loops): 94 | min_index = i*batch_size 95 | max_index = min((i+1)*batch_size,len(x_vals_train)) 96 | x_batch = x_vals_test[min_index:max_index] 97 | y_batch = y_vals_test[min_index:max_index] 98 | predictions = sess.run(prediction, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, 99 | y_target_train: y_vals_train, y_target_test: y_batch}) 100 | batch_mse = sess.run(mse, feed_dict={x_data_train: x_vals_train, x_data_test: x_batch, 101 | y_target_train: y_vals_train, y_target_test: y_batch}) 102 | 103 | print('Batch #' + str(i+1) + ' MSE: ' + str(np.round(batch_mse,3))) 104 | 105 | # Plot prediction and actual distribution 106 | bins = np.linspace(5, 50, 45) 107 | 108 | plt.hist(predictions, bins, alpha=0.5, label='Prediction') 109 | plt.hist(y_batch, bins, alpha=0.5, label='Actual') 110 | plt.title('Histogram of Predicted and Actual Values') 111 | plt.xlabel('Med Home Value in $1,000s') 112 | plt.ylabel('Frequency') 113 | plt.legend(loc='upper right') 114 | plt.show() 115 | 116 | -------------------------------------------------------------------------------- /Chapter 05/text_distances.py: -------------------------------------------------------------------------------- 1 | # Text Distances 2 | #---------------------------------- 3 | # 4 | # This function illustrates how to use 5 | # the Levenstein distance (edit distance) 6 | # in Tensorflow. 7 | 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # Start Graph Session 13 | sess = tf.Session() 14 | 15 | #---------------------------------- 16 | # First compute the edit distance between 'bear' and 'beers' 17 | hypothesis = list('bear') 18 | truth = list('beers') 19 | h1 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3]], 20 | hypothesis, 21 | [1,1,1]) 22 | 23 | t1 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3],[0,0,4]], 24 | truth, 25 | [1,1,1]) 26 | 27 | print(sess.run(tf.edit_distance(h1, t1, normalize=False))) 28 | 29 | #---------------------------------- 30 | # Compute the edit distance between ('bear','beer') and 'beers': 31 | hypothesis2 = list('bearbeer') 32 | truth2 = list('beersbeers') 33 | h2 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3], [0,1,0], [0,1,1], [0,1,2], [0,1,3]], 34 | hypothesis2, 35 | [1,2,4]) 36 | 37 | t2 = tf.SparseTensor([[0,0,0], [0,0,1], [0,0,2], [0,0,3], [0,0,4], [0,1,0], [0,1,1], [0,1,2], [0,1,3], [0,1,4]], 38 | truth2, 39 | [1,2,5]) 40 | 41 | print(sess.run(tf.edit_distance(h2, t2, normalize=True))) 42 | 43 | #---------------------------------- 44 | # Now compute distance between four words and 'beers' more efficiently: 45 | hypothesis_words = ['bear','bar','tensor','flow'] 46 | truth_word = ['beers'] 47 | 48 | num_h_words = len(hypothesis_words) 49 | h_indices = [[xi, 0, yi] for xi,x in enumerate(hypothesis_words) for yi,y in enumerate(x)] 50 | h_chars = list(''.join(hypothesis_words)) 51 | 52 | h3 = tf.SparseTensor(h_indices, h_chars, [num_h_words,1,1]) 53 | 54 | truth_word_vec = truth_word*num_h_words 55 | t_indices = [[xi, 0, yi] for xi,x in enumerate(truth_word_vec) for yi,y in enumerate(x)] 56 | t_chars = list(''.join(truth_word_vec)) 57 | 58 | t3 = tf.SparseTensor(t_indices, t_chars, [num_h_words,1,1]) 59 | 60 | print(sess.run(tf.edit_distance(h3, t3, normalize=True))) 61 | 62 | #---------------------------------- 63 | # Now we show how to use sparse tensors in a feed dictionary 64 | 65 | # Create input data 66 | hypothesis_words = ['bear','bar','tensor','flow'] 67 | truth_word = ['beers'] 68 | 69 | def create_sparse_vec(word_list): 70 | num_words = len(word_list) 71 | indices = [[xi, 0, yi] for xi,x in enumerate(word_list) for yi,y in enumerate(x)] 72 | chars = list(''.join(word_list)) 73 | return(tf.SparseTensorValue(indices, chars, [num_words,1,1])) 74 | 75 | hyp_string_sparse = create_sparse_vec(hypothesis_words) 76 | truth_string_sparse = create_sparse_vec(truth_word*len(hypothesis_words)) 77 | 78 | hyp_input = tf.sparse_placeholder(dtype=tf.string) 79 | truth_input = tf.sparse_placeholder(dtype=tf.string) 80 | 81 | edit_distances = tf.edit_distance(hyp_input, truth_input, normalize=True) 82 | 83 | feed_dict = {hyp_input: hyp_string_sparse, 84 | truth_input: truth_string_sparse} 85 | 86 | print(sess.run(edit_distances, feed_dict=feed_dict)) -------------------------------------------------------------------------------- /Chapter 06/activation_functions.py: -------------------------------------------------------------------------------- 1 | # Combining Gates and Activation Functions 2 | #---------------------------------- 3 | # 4 | # This function shows how to implement 5 | # various gates with activation functions 6 | # in Tensorflow 7 | # 8 | # This function is an extension of the 9 | # prior gates, but with various activation 10 | # functions. 11 | 12 | import tensorflow as tf 13 | import numpy as np 14 | import matplotlib.pyplot as plt 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | # Start Graph Session 19 | sess = tf.Session() 20 | tf.set_random_seed(5) 21 | np.random.seed(42) 22 | 23 | batch_size = 50 24 | 25 | a1 = tf.Variable(tf.random_normal(shape=[1,1])) 26 | b1 = tf.Variable(tf.random_uniform(shape=[1,1])) 27 | a2 = tf.Variable(tf.random_normal(shape=[1,1])) 28 | b2 = tf.Variable(tf.random_uniform(shape=[1,1])) 29 | x = np.random.normal(2, 0.1, 500) 30 | x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32) 31 | 32 | sigmoid_activation = tf.sigmoid(tf.add(tf.matmul(x_data, a1), b1)) 33 | 34 | relu_activation = tf.nn.relu(tf.add(tf.matmul(x_data, a2), b2)) 35 | 36 | # Declare the loss function as the difference between 37 | # the output and a target value, 0.75. 38 | loss1 = tf.reduce_mean(tf.square(tf.subtract(sigmoid_activation, 0.75))) 39 | loss2 = tf.reduce_mean(tf.square(tf.subtract(relu_activation, 0.75))) 40 | 41 | # Initialize variables 42 | init = tf.global_variables_initializer() 43 | sess.run(init) 44 | 45 | # Declare optimizer 46 | my_opt = tf.train.GradientDescentOptimizer(0.01) 47 | train_step_sigmoid = my_opt.minimize(loss1) 48 | train_step_relu = my_opt.minimize(loss2) 49 | 50 | # Run loop across gate 51 | print('\nOptimizing Sigmoid AND Relu Output to 0.75') 52 | loss_vec_sigmoid = [] 53 | loss_vec_relu = [] 54 | activation_sigmoid = [] 55 | activation_relu = [] 56 | for i in range(750): 57 | rand_indices = np.random.choice(len(x), size=batch_size) 58 | x_vals = np.transpose([x[rand_indices]]) 59 | sess.run(train_step_sigmoid, feed_dict={x_data: x_vals}) 60 | sess.run(train_step_relu, feed_dict={x_data: x_vals}) 61 | 62 | loss_vec_sigmoid.append(sess.run(loss1, feed_dict={x_data: x_vals})) 63 | loss_vec_relu.append(sess.run(loss2, feed_dict={x_data: x_vals})) 64 | 65 | activation_sigmoid.append(np.mean(sess.run(sigmoid_activation, feed_dict={x_data: x_vals}))) 66 | activation_relu.append(np.mean(sess.run(relu_activation, feed_dict={x_data: x_vals}))) 67 | 68 | 69 | # Plot the activation values 70 | plt.plot(activation_sigmoid, 'k-', label='Sigmoid Activation') 71 | plt.plot(activation_relu, 'r--', label='Relu Activation') 72 | plt.ylim([0, 1.0]) 73 | plt.title('Activation Outputs') 74 | plt.xlabel('Generation') 75 | plt.ylabel('Outputs') 76 | plt.legend(loc='upper right') 77 | plt.show() 78 | 79 | 80 | # Plot the loss 81 | plt.plot(loss_vec_sigmoid, 'k-', label='Sigmoid Loss') 82 | plt.plot(loss_vec_relu, 'r--', label='Relu Loss') 83 | plt.ylim([0, 1.0]) 84 | plt.title('Loss per Generation') 85 | plt.xlabel('Generation') 86 | plt.ylabel('Loss') 87 | plt.legend(loc='upper right') 88 | plt.show() -------------------------------------------------------------------------------- /Chapter 06/gates.py: -------------------------------------------------------------------------------- 1 | # Implementing Gates 2 | #---------------------------------- 3 | # 4 | # This function shows how to implement 5 | # various gates in Tensorflow 6 | # 7 | # One gate will be one operation with 8 | # a variable and a placeholder. 9 | # We will ask Tensorflow to change the 10 | # variable based on our loss function 11 | 12 | import tensorflow as tf 13 | from tensorflow.python.framework import ops 14 | ops.reset_default_graph() 15 | 16 | # Start Graph Session 17 | sess = tf.Session() 18 | 19 | #---------------------------------- 20 | # Create a multiplication gate: 21 | # f(x) = a * x 22 | # 23 | # a -- 24 | # | 25 | # |---- (multiply) --> output 26 | # x --| 27 | # 28 | 29 | a = tf.Variable(tf.constant(4.)) 30 | x_val = 5. 31 | x_data = tf.placeholder(dtype=tf.float32) 32 | 33 | multiplication = tf.multiply(a, x_data) 34 | 35 | # Declare the loss function as the difference between 36 | # the output and a target value, 50. 37 | loss = tf.square(tf.subtract(multiplication, 50.)) 38 | 39 | # Initialize variables 40 | init = tf.global_variables_initializer() 41 | sess.run(init) 42 | 43 | # Declare optimizer 44 | my_opt = tf.train.GradientDescentOptimizer(0.01) 45 | train_step = my_opt.minimize(loss) 46 | 47 | # Run loop across gate 48 | print('Optimizing a Multiplication Gate Output to 50.') 49 | for i in range(10): 50 | sess.run(train_step, feed_dict={x_data: x_val}) 51 | a_val = sess.run(a) 52 | mult_output = sess.run(multiplication, feed_dict={x_data: x_val}) 53 | print(str(a_val) + ' * ' + str(x_val) + ' = ' + str(mult_output)) 54 | 55 | #---------------------------------- 56 | # Create a nested gate: 57 | # f(x) = a * x + b 58 | # 59 | # a -- 60 | # | 61 | # |-- (multiply)-- 62 | # x --| | 63 | # |-- (add) --> output 64 | # b --| 65 | # 66 | # 67 | 68 | # Start a New Graph Session 69 | ops.reset_default_graph() 70 | sess = tf.Session() 71 | 72 | a = tf.Variable(tf.constant(1.)) 73 | b = tf.Variable(tf.constant(1.)) 74 | x_val = 5. 75 | x_data = tf.placeholder(dtype=tf.float32) 76 | 77 | two_gate = tf.add(tf.multiply(a, x_data), b) 78 | 79 | # Declare the loss function as the difference between 80 | # the output and a target value, 50. 81 | loss = tf.square(tf.subtract(two_gate, 50.)) 82 | 83 | # Initialize variables 84 | init = tf.global_variables_initializer() 85 | sess.run(init) 86 | 87 | # Declare optimizer 88 | my_opt = tf.train.GradientDescentOptimizer(0.01) 89 | train_step = my_opt.minimize(loss) 90 | 91 | # Run loop across gate 92 | print('\nOptimizing Two Gate Output to 50.') 93 | for i in range(10): 94 | sess.run(train_step, feed_dict={x_data: x_val}) 95 | a_val, b_val = (sess.run(a), sess.run(b)) 96 | two_gate_output = sess.run(two_gate, feed_dict={x_data: x_val}) 97 | print(str(a_val) + ' * ' + str(x_val) + ' + ' + str(b_val) + ' = ' + str(two_gate_output)) -------------------------------------------------------------------------------- /Chapter 06/improving_linear_regression.py: -------------------------------------------------------------------------------- 1 | # Improving Linear Regression with Neural Networks (Logistic Regression) 2 | # ---------------------------------- 3 | # 4 | # This function shows how to use Tensorflow to 5 | # solve logistic regression with a multiple layer neural network 6 | # y = sigmoid(A3 * sigmoid(A2* sigmoid(A1*x + b1) + b2) + b3) 7 | # 8 | # We will use the low birth weight data, specifically: 9 | # y = 0 or 1 = low birth weight 10 | # x = demographic and medical history data 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | import requests 16 | from tensorflow.python.framework import ops 17 | 18 | ops.reset_default_graph() 19 | 20 | # Create graph 21 | sess = tf.Session() 22 | 23 | birthdata_url = 'https://www.umass.edu/statdata/statdata/data/lowbwt.dat' 24 | birth_file = requests.get(birthdata_url) 25 | birth_data = birth_file.text.split('\r\n')[5:] 26 | birth_header = [x for x in birth_data[0].split(' ') if len(x) >= 1] 27 | birth_data = [[float(x) for x in y.split(' ') if len(x) >= 1] for y in birth_data[1:] if len(y) >= 1] 28 | # Pull out target variable 29 | y_vals = np.array([x[1] for x in birth_data]) 30 | # Pull out predictor variables (not id, not target, and not birthweight) 31 | x_vals = np.array([x[2:9] for x in birth_data]) 32 | 33 | # Split data into train/test = 80%/20% 34 | train_indices = np.random.choice(len(x_vals), round(len(x_vals) * 0.8), replace=False) 35 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 36 | x_vals_train = x_vals[train_indices] 37 | x_vals_test = x_vals[test_indices] 38 | y_vals_train = y_vals[train_indices] 39 | y_vals_test = y_vals[test_indices] 40 | 41 | 42 | # Normalize by column (min-max norm) 43 | def normalize_cols(m): 44 | col_max = m.max(axis=0) 45 | col_min = m.min(axis=0) 46 | return (m - col_min) / (col_max - col_min) 47 | 48 | 49 | x_vals_train = np.nan_to_num(normalize_cols(x_vals_train)) 50 | x_vals_test = np.nan_to_num(normalize_cols(x_vals_test)) 51 | 52 | # Declare batch size 53 | batch_size = 90 54 | 55 | # Initialize placeholders 56 | x_data = tf.placeholder(shape=[None, 7], dtype=tf.float32) 57 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 58 | 59 | 60 | # Create variable definition 61 | def init_variable(shape): 62 | return (tf.Variable(tf.random_normal(shape=shape))) 63 | 64 | 65 | # Create a logistic layer definition 66 | def logistic(input_layer, multiplication_weight, bias_weight, activation=True): 67 | linear_layer = tf.add(tf.matmul(input_layer, multiplication_weight), bias_weight) 68 | # We separate the activation at the end because the loss function will 69 | # implement the last sigmoid necessary 70 | if activation: 71 | return (tf.nn.sigmoid(linear_layer)) 72 | else: 73 | return (linear_layer) 74 | 75 | 76 | # First logistic layer (7 inputs to 7 hidden nodes) 77 | A1 = init_variable(shape=[7, 14]) 78 | b1 = init_variable(shape=[14]) 79 | logistic_layer1 = logistic(x_data, A1, b1) 80 | 81 | # Second logistic layer (7 hidden inputs to 5 hidden nodes) 82 | A2 = init_variable(shape=[14, 5]) 83 | b2 = init_variable(shape=[5]) 84 | logistic_layer2 = logistic(logistic_layer1, A2, b2) 85 | 86 | # Final output layer (5 hidden nodes to 1 output) 87 | A3 = init_variable(shape=[5, 1]) 88 | b3 = init_variable(shape=[1]) 89 | final_output = logistic(logistic_layer2, A3, b3, activation=False) 90 | 91 | # Declare loss function (Cross Entropy loss) 92 | loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=final_output, logits=y_target)) 93 | 94 | # Declare optimizer 95 | my_opt = tf.train.AdamOptimizer(learning_rate=0.005) 96 | train_step = my_opt.minimize(loss) 97 | 98 | # Initialize variables 99 | init = tf.global_variables_initializer() 100 | sess.run(init) 101 | 102 | # Actual Prediction 103 | prediction = tf.round(tf.nn.sigmoid(final_output)) 104 | predictions_correct = tf.cast(tf.equal(prediction, y_target), tf.float32) 105 | accuracy = tf.reduce_mean(predictions_correct) 106 | 107 | # Training loop 108 | loss_vec = [] 109 | train_acc = [] 110 | test_acc = [] 111 | for i in range(1500): 112 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 113 | rand_x = x_vals_train[rand_index] 114 | rand_y = np.transpose([y_vals_train[rand_index]]) 115 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 116 | 117 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 118 | loss_vec.append(temp_loss) 119 | temp_acc_train = sess.run(accuracy, feed_dict={x_data: x_vals_train, y_target: np.transpose([y_vals_train])}) 120 | train_acc.append(temp_acc_train) 121 | temp_acc_test = sess.run(accuracy, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])}) 122 | test_acc.append(temp_acc_test) 123 | if (i + 1) % 150 == 0: 124 | print('Loss = ' + str(temp_loss)) 125 | 126 | # Plot loss over time 127 | plt.plot(loss_vec, 'k-') 128 | plt.title('Cross Entropy Loss per Generation') 129 | plt.xlabel('Generation') 130 | plt.ylabel('Cross Entropy Loss') 131 | plt.show() 132 | 133 | # Plot train and test accuracy 134 | plt.plot(train_acc, 'k-', label='Train Set Accuracy') 135 | plt.plot(test_acc, 'r--', label='Test Set Accuracy') 136 | plt.title('Train and Test Accuracy') 137 | plt.xlabel('Generation') 138 | plt.ylabel('Accuracy') 139 | plt.legend(loc='lower right') 140 | plt.show() 141 | -------------------------------------------------------------------------------- /Chapter 06/single_hidden_layer_network.py: -------------------------------------------------------------------------------- 1 | # Implementing a one-layer Neural Network 2 | #--------------------------------------- 3 | # 4 | # We will illustrate how to create a one hidden layer NN 5 | # 6 | # We will use the iris data for this exercise 7 | # 8 | # We will build a one-hidden layer neural network 9 | # to predict the fourth attribute, Petal Width from 10 | # the other three (Sepal length, Sepal width, Petal length). 11 | 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import tensorflow as tf 15 | from sklearn import datasets 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | iris = datasets.load_iris() 20 | x_vals = np.array([x[0:3] for x in iris.data]) 21 | y_vals = np.array([x[3] for x in iris.data]) 22 | 23 | # Create graph session 24 | sess = tf.Session() 25 | 26 | # Set Seed 27 | seed = 3 28 | tf.set_random_seed(seed) 29 | np.random.seed(seed) 30 | 31 | # Split data into train/test = 80%/20% 32 | train_indices = np.random.choice(len(x_vals), int(round(len(x_vals)*0.8)), replace=False) 33 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 34 | x_vals_train = x_vals[train_indices] 35 | x_vals_test = x_vals[test_indices] 36 | y_vals_train = y_vals[train_indices] 37 | y_vals_test = y_vals[test_indices] 38 | 39 | # Normalize by column (min-max norm) 40 | def normalize_cols(m): 41 | col_max = m.max(axis=0) 42 | col_min = m.min(axis=0) 43 | return (m-col_min) / (col_max - col_min) 44 | 45 | x_vals_train = np.nan_to_num(normalize_cols(x_vals_train)) 46 | x_vals_test = np.nan_to_num(normalize_cols(x_vals_test)) 47 | 48 | # Declare batch size 49 | batch_size = 50 50 | 51 | # Initialize placeholders 52 | x_data = tf.placeholder(shape=[None, 3], dtype=tf.float32) 53 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 54 | 55 | # Create variables for both Neural Network Layers 56 | hidden_layer_nodes = 10 57 | A1 = tf.Variable(tf.random_normal(shape=[3,hidden_layer_nodes])) # inputs -> hidden nodes 58 | b1 = tf.Variable(tf.random_normal(shape=[hidden_layer_nodes])) # one biases for each hidden node 59 | A2 = tf.Variable(tf.random_normal(shape=[hidden_layer_nodes,1])) # hidden inputs -> 1 output 60 | b2 = tf.Variable(tf.random_normal(shape=[1])) # 1 bias for the output 61 | 62 | 63 | # Declare model operations 64 | hidden_output = tf.nn.relu(tf.add(tf.matmul(x_data, A1), b1)) 65 | final_output = tf.nn.relu(tf.add(tf.matmul(hidden_output, A2), b2)) 66 | 67 | # Declare loss function 68 | loss = tf.reduce_mean(tf.square(y_target - final_output)) 69 | 70 | # Declare optimizer 71 | my_opt = tf.train.GradientDescentOptimizer(0.005) 72 | train_step = my_opt.minimize(loss) 73 | 74 | # Initialize variables 75 | init = tf.global_variables_initializer() 76 | sess.run(init) 77 | 78 | # Training loop 79 | loss_vec = [] 80 | test_loss = [] 81 | for i in range(500): 82 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 83 | rand_x = x_vals_train[rand_index] 84 | rand_y = np.transpose([y_vals_train[rand_index]]) 85 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 86 | 87 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 88 | loss_vec.append(np.sqrt(temp_loss)) 89 | 90 | test_temp_loss = sess.run(loss, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])}) 91 | test_loss.append(np.sqrt(test_temp_loss)) 92 | if (i+1)%50==0: 93 | print('Generation: ' + str(i+1) + '. Loss = ' + str(temp_loss)) 94 | 95 | 96 | # Plot loss (MSE) over time 97 | plt.plot(loss_vec, 'k-', label='Train Loss') 98 | plt.plot(test_loss, 'r--', label='Test Loss') 99 | plt.title('Loss (MSE) per Generation') 100 | plt.xlabel('Generation') 101 | plt.ylabel('Loss') 102 | plt.legend(loc='upper right') 103 | plt.show() -------------------------------------------------------------------------------- /Chapter 06/tic_tac_toe/base_tic_tac_toe_moves.csv: -------------------------------------------------------------------------------- 1 | 0,0,0,0,-1,0,0,0,0,0 2 | 0,-1,0,0,0,0,0,0,0,0 3 | 0,0,0,0,0,-1,0,0,0,6 4 | -1,0,0,0,0,0,0,0,0,4 5 | 0,0,0,0,0,0,1,-1,-1,3 6 | 0,-1,0,0,1,0,0,0,-1,0 7 | 0,-1,1,0,0,-1,0,0,0,7 8 | -1,0,0,0,-1,0,0,0,1,6 9 | 0,0,1,0,0,-1,-1,0,0,4 10 | 0,0,-1,0,0,0,0,-1,1,4 11 | 1,0,0,-1,0,0,0,-1,0,2 12 | 0,0,-1,0,1,0,-1,0,0,5 13 | -1,0,0,1,-1,-1,0,0,1,6 14 | -1,1,-1,0,1,0,0,1,0,8 15 | 0,0,0,-1,0,1,1,-1,-1,1 16 | -1,1,0,0,0,-1,0,-1,1,3 17 | 0,-1,1,0,1,-1,-1,0,0,8 18 | 0,0,-1,1,0,-1,0,-1,1,0 19 | 1,-1,0,0,-1,0,0,0,0,7 20 | 1,0,-1,0,-1,0,0,0,0,6 21 | 1,0,0,0,-1,0,-1,0,0,2 22 | 1,0,0,0,-1,-1,0,0,0,3 23 | 1,0,0,0,-1,0,0,0,-1,6 24 | 1,-1,0,-1,-1,0,0,1,0,5 25 | 1,-1,0,0,-1,0,-1,1,0,2 26 | 1,-1,-1,0,-1,0,0,1,0,6 27 | 1,-1,0,0,-1,-1,0,1,0,3 28 | 1,0,-1,-1,-1,0,1,0,0,8 29 | 1,-1,1,0,-1,0,-1,0,0,7 30 | 1,0,0,1,-1,-1,-1,0,0,2 31 | 1,0,0,-1,-1,0,1,0,-1,5 32 | -------------------------------------------------------------------------------- /Chapter 06/using_a_multiple_layer_network.py: -------------------------------------------------------------------------------- 1 | # Using a Multiple Layer Network 2 | #--------------------------------------- 3 | # 4 | # We will illustrate how to use a Multiple 5 | # Layer Network in Tensorflow 6 | # 7 | # Low Birthrate data: 8 | # 9 | #Columns Variable Abbreviation 10 | #----------------------------------------------------------------------------- 11 | # Identification Code ID 12 | # Low Birth Weight (0 = Birth Weight >= 2500g, LOW 13 | # 1 = Birth Weight < 2500g) 14 | # Age of the Mother in Years AGE 15 | # Weight in Pounds at the Last Menstrual Period LWT 16 | # Race (1 = White, 2 = Black, 3 = Other) RACE 17 | # Smoking Status During Pregnancy (1 = Yes, 0 = No) SMOKE 18 | # History of Premature Labor (0 = None 1 = One, etc.) PTL 19 | # History of Hypertension (1 = Yes, 0 = No) HT 20 | # Presence of Uterine Irritability (1 = Yes, 0 = No) UI 21 | # Number of Physician Visits During the First Trimester FTV 22 | # (0 = None, 1 = One, 2 = Two, etc.) 23 | # Birth Weight in Grams BWT 24 | #------------------------------ 25 | # The multiple neural network layer we will create will be composed of 26 | # three fully connected hidden layers, with node sizes 25, 10, and 3 27 | 28 | import tensorflow as tf 29 | import matplotlib.pyplot as plt 30 | import requests 31 | import numpy as np 32 | from tensorflow.python.framework import ops 33 | ops.reset_default_graph() 34 | 35 | # Set Seed 36 | seed = 3 37 | tf.set_random_seed(seed) 38 | np.random.seed(seed) 39 | 40 | 41 | birthdata_url = 'https://www.umass.edu/statdata/statdata/data/lowbwt.dat' 42 | birth_file = requests.get(birthdata_url) 43 | birth_data = birth_file.text.split('\r\n')[5:] 44 | birth_header = [x for x in birth_data[0].split(' ') if len(x)>=1] 45 | birth_data = [[float(x) for x in y.split(' ') if len(x)>=1] for y in birth_data[1:] if len(y)>=1] 46 | 47 | 48 | batch_size = 100 49 | 50 | # Extract y-target (birth weight) 51 | y_vals = np.array([x[10] for x in birth_data]) 52 | 53 | # Filter for features of interest 54 | cols_of_interest = ['AGE', 'LWT', 'RACE', 'SMOKE', 'PTL', 'HT', 'UI', 'FTV'] 55 | x_vals = np.array([[x[ix] for ix, feature in enumerate(birth_header) if feature in cols_of_interest] for x in birth_data]) 56 | 57 | # Create graph session 58 | sess = tf.Session() 59 | 60 | # Split data into train/test = 80%/20% 61 | train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False) 62 | test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices))) 63 | x_vals_train = x_vals[train_indices] 64 | x_vals_test = x_vals[test_indices] 65 | y_vals_train = y_vals[train_indices] 66 | y_vals_test = y_vals[test_indices] 67 | 68 | 69 | # Normalize by column (min-max norm to be between 0 and 1) 70 | def normalize_cols(m): 71 | col_max = m.max(axis=0) 72 | col_min = m.min(axis=0) 73 | return (m-col_min) / (col_max - col_min) 74 | 75 | x_vals_train = np.nan_to_num(normalize_cols(x_vals_train)) 76 | x_vals_test = np.nan_to_num(normalize_cols(x_vals_test)) 77 | 78 | 79 | # Define Variable Functions (weights and bias) 80 | def init_weight(shape, st_dev): 81 | weight = tf.Variable(tf.random_normal(shape, stddev=st_dev)) 82 | return(weight) 83 | 84 | 85 | def init_bias(shape, st_dev): 86 | bias = tf.Variable(tf.random_normal(shape, stddev=st_dev)) 87 | return(bias) 88 | 89 | 90 | # Create Placeholders 91 | x_data = tf.placeholder(shape=[None, 8], dtype=tf.float32) 92 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 93 | 94 | 95 | # Create a fully connected layer: 96 | def fully_connected(input_layer, weights, biases): 97 | layer = tf.add(tf.matmul(input_layer, weights), biases) 98 | return(tf.nn.relu(layer)) 99 | 100 | 101 | #--------Create the first layer (25 hidden nodes)-------- 102 | weight_1 = init_weight(shape=[8, 25], st_dev=10.0) 103 | bias_1 = init_bias(shape=[25], st_dev=10.0) 104 | layer_1 = fully_connected(x_data, weight_1, bias_1) 105 | 106 | #--------Create second layer (10 hidden nodes)-------- 107 | weight_2 = init_weight(shape=[25, 10], st_dev=10.0) 108 | bias_2 = init_bias(shape=[10], st_dev=10.0) 109 | layer_2 = fully_connected(layer_1, weight_2, bias_2) 110 | 111 | 112 | #--------Create third layer (3 hidden nodes)-------- 113 | weight_3 = init_weight(shape=[10, 3], st_dev=10.0) 114 | bias_3 = init_bias(shape=[3], st_dev=10.0) 115 | layer_3 = fully_connected(layer_2, weight_3, bias_3) 116 | 117 | 118 | #--------Create output layer (1 output value)-------- 119 | weight_4 = init_weight(shape=[3, 1], st_dev=10.0) 120 | bias_4 = init_bias(shape=[1], st_dev=10.0) 121 | final_output = fully_connected(layer_3, weight_4, bias_4) 122 | 123 | # Declare loss function (L1) 124 | loss = tf.reduce_mean(tf.abs(y_target - final_output)) 125 | 126 | # Declare optimizer 127 | my_opt = tf.train.AdamOptimizer(0.05) 128 | train_step = my_opt.minimize(loss) 129 | 130 | # Initialize Variables 131 | init = tf.global_variables_initializer() 132 | sess.run(init) 133 | 134 | # Training loop 135 | loss_vec = [] 136 | test_loss = [] 137 | for i in range(200): 138 | rand_index = np.random.choice(len(x_vals_train), size=batch_size) 139 | rand_x = x_vals_train[rand_index] 140 | rand_y = np.transpose([y_vals_train[rand_index]]) 141 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 142 | 143 | temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 144 | loss_vec.append(temp_loss) 145 | 146 | test_temp_loss = sess.run(loss, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])}) 147 | test_loss.append(test_temp_loss) 148 | if (i+1)%25==0: 149 | print('Generation: ' + str(i+1) + '. Loss = ' + str(temp_loss)) 150 | 151 | 152 | # Plot loss over time 153 | plt.plot(loss_vec, 'k-', label='Train Loss') 154 | plt.plot(test_loss, 'r--', label='Test Loss') 155 | plt.title('Loss per Generation') 156 | plt.xlabel('Generation') 157 | plt.ylabel('Loss') 158 | plt.legend(loc="upper right") 159 | plt.show() 160 | 161 | # Find the % classified correctly above/below the cutoff of 2500 g 162 | # >= 2500 g = 0 163 | # < 2500 g = 1 164 | actuals = np.array([x[1] for x in birth_data]) 165 | test_actuals = actuals[test_indices] 166 | train_actuals = actuals[train_indices] 167 | 168 | test_preds = [x[0] for x in sess.run(final_output, feed_dict={x_data: x_vals_test})] 169 | train_preds = [x[0] for x in sess.run(final_output, feed_dict={x_data: x_vals_train})] 170 | test_preds = np.array([1.0 if x<2500.0 else 0.0 for x in test_preds]) 171 | train_preds = np.array([1.0 if x<2500.0 else 0.0 for x in train_preds]) 172 | 173 | # Print out accuracies 174 | test_acc = np.mean([x==y for x,y in zip(test_preds, test_actuals)]) 175 | train_acc = np.mean([x==y for x,y in zip(train_preds, train_actuals)]) 176 | print('On predicting the category of low birthweight from regression output (<2500g):') 177 | print('Test Accuracy: {}'.format(test_acc)) 178 | print('Train Accuracy: {}'.format(train_acc)) -------------------------------------------------------------------------------- /Chapter 07/bag_of_words.py: -------------------------------------------------------------------------------- 1 | # Working with Bag of Words 2 | # --------------------------------------- 3 | # 4 | # In this example, we will download and preprocess the ham/spam 5 | # text data. We will then use a one-hot-encoding to make a 6 | # bag of words set of features to use in logistic regression. 7 | # 8 | # We will use these one-hot-vectors for logistic regression to 9 | # predict if a text is spam or ham. 10 | 11 | import tensorflow as tf 12 | import matplotlib.pyplot as plt 13 | import os 14 | import numpy as np 15 | import csv 16 | import string 17 | import requests 18 | import io 19 | from zipfile import ZipFile 20 | from tensorflow.contrib import learn 21 | from tensorflow.python.framework import ops 22 | 23 | ops.reset_default_graph() 24 | 25 | # Start a graph session 26 | sess = tf.Session() 27 | 28 | # Check if data was downloaded, otherwise download it and save for future use 29 | save_file_name = os.path.join('temp', 'temp_spam_data.csv') 30 | if os.path.isfile(save_file_name): 31 | text_data = [] 32 | with open(save_file_name, 'r') as temp_output_file: 33 | reader = csv.reader(temp_output_file) 34 | for row in reader: 35 | text_data.append(row) 36 | else: 37 | zip_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip' 38 | r = requests.get(zip_url) 39 | z = ZipFile(io.BytesIO(r.content)) 40 | file = z.read('SMSSpamCollection') 41 | # Format Data 42 | text_data = file.decode() 43 | text_data = text_data.encode('ascii', errors='ignore') 44 | text_data = text_data.decode().split('\n') 45 | text_data = [x.split('\t') for x in text_data if len(x) >= 1] 46 | 47 | # And write to csv 48 | with open(save_file_name, 'w') as temp_output_file: 49 | writer = csv.writer(temp_output_file) 50 | writer.writerows(text_data) 51 | 52 | texts = [x[1] for x in text_data] 53 | target = [x[0] for x in text_data] 54 | 55 | # Relabel 'spam' as 1, 'ham' as 0 56 | target = [1 if x == 'spam' else 0 for x in target] 57 | 58 | # Normalize text 59 | # Lower case 60 | texts = [x.lower() for x in texts] 61 | 62 | # Remove punctuation 63 | texts = [''.join(c for c in x if c not in string.punctuation) for x in texts] 64 | 65 | # Remove numbers 66 | texts = [''.join(c for c in x if c not in '0123456789') for x in texts] 67 | 68 | # Trim extra whitespace 69 | texts = [' '.join(x.split()) for x in texts] 70 | 71 | # Plot histogram of text lengths 72 | text_lengths = [len(x.split()) for x in texts] 73 | text_lengths = [x for x in text_lengths if x < 50] 74 | plt.hist(text_lengths, bins=25) 75 | plt.title('Histogram of # of Words in Texts') 76 | 77 | # Choose max text word length at 25 78 | sentence_size = 25 79 | min_word_freq = 3 80 | 81 | # Setup vocabulary processor 82 | vocab_processor = learn.preprocessing.VocabularyProcessor(sentence_size, min_frequency=min_word_freq) 83 | 84 | # Have to fit transform to get length of unique words. 85 | vocab_processor.fit_transform(texts) 86 | embedding_size = len(vocab_processor.vocabulary_) 87 | 88 | # Split up data set into train/test 89 | train_indices = np.random.choice(len(texts), int(round(len(texts) * 0.8)), replace=False) 90 | test_indices = np.array(list(set(range(len(texts))) - set(train_indices))) 91 | texts_train = [x for ix, x in enumerate(texts) if ix in train_indices] 92 | texts_test = [x for ix, x in enumerate(texts) if ix in test_indices] 93 | target_train = [x for ix, x in enumerate(target) if ix in train_indices] 94 | target_test = [x for ix, x in enumerate(target) if ix in test_indices] 95 | 96 | # Setup Index Matrix for one-hot-encoding 97 | identity_mat = tf.diag(tf.ones(shape=[embedding_size])) 98 | 99 | # Create variables for logistic regression 100 | A = tf.Variable(tf.random_normal(shape=[embedding_size, 1])) 101 | b = tf.Variable(tf.random_normal(shape=[1, 1])) 102 | 103 | # Initialize placeholders 104 | x_data = tf.placeholder(shape=[sentence_size], dtype=tf.int32) 105 | y_target = tf.placeholder(shape=[1, 1], dtype=tf.float32) 106 | 107 | # Text-Vocab Embedding 108 | x_embed = tf.nn.embedding_lookup(identity_mat, x_data) 109 | x_col_sums = tf.reduce_sum(x_embed, 0) 110 | 111 | # Declare model operations 112 | x_col_sums_2D = tf.expand_dims(x_col_sums, 0) 113 | model_output = tf.add(tf.matmul(x_col_sums_2D, A), b) 114 | 115 | # Declare loss function (Cross Entropy loss) 116 | loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=model_output, logits=y_target)) 117 | 118 | # Prediction operation 119 | prediction = tf.sigmoid(model_output) 120 | 121 | # Declare optimizer 122 | my_opt = tf.train.GradientDescentOptimizer(0.001) 123 | train_step = my_opt.minimize(loss) 124 | 125 | # Intitialize Variables 126 | init = tf.global_variables_initializer() 127 | sess.run(init) 128 | 129 | # Start Logistic Regression 130 | print('Starting Training Over {} Sentences.'.format(len(texts_train))) 131 | loss_vec = [] 132 | train_acc_all = [] 133 | train_acc_avg = [] 134 | for ix, t in enumerate(vocab_processor.fit_transform(texts_train)): 135 | y_data = [[target_train[ix]]] 136 | 137 | sess.run(train_step, feed_dict={x_data: t, y_target: y_data}) 138 | temp_loss = sess.run(loss, feed_dict={x_data: t, y_target: y_data}) 139 | loss_vec.append(temp_loss) 140 | 141 | if (ix + 1) % 10 == 0: 142 | print('Training Observation #' + str(ix + 1) + ': Loss = ' + str(temp_loss)) 143 | 144 | # Keep trailing average of past 50 observations accuracy 145 | # Get prediction of single observation 146 | [[temp_pred]] = sess.run(prediction, feed_dict={x_data: t, y_target: y_data}) 147 | # Get True/False if prediction is accurate 148 | train_acc_temp = target_train[ix] == np.round(temp_pred) 149 | train_acc_all.append(train_acc_temp) 150 | if len(train_acc_all) >= 50: 151 | train_acc_avg.append(np.mean(train_acc_all[-50:])) 152 | 153 | # Get test set accuracy 154 | print('Getting Test Set Accuracy For {} Sentences.'.format(len(texts_test))) 155 | test_acc_all = [] 156 | for ix, t in enumerate(vocab_processor.fit_transform(texts_test)): 157 | y_data = [[target_test[ix]]] 158 | 159 | if (ix + 1) % 50 == 0: 160 | print('Test Observation #' + str(ix + 1)) 161 | 162 | # Keep trailing average of past 50 observations accuracy 163 | # Get prediction of single observation 164 | [[temp_pred]] = sess.run(prediction, feed_dict={x_data: t, y_target: y_data}) 165 | # Get True/False if prediction is accurate 166 | test_acc_temp = target_test[ix] == np.round(temp_pred) 167 | test_acc_all.append(test_acc_temp) 168 | 169 | print('\nOverall Test Accuracy: {}'.format(np.mean(test_acc_all))) 170 | 171 | # Plot training accuracy over time 172 | plt.plot(range(len(train_acc_avg)), train_acc_avg, 'k-', label='Train Accuracy') 173 | plt.title('Avg Training Acc Over Past 50 Generations') 174 | plt.xlabel('Generation') 175 | plt.ylabel('Training Accuracy') 176 | plt.show() 177 | -------------------------------------------------------------------------------- /Chapter 07/text_helpers.py: -------------------------------------------------------------------------------- 1 | # Text Helper Functions 2 | #--------------------------------------- 3 | # 4 | # We pull out text helper functions to reduce redundant code 5 | 6 | import string 7 | import os 8 | import urllib.request 9 | import io 10 | import tarfile 11 | import collections 12 | import numpy as np 13 | 14 | # Normalize text 15 | def normalize_text(texts, stops): 16 | # Lower case 17 | texts = [x.lower() for x in texts] 18 | 19 | # Remove punctuation 20 | texts = [''.join(c for c in x if c not in string.punctuation) for x in texts] 21 | 22 | # Remove numbers 23 | texts = [''.join(c for c in x if c not in '0123456789') for x in texts] 24 | 25 | # Remove stopwords 26 | texts = [' '.join([word for word in x.split() if word not in (stops)]) for x in texts] 27 | 28 | # Trim extra whitespace 29 | texts = [' '.join(x.split()) for x in texts] 30 | 31 | return(texts) 32 | 33 | 34 | # Build dictionary of words 35 | def build_dictionary(sentences, vocabulary_size): 36 | # Turn sentences (list of strings) into lists of words 37 | split_sentences = [s.split() for s in sentences] 38 | words = [x for sublist in split_sentences for x in sublist] 39 | 40 | # Initialize list of [word, word_count] for each word, starting with unknown 41 | count = [['RARE', -1]] 42 | 43 | # Now add most frequent words, limited to the N-most frequent (N=vocabulary size) 44 | count.extend(collections.Counter(words).most_common(vocabulary_size-1)) 45 | 46 | # Now create the dictionary 47 | word_dict = {} 48 | # For each word, that we want in the dictionary, add it, then make it 49 | # the value of the prior dictionary length 50 | for word, word_count in count: 51 | word_dict[word] = len(word_dict) 52 | 53 | return(word_dict) 54 | 55 | 56 | # Turn text data into lists of integers from dictionary 57 | def text_to_numbers(sentences, word_dict): 58 | # Initialize the returned data 59 | data = [] 60 | for sentence in sentences: 61 | sentence_data = [] 62 | # For each word, either use selected index or rare word index 63 | for word in sentence.split(): 64 | if word in word_dict: 65 | word_ix = word_dict[word] 66 | else: 67 | word_ix = 0 68 | sentence_data.append(word_ix) 69 | data.append(sentence_data) 70 | return(data) 71 | 72 | 73 | # Generate data randomly (N words behind, target, N words ahead) 74 | def generate_batch_data(sentences, batch_size, window_size, method='skip_gram'): 75 | # Fill up data batch 76 | batch_data = [] 77 | label_data = [] 78 | while len(batch_data) < batch_size: 79 | # select random sentence to start 80 | rand_sentence_ix = int(np.random.choice(len(sentences), size=1)) 81 | rand_sentence = sentences[rand_sentence_ix] 82 | # Generate consecutive windows to look at 83 | window_sequences = [rand_sentence[max((ix-window_size),0):(ix+window_size+1)] for ix, x in enumerate(rand_sentence)] 84 | # Denote which element of each window is the center word of interest 85 | label_indices = [ix if ix= 1] 51 | 52 | # And write to csv 53 | with open(save_file_name, 'w') as temp_output_file: 54 | writer = csv.writer(temp_output_file) 55 | writer.writerows(text_data) 56 | 57 | texts = [x[1] for x in text_data] 58 | target = [x[0] for x in text_data] 59 | 60 | # Relabel 'spam' as 1, 'ham' as 0 61 | target = [1. if x == 'spam' else 0. for x in target] 62 | 63 | # Normalize text 64 | # Lower case 65 | texts = [x.lower() for x in texts] 66 | 67 | # Remove punctuation 68 | texts = [''.join(c for c in x if c not in string.punctuation) for x in texts] 69 | 70 | # Remove numbers 71 | texts = [''.join(c for c in x if c not in '0123456789') for x in texts] 72 | 73 | # Trim extra whitespace 74 | texts = [' '.join(x.split()) for x in texts] 75 | 76 | 77 | # Define tokenizer 78 | def tokenizer(text): 79 | words = nltk.word_tokenize(text) 80 | return words 81 | 82 | 83 | # Create TF-IDF of texts 84 | tfidf = TfidfVectorizer(tokenizer=tokenizer, stop_words='english', max_features=max_features) 85 | sparse_tfidf_texts = tfidf.fit_transform(texts) 86 | 87 | # Split up data set into train/test 88 | train_indices = np.random.choice(sparse_tfidf_texts.shape[0], round(0.8 * sparse_tfidf_texts.shape[0]), replace=False) 89 | test_indices = np.array(list(set(range(sparse_tfidf_texts.shape[0])) - set(train_indices))) 90 | texts_train = sparse_tfidf_texts[train_indices] 91 | texts_test = sparse_tfidf_texts[test_indices] 92 | target_train = np.array([x for ix, x in enumerate(target) if ix in train_indices]) 93 | target_test = np.array([x for ix, x in enumerate(target) if ix in test_indices]) 94 | 95 | # Create variables for logistic regression 96 | A = tf.Variable(tf.random_normal(shape=[max_features, 1])) 97 | b = tf.Variable(tf.random_normal(shape=[1, 1])) 98 | 99 | # Initialize placeholders 100 | x_data = tf.placeholder(shape=[None, max_features], dtype=tf.float32) 101 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 102 | 103 | # Declare logistic model (sigmoid in loss function) 104 | model_output = tf.add(tf.matmul(x_data, A), b) 105 | 106 | # Declare loss function (Cross Entropy loss) 107 | loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=model_output, logits=y_target)) 108 | 109 | # Actual Prediction 110 | prediction = tf.round(tf.sigmoid(model_output)) 111 | predictions_correct = tf.cast(tf.equal(prediction, y_target), tf.float32) 112 | accuracy = tf.reduce_mean(predictions_correct) 113 | 114 | # Declare optimizer 115 | my_opt = tf.train.GradientDescentOptimizer(0.0025) 116 | train_step = my_opt.minimize(loss) 117 | 118 | # Intitialize Variables 119 | init = tf.global_variables_initializer() 120 | sess.run(init) 121 | 122 | # Start Logistic Regression 123 | train_loss = [] 124 | test_loss = [] 125 | train_acc = [] 126 | test_acc = [] 127 | i_data = [] 128 | for i in range(10000): 129 | rand_index = np.random.choice(texts_train.shape[0], size=batch_size) 130 | rand_x = texts_train[rand_index].todense() 131 | rand_y = np.transpose([target_train[rand_index]]) 132 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 133 | 134 | # Only record loss and accuracy every 100 generations 135 | if (i + 1) % 100 == 0: 136 | i_data.append(i + 1) 137 | train_loss_temp = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 138 | train_loss.append(train_loss_temp) 139 | 140 | test_loss_temp = sess.run(loss, feed_dict={x_data: texts_test.todense(), y_target: np.transpose([target_test])}) 141 | test_loss.append(test_loss_temp) 142 | 143 | train_acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x, y_target: rand_y}) 144 | train_acc.append(train_acc_temp) 145 | 146 | test_acc_temp = sess.run(accuracy, 147 | feed_dict={x_data: texts_test.todense(), y_target: np.transpose([target_test])}) 148 | test_acc.append(test_acc_temp) 149 | if (i + 1) % 500 == 0: 150 | acc_and_loss = [i + 1, train_loss_temp, test_loss_temp, train_acc_temp, test_acc_temp] 151 | acc_and_loss = [np.round(x, 2) for x in acc_and_loss] 152 | print('Generation # {}. Train Loss (Test Loss): {:.2f} ({:.2f}). Train Acc (Test Acc): {:.2f} ({:.2f})'.format( 153 | *acc_and_loss)) 154 | 155 | # Plot loss over time 156 | plt.plot(i_data, train_loss, 'k-', label='Train Loss') 157 | plt.plot(i_data, test_loss, 'r--', label='Test Loss', linewidth=4) 158 | plt.title('Cross Entropy Loss per Generation') 159 | plt.xlabel('Generation') 160 | plt.ylabel('Cross Entropy Loss') 161 | plt.legend(loc='upper right') 162 | plt.show() 163 | 164 | # Plot train and test accuracy 165 | plt.plot(i_data, train_acc, 'k-', label='Train Set Accuracy') 166 | plt.plot(i_data, test_acc, 'r--', label='Test Set Accuracy', linewidth=4) 167 | plt.title('Train and Test Accuracy') 168 | plt.xlabel('Generation') 169 | plt.ylabel('Accuracy') 170 | plt.legend(loc='lower right') 171 | plt.show() 172 | -------------------------------------------------------------------------------- /Chapter 07/using_word2vec.py: -------------------------------------------------------------------------------- 1 | # Using Word2Vec for prediction 2 | #--------------------------------------- 3 | # 4 | # In this example, we will load our prior CBOW trained embeddings 5 | # to perform logistic regression model for movie review predictions 6 | # 7 | # From this data set we will compute/fit the CBOW model of 8 | # the Word2Vec Algorithm 9 | import tensorflow as tf 10 | import matplotlib.pyplot as plt 11 | import numpy as np 12 | import random 13 | import os 14 | import pickle 15 | import string 16 | import requests 17 | import collections 18 | import io 19 | import tarfile 20 | import urllib.request 21 | import text_helpers 22 | from nltk.corpus import stopwords 23 | from tensorflow.python.framework import ops 24 | ops.reset_default_graph() 25 | 26 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 27 | 28 | # Start a graph session 29 | sess = tf.Session() 30 | 31 | # Declare model parameters 32 | embedding_size = 200 33 | vocabulary_size = 2000 34 | batch_size = 100 35 | max_words = 100 36 | 37 | # Declare stop words 38 | stops = stopwords.words('english') 39 | 40 | # Load Data 41 | print('Loading Data') 42 | data_folder_name = 'temp' 43 | texts, target = text_helpers.load_movie_data(data_folder_name) 44 | 45 | # Normalize text 46 | print('Normalizing Text Data') 47 | texts = text_helpers.normalize_text(texts, stops) 48 | 49 | # Texts must contain at least 3 words 50 | target = [target[ix] for ix, x in enumerate(texts) if len(x.split()) > 2] 51 | texts = [x for x in texts if len(x.split()) > 2] 52 | 53 | # Split up data set into train/test 54 | train_indices = np.random.choice(len(target), int(round(0.8*len(target))), replace=False) 55 | test_indices = np.array(list(set(range(len(target))) - set(train_indices))) 56 | texts_train = [x for ix, x in enumerate(texts) if ix in train_indices] 57 | texts_test = [x for ix, x in enumerate(texts) if ix in test_indices] 58 | target_train = np.array([x for ix, x in enumerate(target) if ix in train_indices]) 59 | target_test = np.array([x for ix, x in enumerate(target) if ix in test_indices]) 60 | 61 | # Load dictionary and embedding matrix 62 | dict_file = os.path.join(data_folder_name, 'movie_vocab.pkl') 63 | word_dictionary = pickle.load(open(dict_file, 'rb')) 64 | 65 | # Convert texts to lists of indices 66 | text_data_train = np.array(text_helpers.text_to_numbers(texts_train, word_dictionary)) 67 | text_data_test = np.array(text_helpers.text_to_numbers(texts_test, word_dictionary)) 68 | 69 | # Pad/crop movie reviews to specific length 70 | text_data_train = np.array([x[0:max_words] for x in [y+[0]*max_words for y in text_data_train]]) 71 | text_data_test = np.array([x[0:max_words] for x in [y+[0]*max_words for y in text_data_test]]) 72 | 73 | print('Creating Model') 74 | # Define Embeddings: 75 | embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) 76 | 77 | # Define model: 78 | # Create variables for logistic regression 79 | A = tf.Variable(tf.random_normal(shape=[embedding_size,1])) 80 | b = tf.Variable(tf.random_normal(shape=[1,1])) 81 | 82 | # Initialize placeholders 83 | x_data = tf.placeholder(shape=[None, max_words], dtype=tf.int32) 84 | y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32) 85 | 86 | # Lookup embeddings vectors 87 | embed = tf.nn.embedding_lookup(embeddings, x_data) 88 | # Take average of all word embeddings in documents 89 | embed_avg = tf.reduce_mean(embed, 1) 90 | 91 | # Declare logistic model (sigmoid in loss function) 92 | model_output = tf.add(tf.matmul(embed_avg, A), b) 93 | 94 | # Declare loss function (Cross Entropy loss) 95 | loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=model_output, logits=y_target)) 96 | 97 | # Actual Prediction 98 | prediction = tf.round(tf.sigmoid(model_output)) 99 | predictions_correct = tf.cast(tf.equal(prediction, y_target), tf.float32) 100 | accuracy = tf.reduce_mean(predictions_correct) 101 | 102 | # Declare optimizer 103 | my_opt = tf.train.AdagradOptimizer(0.005) 104 | train_step = my_opt.minimize(loss) 105 | 106 | # Intitialize Variables 107 | init = tf.global_variables_initializer() 108 | sess.run(init) 109 | 110 | # Load model embeddings 111 | model_checkpoint_path = os.path.join(data_folder_name,'cbow_movie_embeddings.ckpt') 112 | saver = tf.train.Saver({"embeddings": embeddings}) 113 | saver.restore(sess, model_checkpoint_path) 114 | 115 | 116 | # Start Logistic Regression 117 | print('Starting Model Training') 118 | train_loss = [] 119 | test_loss = [] 120 | train_acc = [] 121 | test_acc = [] 122 | i_data = [] 123 | for i in range(10000): 124 | rand_index = np.random.choice(text_data_train.shape[0], size=batch_size) 125 | rand_x = text_data_train[rand_index] 126 | rand_y = np.transpose([target_train[rand_index]]) 127 | sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y}) 128 | 129 | # Only record loss and accuracy every 100 generations 130 | if (i+1)%100==0: 131 | i_data.append(i+1) 132 | train_loss_temp = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y}) 133 | train_loss.append(train_loss_temp) 134 | 135 | test_loss_temp = sess.run(loss, feed_dict={x_data: text_data_test, y_target: np.transpose([target_test])}) 136 | test_loss.append(test_loss_temp) 137 | 138 | train_acc_temp = sess.run(accuracy, feed_dict={x_data: rand_x, y_target: rand_y}) 139 | train_acc.append(train_acc_temp) 140 | 141 | test_acc_temp = sess.run(accuracy, feed_dict={x_data: text_data_test, y_target: np.transpose([target_test])}) 142 | test_acc.append(test_acc_temp) 143 | if (i+1)%500==0: 144 | acc_and_loss = [i+1, train_loss_temp, test_loss_temp, train_acc_temp, test_acc_temp] 145 | acc_and_loss = [np.round(x,2) for x in acc_and_loss] 146 | print('Generation # {}. Train Loss (Test Loss): {:.2f} ({:.2f}). Train Acc (Test Acc): {:.2f} ({:.2f})'.format(*acc_and_loss)) 147 | 148 | 149 | # Plot loss over time 150 | plt.plot(i_data, train_loss, 'k-', label='Train Loss') 151 | plt.plot(i_data, test_loss, 'r--', label='Test Loss', linewidth=4) 152 | plt.title('Cross Entropy Loss per Generation') 153 | plt.xlabel('Generation') 154 | plt.ylabel('Cross Entropy Loss') 155 | plt.legend(loc='upper right') 156 | plt.show() 157 | 158 | # Plot train and test accuracy 159 | plt.plot(i_data, train_acc, 'k-', label='Train Set Accuracy') 160 | plt.plot(i_data, test_acc, 'r--', label='Test Set Accuracy', linewidth=4) 161 | plt.title('Train and Test Accuracy') 162 | plt.xlabel('Generation') 163 | plt.ylabel('Accuracy') 164 | plt.legend(loc='lower right') 165 | plt.show() -------------------------------------------------------------------------------- /Chapter 07/word2vec_cbow.py: -------------------------------------------------------------------------------- 1 | # Word2Vec: CBOW Model (Continuous Bag of Words) 2 | #--------------------------------------- 3 | # 4 | # In this example, we will download and preprocess the movie 5 | # review data. 6 | # 7 | # From this data set we will compute/fit the CBOW model of 8 | # the Word2Vec Algorithm 9 | import tensorflow as tf 10 | import matplotlib.pyplot as plt 11 | import numpy as np 12 | import random 13 | import os 14 | import pickle 15 | import string 16 | import requests 17 | import collections 18 | import io 19 | import tarfile 20 | import urllib.request 21 | import text_helpers 22 | from nltk.corpus import stopwords 23 | from tensorflow.python.framework import ops 24 | ops.reset_default_graph() 25 | 26 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 27 | 28 | # Make a saving directory if it doesn't exist 29 | data_folder_name = 'temp' 30 | if not os.path.exists(data_folder_name): 31 | os.makedirs(data_folder_name) 32 | 33 | # Start a graph session 34 | sess = tf.Session() 35 | 36 | # Declare model parameters 37 | batch_size = 500 38 | embedding_size = 200 39 | vocabulary_size = 2000 40 | generations = 50000 41 | model_learning_rate = 0.001 42 | 43 | num_sampled = int(batch_size/2) # Number of negative examples to sample. 44 | window_size = 3 # How many words to consider left and right. 45 | 46 | # Add checkpoints to training 47 | save_embeddings_every = 5000 48 | print_valid_every = 5000 49 | print_loss_every = 100 50 | 51 | # Declare stop words 52 | stops = stopwords.words('english') 53 | 54 | # We pick some test words. We are expecting synonyms to appear 55 | valid_words = ['love', 'hate', 'happy', 'sad', 'man', 'woman'] 56 | # Later we will have to transform these into indices 57 | 58 | # Load the movie review data 59 | print('Loading Data') 60 | texts, target = text_helpers.load_movie_data(data_folder_name) 61 | 62 | # Normalize text 63 | print('Normalizing Text Data') 64 | texts = text_helpers.normalize_text(texts, stops) 65 | 66 | # Texts must contain at least 3 words 67 | target = [target[ix] for ix, x in enumerate(texts) if len(x.split()) > 2] 68 | texts = [x for x in texts if len(x.split()) > 2] 69 | 70 | # Build our data set and dictionaries 71 | print('Creating Dictionary') 72 | word_dictionary = text_helpers.build_dictionary(texts, vocabulary_size) 73 | word_dictionary_rev = dict(zip(word_dictionary.values(), word_dictionary.keys())) 74 | text_data = text_helpers.text_to_numbers(texts, word_dictionary) 75 | 76 | # Get validation word keys 77 | valid_examples = [word_dictionary[x] for x in valid_words] 78 | 79 | print('Creating Model') 80 | # Define Embeddings: 81 | embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) 82 | 83 | # NCE loss parameters 84 | nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size], 85 | stddev=1.0 / np.sqrt(embedding_size))) 86 | nce_biases = tf.Variable(tf.zeros([vocabulary_size])) 87 | 88 | # Create data/target placeholders 89 | x_inputs = tf.placeholder(tf.int32, shape=[batch_size, 2*window_size]) 90 | y_target = tf.placeholder(tf.int32, shape=[batch_size, 1]) 91 | valid_dataset = tf.constant(valid_examples, dtype=tf.int32) 92 | 93 | # Lookup the word embedding 94 | # Add together window embeddings: 95 | embed = tf.zeros([batch_size, embedding_size]) 96 | for element in range(2*window_size): 97 | embed += tf.nn.embedding_lookup(embeddings, x_inputs[:, element]) 98 | 99 | # Get loss from prediction 100 | loss = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, embed, y_target, 101 | num_sampled, vocabulary_size)) 102 | 103 | # Create optimizer 104 | optimizer = tf.train.GradientDescentOptimizer(learning_rate=model_learning_rate).minimize(loss) 105 | 106 | # Cosine similarity between words 107 | norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) 108 | normalized_embeddings = embeddings / norm 109 | valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) 110 | similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) 111 | 112 | # Create model saving operation 113 | saver = tf.train.Saver({"embeddings": embeddings}) 114 | 115 | #Add variable initializer. 116 | init = tf.global_variables_initializer() 117 | sess.run(init) 118 | 119 | # Run the skip gram model. 120 | print('Starting Training') 121 | loss_vec = [] 122 | loss_x_vec = [] 123 | for i in range(generations): 124 | batch_inputs, batch_labels = text_helpers.generate_batch_data(text_data, batch_size, 125 | window_size, method='cbow') 126 | feed_dict = {x_inputs : batch_inputs, y_target : batch_labels} 127 | 128 | # Run the train step 129 | sess.run(optimizer, feed_dict=feed_dict) 130 | 131 | # Return the loss 132 | if (i+1) % print_loss_every == 0: 133 | loss_val = sess.run(loss, feed_dict=feed_dict) 134 | loss_vec.append(loss_val) 135 | loss_x_vec.append(i+1) 136 | print('Loss at step {} : {}'.format(i+1, loss_val)) 137 | 138 | # Validation: Print some random words and top 5 related words 139 | if (i+1) % print_valid_every == 0: 140 | sim = sess.run(similarity, feed_dict=feed_dict) 141 | for j in range(len(valid_words)): 142 | valid_word = word_dictionary_rev[valid_examples[j]] 143 | top_k = 5 # number of nearest neighbors 144 | nearest = (-sim[j, :]).argsort()[1:top_k+1] 145 | log_str = "Nearest to {}:".format(valid_word) 146 | for k in range(top_k): 147 | close_word = word_dictionary_rev[nearest[k]] 148 | log_str = '{} {},' .format(log_str, close_word) 149 | print(log_str) 150 | 151 | # Save dictionary + embeddings 152 | if (i+1) % save_embeddings_every == 0: 153 | # Save vocabulary dictionary 154 | with open(os.path.join(data_folder_name,'movie_vocab.pkl'), 'wb') as f: 155 | pickle.dump(word_dictionary, f) 156 | 157 | # Save embeddings 158 | model_checkpoint_path = os.path.join(os.getcwd(),data_folder_name,'cbow_movie_embeddings.ckpt') 159 | save_path = saver.save(sess, model_checkpoint_path) 160 | print('Model saved in file: {}'.format(save_path)) -------------------------------------------------------------------------------- /Chapter 08/download_cifar10.py: -------------------------------------------------------------------------------- 1 | # Download/Saving CIFAR-10 images in Inception format 2 | #--------------------------------------- 3 | # 4 | # In this script, we download the CIFAR-10 images and 5 | # transform/save them in the Inception Retrianing Format 6 | # 7 | # The end purpose of the files is for retrianing the 8 | # Google Inception tensorflow model to work on the CIFAR-10. 9 | 10 | import os 11 | import tarfile 12 | import _pickle as cPickle 13 | import numpy as np 14 | import urllib.request 15 | import scipy.misc 16 | 17 | cifar_link = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' 18 | data_dir = 'temp' 19 | if not os.path.isdir(data_dir): 20 | os.makedirs(data_dir) 21 | 22 | # Download tar file 23 | target_file = os.path.join(data_dir, 'cifar-10-python.tar.gz') 24 | if not os.path.isfile(target_file): 25 | print('CIFAR-10 file not found. Downloading CIFAR data (Size = 163MB)') 26 | print('This may take a few minutes, please wait.') 27 | filename, headers = urllib.request.urlretrieve(cifar_link, target_file) 28 | 29 | # Extract into memory 30 | tar = tarfile.open(target_file) 31 | tar.extractall(path=data_dir) 32 | tar.close() 33 | objects = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] 34 | 35 | # Create train image folders 36 | train_folder = 'train_dir' 37 | if not os.path.isdir(os.path.join(data_dir, train_folder)): 38 | for i in range(10): 39 | folder = os.path.join(data_dir, train_folder, objects[i]) 40 | os.makedirs(folder) 41 | # Create test image folders 42 | test_folder = 'validation_dir' 43 | if not os.path.isdir(os.path.join(data_dir, test_folder)): 44 | for i in range(10): 45 | folder = os.path.join(data_dir, test_folder, objects[i]) 46 | os.makedirs(folder) 47 | 48 | # Extract images accordingly 49 | data_location = os.path.join(data_dir, 'cifar-10-batches-py') 50 | train_names = ['data_batch_' + str(x) for x in range(1,6)] 51 | test_names = ['test_batch'] 52 | 53 | 54 | def load_batch_from_file(file): 55 | file_conn = open(file, 'rb') 56 | image_dictionary = cPickle.load(file_conn, encoding='latin1') 57 | file_conn.close() 58 | return(image_dictionary) 59 | 60 | 61 | def save_images_from_dict(image_dict, folder='data_dir'): 62 | # image_dict.keys() = 'labels', 'filenames', 'data', 'batch_label' 63 | for ix, label in enumerate(image_dict['labels']): 64 | folder_path = os.path.join(data_dir, folder, objects[label]) 65 | filename = image_dict['filenames'][ix] 66 | #Transform image data 67 | image_array = image_dict['data'][ix] 68 | image_array.resize([3, 32, 32]) 69 | # Save image 70 | output_location = os.path.join(folder_path, filename) 71 | scipy.misc.imsave(output_location,image_array.transpose()) 72 | 73 | # Sort train images 74 | for file in train_names: 75 | print('Saving images from file: {}'.format(file)) 76 | file_location = os.path.join(data_dir, 'cifar-10-batches-py', file) 77 | image_dict = load_batch_from_file(file_location) 78 | save_images_from_dict(image_dict, folder=train_folder) 79 | 80 | # Sort test images 81 | for file in test_names: 82 | print('Saving images from file: {}'.format(file)) 83 | file_location = os.path.join(data_dir, 'cifar-10-batches-py', file) 84 | image_dict = load_batch_from_file(file_location) 85 | save_images_from_dict(image_dict, folder=test_folder) 86 | 87 | # Create labels file 88 | cifar_labels_file = os.path.join(data_dir,'cifar10_labels.txt') 89 | print('Writing labels file, {}'.format(cifar_labels_file)) 90 | with open(cifar_labels_file, 'w') as labels_file: 91 | for item in objects: 92 | labels_file.write("{}\n".format(item)) -------------------------------------------------------------------------------- /Chapter 08/stylenet.py: -------------------------------------------------------------------------------- 1 | # Using Tensorflow for Stylenet/NeuralStyle 2 | #--------------------------------------- 3 | # 4 | # We use two images, an original image and a style image 5 | # and try to make the original image in the style of the style image. 6 | # 7 | # Reference paper: 8 | # https://arxiv.org/abs/1508.06576 9 | # 10 | # Need to download the model 'imagenet-vgg-verydee-19.mat' from: 11 | # http://www.vlfeat.org/matconvnet/models/beta16/imagenet-vgg-verydeep-19.mat 12 | 13 | import os 14 | import scipy.misc 15 | import numpy as np 16 | import tensorflow as tf 17 | from tensorflow.python.framework import ops 18 | ops.reset_default_graph() 19 | 20 | # Start a graph session 21 | sess = tf.Session() 22 | 23 | os.chdir('/home/nick/OneDrive/Documents/tensor_flow_book/Code/8_Convolutional_Neural_Networks') 24 | 25 | # Image Files 26 | original_image_file = 'temp/book_cover.jpg' 27 | style_image_file = 'temp/starry_night.jpg' 28 | 29 | # Saved VGG Network path 30 | vgg_path = '/home/nick/Documents/tensorflow/vgg_19_models/imagenet-vgg-verydeep-19.mat' 31 | 32 | # Default Arguments 33 | original_image_weight = 5.0 34 | style_image_weight = 200.0 35 | regularization_weight = 50.0 36 | learning_rate = 0.1 37 | generations = 10000 38 | output_generations = 500 39 | 40 | # Read in images 41 | original_image = scipy.misc.imread(original_image_file) 42 | style_image = scipy.misc.imread(style_image_file) 43 | 44 | # Get shape of target and make the style image the same 45 | target_shape = original_image.shape 46 | style_image = scipy.misc.imresize(style_image, target_shape[1] / style_image.shape[1]) 47 | 48 | # VGG-19 Layer Setup 49 | # From paper 50 | vgg_layers = ['conv1_1', 'relu1_1', 51 | 'conv1_2', 'relu1_2', 'pool1', 52 | 'conv2_1', 'relu2_1', 53 | 'conv2_2', 'relu2_2', 'pool2', 54 | 'conv3_1', 'relu3_1', 55 | 'conv3_2', 'relu3_2', 56 | 'conv3_3', 'relu3_3', 57 | 'conv3_4', 'relu3_4', 'pool3', 58 | 'conv4_1', 'relu4_1', 59 | 'conv4_2', 'relu4_2', 60 | 'conv4_3', 'relu4_3', 61 | 'conv4_4', 'relu4_4', 'pool4', 62 | 'conv5_1', 'relu5_1', 63 | 'conv5_2', 'relu5_2', 64 | 'conv5_3', 'relu5_3', 65 | 'conv5_4', 'relu5_4'] 66 | 67 | # Extract weights and matrix means 68 | def extract_net_info(path_to_params): 69 | vgg_data = scipy.io.loadmat(path_to_params) 70 | normalization_matrix = vgg_data['normalization'][0][0][0] 71 | mat_mean = np.mean(normalization_matrix, axis=(0,1)) 72 | network_weights = vgg_data['layers'][0] 73 | return(mat_mean, network_weights) 74 | 75 | 76 | # Create the VGG-19 Network 77 | def vgg_network(network_weights, init_image): 78 | network = {} 79 | image = init_image 80 | 81 | for i, layer in enumerate(vgg_layers): 82 | if layer[1] == 'c': 83 | weights, bias = network_weights[i][0][0][0][0] 84 | weights = np.transpose(weights, (1, 0, 2, 3)) 85 | bias = bias.reshape(-1) 86 | conv_layer = tf.nn.conv2d(image, tf.constant(weights), (1, 1, 1, 1), 'SAME') 87 | image = tf.nn.bias_add(conv_layer, bias) 88 | elif layer[1] == 'r': 89 | image = tf.nn.relu(image) 90 | else: 91 | image = tf.nn.max_pool(image, (1, 2, 2, 1), (1, 2, 2, 1), 'SAME') 92 | network[layer] = image 93 | return(network) 94 | 95 | # Here we define which layers apply to the original or style image 96 | original_layer = 'relu4_2' 97 | style_layers = ['relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'relu5_1'] 98 | 99 | # Get network parameters 100 | normalization_mean, network_weights = extract_net_info(vgg_path) 101 | 102 | shape = (1,) + original_image.shape 103 | style_shape = (1,) + style_image.shape 104 | original_features = {} 105 | style_features = {} 106 | 107 | # Get network parameters 108 | image = tf.placeholder('float', shape=shape) 109 | vgg_net = vgg_network(network_weights, image) 110 | 111 | # Normalize original image 112 | original_minus_mean = original_image - normalization_mean 113 | original_norm = np.array([original_minus_mean]) 114 | original_features[original_layer] = sess.run(vgg_net[original_layer], 115 | feed_dict={image: original_norm}) 116 | 117 | # Get style image network 118 | image = tf.placeholder('float', shape=style_shape) 119 | vgg_net = vgg_network(network_weights, image) 120 | style_minus_mean = style_image - normalization_mean 121 | style_norm = np.array([style_minus_mean]) 122 | 123 | for layer in style_layers: 124 | layer_output = sess.run(vgg_net[layer], feed_dict={image: style_norm}) 125 | layer_output = np.reshape(layer_output, (-1, layer_output.shape[3])) 126 | style_gram_matrix = np.matmul(layer_output.T, layer_output) / layer_output.size 127 | style_features[layer] = style_gram_matrix 128 | 129 | # Make Combined Image 130 | initial = tf.random_normal(shape) * 0.05 131 | image = tf.Variable(initial) 132 | vgg_net = vgg_network(network_weights, image) 133 | 134 | # Loss 135 | original_loss = original_image_weight * (2 * tf.nn.l2_loss(vgg_net[original_layer] - original_features[original_layer]) / 136 | original_features[original_layer].size) 137 | 138 | # Loss from Style Image 139 | style_loss = 0 140 | style_losses = [] 141 | for style_layer in style_layers: 142 | layer = vgg_net[style_layer] 143 | feats, height, width, channels = [x.value for x in layer.get_shape()] 144 | size = height * width * channels 145 | features = tf.reshape(layer, (-1, channels)) 146 | style_gram_matrix = tf.matmul(tf.transpose(features), features) / size 147 | style_expected = style_features[style_layer] 148 | style_losses.append(2 * tf.nn.l2_loss(style_gram_matrix - style_expected) / style_expected.size) 149 | style_loss += style_image_weight * tf.reduce_sum(style_losses) 150 | 151 | # To Smooth the resuts, we add in total variation loss 152 | total_var_x = sess.run(tf.reduce_prod(image[:,1:,:,:].get_shape())) 153 | total_var_y = sess.run(tf.reduce_prod(image[:,:,1:,:].get_shape())) 154 | first_term = regularization_weight * 2 155 | second_term_numerator = tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) 156 | second_term = second_term_numerator / total_var_y 157 | third_term = (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / total_var_x) 158 | total_variation_loss = first_term * (second_term + third_term) 159 | 160 | # Combined Loss 161 | loss = original_loss + style_loss + total_variation_loss 162 | 163 | # Declare Optimization Algorithm 164 | optimizer = tf.train.GradientDescentOptimizer(learning_rate) 165 | train_step = optimizer.minimize(loss) 166 | 167 | # Initialize Variables and start Training 168 | sess.run(tf.global_variables_initializer()) 169 | for i in range(generations): 170 | 171 | sess.run(train_step) 172 | 173 | # Print update and save temporary output 174 | if (i+1) % output_generations == 0: 175 | print('Generation {} out of {}'.format(i + 1, generations)) 176 | image_eval = sess.run(image) 177 | best_image_add_mean = image_eval.reshape(shape[1:]) + normalization_mean 178 | output_file = 'temp_output_{}.jpg'.format(i) 179 | scipy.misc.imsave(output_file, best_image_add_mean) 180 | 181 | 182 | # Save final image 183 | image_eval = sess.run(image) 184 | best_image_add_mean = image_eval.reshape(shape[1:]) + normalization_mean 185 | output_file = 'final_output.jpg' 186 | scipy.misc.imsave(output_file, best_image_add_mean) -------------------------------------------------------------------------------- /Chapter 09/implementing_rnn.py: -------------------------------------------------------------------------------- 1 | # Implementing an RNN in Tensorflow 2 | #---------------------------------- 3 | # 4 | # We implement an RNN in Tensorflow to predict spam/ham from texts 5 | # 6 | 7 | import os 8 | import re 9 | import io 10 | import requests 11 | import numpy as np 12 | import matplotlib.pyplot as plt 13 | import tensorflow as tf 14 | from zipfile import ZipFile 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | # Start a graph 19 | sess = tf.Session() 20 | 21 | # Set RNN parameters 22 | epochs = 20 23 | batch_size = 250 24 | max_sequence_length = 25 25 | rnn_size = 10 26 | embedding_size = 50 27 | min_word_frequency = 10 28 | learning_rate = 0.0005 29 | dropout_keep_prob = tf.placeholder(tf.float32) 30 | 31 | 32 | # Download or open data 33 | data_dir = 'temp' 34 | data_file = 'text_data.txt' 35 | if not os.path.exists(data_dir): 36 | os.makedirs(data_dir) 37 | 38 | if not os.path.isfile(os.path.join(data_dir, data_file)): 39 | zip_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip' 40 | r = requests.get(zip_url) 41 | z = ZipFile(io.BytesIO(r.content)) 42 | file = z.read('SMSSpamCollection') 43 | # Format Data 44 | text_data = file.decode() 45 | text_data = text_data.encode('ascii',errors='ignore') 46 | text_data = text_data.decode().split('\n') 47 | 48 | # Save data to text file 49 | with open(os.path.join(data_dir, data_file), 'w') as file_conn: 50 | for text in text_data: 51 | file_conn.write("{}\n".format(text)) 52 | else: 53 | # Open data from text file 54 | text_data = [] 55 | with open(os.path.join(data_dir, data_file), 'r') as file_conn: 56 | for row in file_conn: 57 | text_data.append(row) 58 | text_data = text_data[:-1] 59 | 60 | text_data = [x.split('\t') for x in text_data if len(x)>=1] 61 | [text_data_target, text_data_train] = [list(x) for x in zip(*text_data)] 62 | 63 | 64 | # Create a text cleaning function 65 | def clean_text(text_string): 66 | text_string = re.sub(r'([^\s\w]|_|[0-9])+', '', text_string) 67 | text_string = " ".join(text_string.split()) 68 | text_string = text_string.lower() 69 | return(text_string) 70 | 71 | # Clean texts 72 | text_data_train = [clean_text(x) for x in text_data_train] 73 | 74 | # Change texts into numeric vectors 75 | vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor(max_sequence_length, 76 | min_frequency=min_word_frequency) 77 | text_processed = np.array(list(vocab_processor.fit_transform(text_data_train))) 78 | 79 | # Shuffle and split data 80 | text_processed = np.array(text_processed) 81 | text_data_target = np.array([1 if x=='ham' else 0 for x in text_data_target]) 82 | shuffled_ix = np.random.permutation(np.arange(len(text_data_target))) 83 | x_shuffled = text_processed[shuffled_ix] 84 | y_shuffled = text_data_target[shuffled_ix] 85 | 86 | # Split train/test set 87 | ix_cutoff = int(len(y_shuffled)*0.80) 88 | x_train, x_test = x_shuffled[:ix_cutoff], x_shuffled[ix_cutoff:] 89 | y_train, y_test = y_shuffled[:ix_cutoff], y_shuffled[ix_cutoff:] 90 | vocab_size = len(vocab_processor.vocabulary_) 91 | print("Vocabulary Size: {:d}".format(vocab_size)) 92 | print("80-20 Train Test split: {:d} -- {:d}".format(len(y_train), len(y_test))) 93 | 94 | # Create placeholders 95 | x_data = tf.placeholder(tf.int32, [None, max_sequence_length]) 96 | y_output = tf.placeholder(tf.int32, [None]) 97 | 98 | # Create embedding 99 | embedding_mat = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0)) 100 | embedding_output = tf.nn.embedding_lookup(embedding_mat, x_data) 101 | #embedding_output_expanded = tf.expand_dims(embedding_output, -1) 102 | 103 | 104 | # Define the RNN cell 105 | cell = tf.nn.rnn_cell.BasicRNNCell(num_units = rnn_size) 106 | output, state = tf.nn.dynamic_rnn(cell, embedding_output, dtype=tf.float32) 107 | output = tf.nn.dropout(output, dropout_keep_prob) 108 | 109 | # Get output of RNN sequence 110 | output = tf.transpose(output, [1, 0, 2]) 111 | last = tf.gather(output, int(output.get_shape()[0]) - 1) 112 | 113 | 114 | weight = tf.Variable(tf.truncated_normal([rnn_size, 2], stddev=0.1)) 115 | bias = tf.Variable(tf.constant(0.1, shape=[2])) 116 | logits_out = tf.nn.softmax(tf.matmul(last, weight) + bias) 117 | 118 | # Loss function 119 | losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits_out, y_output) # logits=float32, labels=int32 120 | loss = tf.reduce_mean(losses) 121 | 122 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits_out, 1), tf.cast(y_output, tf.int64)), tf.float32)) 123 | 124 | optimizer = tf.train.RMSPropOptimizer(learning_rate) 125 | train_step = optimizer.minimize(loss) 126 | 127 | init = tf.global_variables_initializer() 128 | sess.run(init) 129 | 130 | train_loss = [] 131 | test_loss = [] 132 | train_accuracy = [] 133 | test_accuracy = [] 134 | # Start training 135 | for epoch in range(epochs): 136 | 137 | # Shuffle training data 138 | shuffled_ix = np.random.permutation(np.arange(len(x_train))) 139 | x_train = x_train[shuffled_ix] 140 | y_train = y_train[shuffled_ix] 141 | num_batches = int(len(x_train)/batch_size) + 1 142 | # TO DO CALCULATE GENERATIONS ExACTLY 143 | for i in range(num_batches): 144 | # Select train data 145 | min_ix = i * batch_size 146 | max_ix = np.min([len(x_train), ((i+1) * batch_size)]) 147 | x_train_batch = x_train[min_ix:max_ix] 148 | y_train_batch = y_train[min_ix:max_ix] 149 | 150 | # Run train step 151 | train_dict = {x_data: x_train_batch, y_output: y_train_batch, dropout_keep_prob:0.5} 152 | sess.run(train_step, feed_dict=train_dict) 153 | 154 | # Run loss and accuracy for training 155 | temp_train_loss, temp_train_acc = sess.run([loss, accuracy], feed_dict=train_dict) 156 | train_loss.append(temp_train_loss) 157 | train_accuracy.append(temp_train_acc) 158 | 159 | # Run Eval Step 160 | test_dict = {x_data: x_test, y_output: y_test, dropout_keep_prob:1.0} 161 | temp_test_loss, temp_test_acc = sess.run([loss, accuracy], feed_dict=test_dict) 162 | test_loss.append(temp_test_loss) 163 | test_accuracy.append(temp_test_acc) 164 | print('Epoch: {}, Test Loss: {:.2}, Test Acc: {:.2}'.format(epoch+1, temp_test_loss, temp_test_acc)) 165 | 166 | # Plot loss over time 167 | epoch_seq = np.arange(1, epochs+1) 168 | plt.plot(epoch_seq, train_loss, 'k--', label='Train Set') 169 | plt.plot(epoch_seq, test_loss, 'r-', label='Test Set') 170 | plt.title('Softmax Loss') 171 | plt.xlabel('Epochs') 172 | plt.ylabel('Softmax Loss') 173 | plt.legend(loc='upper left') 174 | plt.show() 175 | 176 | # Plot accuracy over time 177 | plt.plot(epoch_seq, train_accuracy, 'k--', label='Train Set') 178 | plt.plot(epoch_seq, test_accuracy, 'r-', label='Test Set') 179 | plt.title('Test Accuracy') 180 | plt.xlabel('Epochs') 181 | plt.ylabel('Accuracy') 182 | plt.legend(loc='upper left') 183 | plt.show() -------------------------------------------------------------------------------- /Chapter 10/parallelizing_tensorflow.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Parallelizing Tensorflow 3 | #---------------------------------- 4 | # 5 | # We will show how to use Tensorflow distributed 6 | 7 | import tensorflow as tf 8 | 9 | # We will setup a local cluster (on localhost) 10 | 11 | # Cluster for 2 local workers (tasks 0 and 1): 12 | cluster = tf.train.ClusterSpec({'local': ['localhost:2222', 'localhost:2223']}) 13 | # Server definition: 14 | server = tf.train.Server(cluster, job_name="local", task_index=0) 15 | server = tf.train.Server(cluster, job_name="local", task_index=1) 16 | # Finish and add 17 | #server.join() 18 | 19 | # Have each worker do a task 20 | # Worker 0 : create matrices 21 | # Worker 1 : calculate sum of all elements 22 | mat_dim = 25 23 | matrix_list = {} 24 | 25 | with tf.device('/job:local/task:0'): 26 | for i in range(0, 2): 27 | m_label = 'm_{}'.format(i) 28 | matrix_list[m_label] = tf.random_normal([mat_dim, mat_dim]) 29 | 30 | # Have each worker calculate the Cholesky Decomposition 31 | sum_outs = {} 32 | with tf.device('/job:local/task:1'): 33 | for i in range(0, 2): 34 | A = matrix_list['m_{}'.format(i)] 35 | sum_outs['m_{}'.format(i)] = tf.reduce_sum(A) 36 | 37 | # Sum all the cholesky decompositions 38 | summed_out = tf.add_n(list(sum_outs.values())) 39 | 40 | with tf.Session(server.target) as sess: 41 | result = sess.run(summed_out) 42 | print('Summed Values:{}'.format(result)) -------------------------------------------------------------------------------- /Chapter 10/production_ex_eval.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Tensorflow Production Example (Evaluating) 3 | #---------------------------------- 4 | # 5 | # We pull together everything and create an example 6 | # of best tensorflow production tips 7 | # 8 | # The example we will productionalize is the spam/ham RNN 9 | # from the RNN Chapter. 10 | 11 | import os 12 | import re 13 | import numpy as np 14 | import tensorflow as tf 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | tf.app.flags.DEFINE_string("storage_folder", "temp", "Where to store model and data.") 19 | tf.app.flags.DEFINE_string('model_file', False, 'Model file location.') 20 | tf.app.flags.DEFINE_boolean('run_unit_tests', False, 'If true, run tests.') 21 | FLAGS = tf.app.flags.FLAGS 22 | 23 | 24 | # Create a text cleaning function 25 | def clean_text(text_string): 26 | text_string = re.sub(r'([^\s\w]|_|[0-9])+', '', text_string) 27 | text_string = " ".join(text_string.split()) 28 | text_string = text_string.lower() 29 | return(text_string) 30 | 31 | 32 | # Load vocab processor 33 | def load_vocab(): 34 | vocab_path = os.path.join(FLAGS.storage_folder, "vocab") 35 | vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor.restore(vocab_path) 36 | return(vocab_processor) 37 | 38 | 39 | # Process input data: 40 | def process_data(input_data, vocab_processor): 41 | input_data = clean_text(input_data) 42 | input_data = input_data.split() 43 | processed_input = np.array(list(vocab_processor.transform(input_data))) 44 | return(processed_input) 45 | 46 | 47 | # Get input function 48 | def get_input_data(): 49 | """ 50 | For this function, we just prompt the user for a text message to evaluate 51 | But this function could also potentially read a file in as well. 52 | """ 53 | input_text = input("Please enter a text message to evaluate: ") 54 | vocab_processor = load_vocab() 55 | return(process_data(input_text, vocab_processor)) 56 | 57 | 58 | # Test clean_text function 59 | class clean_test(tf.test.TestCase): 60 | # Make sure cleaning function behaves correctly 61 | def clean_string_test(self): 62 | with self.test_session(): 63 | test_input = '--Tensorflow\'s so Great! Don\t you think so? ' 64 | test_expected = 'tensorflows so great don you think so' 65 | test_out = clean_text(test_input) 66 | self.assertEqual(test_expected, test_out) 67 | 68 | 69 | # Main function 70 | def main(args): 71 | # Get flags 72 | storage_folder = FLAGS.storage_folder 73 | 74 | # Get user input text 75 | x_data = get_input_data() 76 | 77 | # Load model 78 | graph = tf.Graph() 79 | with graph.as_default(): 80 | sess = tf.Session() 81 | with sess.as_default(): 82 | # Load the saved meta graph and restore variables 83 | saver = tf.train.import_meta_graph("{}.meta".format(os.path.join(storage_folder, "model.ckpt"))) 84 | saver.restore(sess, os.path.join(storage_folder, "model.ckpt")) 85 | 86 | # Get the placeholders from the graph by name 87 | x_data_ph = graph.get_operation_by_name("x_data_ph").outputs[0] 88 | dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0] 89 | probability_outputs = graph.get_operation_by_name("probability_outputs").outputs[0] 90 | 91 | # Make the prediction 92 | eval_feed_dict = {x_data_ph: x_data, dropout_keep_prob: 1.0} 93 | probability_prediction = sess.run(tf.reduce_mean(probability_outputs, 0), eval_feed_dict) 94 | 95 | # Print output (Or save to file or DB connection?) 96 | print('Probability of Spam: {:.4}'.format(probability_prediction[1])) 97 | 98 | # Run main module/tf App 99 | if __name__ == "__main__": 100 | if FLAGS.run_unit_tests: 101 | # Perform unit tests 102 | tf.test.main() 103 | else: 104 | # Run evaluation 105 | tf.app.run() -------------------------------------------------------------------------------- /Chapter 10/production_tips_for_tf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Tips for Tensorflow to Production 3 | #---------------------------------- 4 | # 5 | # Various Tips for Taking Tensorflow to Production 6 | 7 | ############################################ 8 | # 9 | # THIS SCRIPT IS NOT RUNNABLE. 10 | # -it only contains tips for production code 11 | # 12 | ############################################ 13 | 14 | # Also you can clear the default graph from memory 15 | import tensorflow as tf 16 | from tensorflow.python.framework import ops 17 | ops.reset_default_graph() 18 | 19 | # Saving Models 20 | # File types created from saving: 21 | # checkpoint file: Holds info on where the most recent models are 22 | # events file: Strictly for viewing graph in Tensorboard 23 | # pbtxt file: Textual protobufs file (uncompressed), used for debugging 24 | # chkp file: Holds data and model weights (large file) 25 | # meta chkp files: Model Graph and Meta-data (learning rate and operations) 26 | 27 | 28 | # Saving data pipeline structures (vocabulary, ) 29 | word_list = ['to', 'be', 'or', 'not', 'to', 'be'] 30 | vocab_list = list(set(word_list)) 31 | vocab2ix_dict = dict(zip(vocab_list, range(len(vocab_list)))) 32 | ix2vocab_dict = {val:key for key,val in vocab2ix_dict.items()} 33 | 34 | # Save vocabulary 35 | import json 36 | with open('vocab2ix_dict.json', 'w') as file_conn: 37 | json.dump(vocab2ix_dict, file_conn) 38 | 39 | # Load vocabulary 40 | with open('vocab2ix_dict.json', 'r') as file_conn: 41 | vocab2ix_dict = json.load(file_conn) 42 | 43 | # After model declaration, add a saving operations 44 | saver = tf.train.Saver() 45 | # Then during training, save every so often, referencing the training generation 46 | for i in range(generations): 47 | ... 48 | if i%save_every == 0: 49 | saver.save(sess, 'my_model', global_step=step) 50 | 51 | # Can also save only specific variables: 52 | saver = tf.train.Saver({"my_var": my_variable}) 53 | 54 | 55 | # other options for saver are 'keep checkpoint_every_n_hours' 56 | # also 'max_to_keep'= default 5. 57 | 58 | # Be sure to name operations, and variables for easy loading for referencing later 59 | conv_weights = tf.Variable(tf.random_normal(), name='conv_weights') 60 | loss = tf.reduce_mean(... , name='loss') 61 | 62 | # Instead of tyring argparse and main(), Tensorflow provides an 'app' function 63 | # to handle running and loading of arguments 64 | 65 | # At the beginning of the file, define the flags. 66 | tf.app.flags.DEFINE_string("worker_locations", "", "List of worker addresses.") 67 | tf.app.flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.') 68 | tf.app.flags.DEFINE_integer('generations', 1000, 'Number of training generations.') 69 | tf.app.flags.DEFINE_boolean('run_unit_tests', False, 'If true, run tests.') 70 | 71 | # Need to define a 'main' function for the app to run 72 | def main(_): 73 | worker_ips = FLAGS.worker_locations.split(",") 74 | learning_rate = FLAGS.learning_rate 75 | generations = FLAGS.generations 76 | run_unit_tests = FLAGS.run_unit_tests 77 | 78 | # Run the Tensorflow app 79 | if __name__ == "__main__": 80 | tf.app.run() 81 | 82 | 83 | # Use of Tensorflow's built in logging: 84 | # Five levels: DEBUG, INFO, WARN, ERROR, and FATAL 85 | tf.logging.set_verbosity(tf.logging.WARN) 86 | # WARN is the default value, but to see more information, you can set it to 87 | # INFO or DEBUG 88 | tf.logging.set_verbosity(tf.logging.DEBUG) 89 | # Note: 'DEBUG' is quite verbose. 90 | 91 | -------------------------------------------------------------------------------- /Chapter 10/using_multiple_devices.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Using Multiple Devices 3 | #---------------------------------- 4 | # 5 | # This function gives us the ways to use 6 | # multiple devices (executors) in Tensorflow. 7 | 8 | import tensorflow as tf 9 | from tensorflow.python.framework import ops 10 | ops.reset_default_graph() 11 | 12 | # To find out where placement occurs, set 'log_device_placement' 13 | sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) 14 | 15 | a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') 16 | b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') 17 | c = tf.matmul(a, b) 18 | 19 | # Runs the op. 20 | print(sess.run(c)) 21 | 22 | 23 | # If we load a graph and want device placement to be forgotten, 24 | # we set a parameter in our session: 25 | config = tf.ConfigProto() 26 | config.allow_soft_placement = True 27 | sess_soft = tf.Session(config=config) 28 | 29 | # GPUs 30 | #--------------------------------- 31 | # Note that the GPU must have a compute capability > 3.5 for TF to use. 32 | # http://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#compute-capability 33 | 34 | 35 | # Careful with GPU memory allocation, TF never releases it. TF starts with almost 36 | # all of the GPU memory allocated. We can slowly grow to that limit with an 37 | # option setting: 38 | 39 | config.gpu_options.allow_growth = True 40 | sess_grow = tf.Session(config=config) 41 | 42 | # Also, we can limit the size of GPU memory used, with the following option 43 | config.gpu_options.per_process_gpu_memory_fraction = 0.4 44 | sess_limited = tf.Session(config=config) 45 | 46 | 47 | # How to set placements on multiple devices. 48 | # Here, assume we have three devies CPU:0, GPU:0, and GPU:1 49 | if tf.test.is_built_with_cuda(): 50 | with tf.device('/cpu:0'): 51 | a = tf.constant([1.0, 3.0, 5.0], shape=[1, 3]) 52 | b = tf.constant([2.0, 4.0, 6.0], shape=[3, 1]) 53 | 54 | with tf.device('/gpu:1'): 55 | c = tf.matmul(a,b) 56 | c = tf.reshape(c, [-1]) 57 | 58 | with tf.device('/gpu:2'): 59 | d = tf.matmul(b,a) 60 | flat_d = tf.reshape(d, [-1]) 61 | 62 | combined = tf.multiply(c, flat_d) 63 | print(sess.run(combined)) -------------------------------------------------------------------------------- /Chapter 11/genetic_algorithm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Implementing a Genetic Algorithm 3 | # ------------------------------- 4 | # 5 | # Genetic Algorithm Optimization in Tensorflow 6 | # 7 | # We are going to implement a genetic algorithm 8 | # to optimize to a ground truth array. The ground 9 | # truth will be an array of 50 floating point 10 | # numbers which are generated by: 11 | # f(x)=sin(2*pi*x/50) where 0= mutation] = 0 102 | 103 | # Run GA step 104 | feed_dict = {truth_ph: truth.reshape([1, features]), 105 | crossover_mat_ph: crossover_mat, 106 | mutation_val_ph: mutation_values} 107 | step.run(feed_dict, session=sess) 108 | best_individual_val = sess.run(best_individual, feed_dict=feed_dict) 109 | 110 | if i % 5 == 0: 111 | best_fit = sess.run(best_val, feed_dict = feed_dict) 112 | print('Generation: {}, Best Fitness (lowest MSE): {:.2}'.format(i, -best_fit)) 113 | 114 | plt.plot(truth, label="True Values") 115 | plt.plot(np.squeeze(best_individual_val), label="Best Individual") 116 | plt.axis((0, features, -1.25, 1.25)) 117 | plt.legend(loc='upper right') 118 | plt.show() -------------------------------------------------------------------------------- /Chapter 11/k_means.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # K-means with Tensorflow 3 | #---------------------------------- 4 | # 5 | # This script shows how to do k-means with Tensorflow 6 | 7 | import numpy as np 8 | import matplotlib.pyplot as plt 9 | import tensorflow as tf 10 | from sklearn import datasets 11 | from scipy.spatial import cKDTree 12 | from sklearn.decomposition import PCA 13 | from sklearn.preprocessing import scale 14 | from tensorflow.python.framework import ops 15 | ops.reset_default_graph() 16 | 17 | sess = tf.Session() 18 | 19 | iris = datasets.load_iris() 20 | 21 | num_pts = len(iris.data) 22 | num_feats = len(iris.data[0]) 23 | 24 | # Set k-means parameters 25 | # There are 3 types of iris flowers, see if we can predict them 26 | k=3 27 | generations = 25 28 | 29 | data_points = tf.Variable(iris.data) 30 | cluster_labels = tf.Variable(tf.zeros([num_pts], dtype=tf.int64)) 31 | 32 | # Randomly choose starting points 33 | rand_starts = np.array([iris.data[np.random.choice(len(iris.data))] for _ in range(k)]) 34 | 35 | centroids = tf.Variable(rand_starts) 36 | 37 | # In order to calculate the distance between every data point and every centroid, we 38 | # repeat the centroids into a (num_points) by k matrix. 39 | centroid_matrix = tf.reshape(tf.tile(centroids, [num_pts, 1]), [num_pts, k, num_feats]) 40 | # Then we reshape the data points into k (3) repeats 41 | point_matrix = tf.reshape(tf.tile(data_points, [1, k]), [num_pts, k, num_feats]) 42 | distances = tf.reduce_sum(tf.square(point_matrix - centroid_matrix), reduction_indices=2) 43 | 44 | #Find the group it belongs to with tf.argmin() 45 | centroid_group = tf.argmin(distances, 1) 46 | 47 | # Find the group average 48 | def data_group_avg(group_ids, data): 49 | # Sum each group 50 | sum_total = tf.unsorted_segment_sum(data, group_ids, 3) 51 | # Count each group 52 | num_total = tf.unsorted_segment_sum(tf.ones_like(data), group_ids, 3) 53 | # Calculate average 54 | avg_by_group = sum_total/num_total 55 | return(avg_by_group) 56 | 57 | means = data_group_avg(centroid_group, data_points) 58 | 59 | update = tf.group(centroids.assign(means), cluster_labels.assign(centroid_group)) 60 | 61 | init = tf.global_variables_initializer() 62 | 63 | sess.run(init) 64 | 65 | for i in range(generations): 66 | print('Calculating gen {}, out of {}.'.format(i, generations)) 67 | _, centroid_group_count = sess.run([update, centroid_group]) 68 | group_count = [] 69 | for ix in range(k): 70 | group_count.append(np.sum(centroid_group_count==ix)) 71 | print('Group counts: {}'.format(group_count)) 72 | 73 | 74 | [centers, assignments] = sess.run([centroids, cluster_labels]) 75 | 76 | # Find which group assignments correspond to which group labels 77 | # First, need a most common element function 78 | def most_common(my_list): 79 | return(max(set(my_list), key=my_list.count)) 80 | 81 | label0 = most_common(list(assignments[0:50])) 82 | label1 = most_common(list(assignments[50:100])) 83 | label2 = most_common(list(assignments[100:150])) 84 | 85 | group0_count = np.sum(assignments[0:50]==label0) 86 | group1_count = np.sum(assignments[50:100]==label1) 87 | group2_count = np.sum(assignments[100:150]==label2) 88 | 89 | accuracy = (group0_count + group1_count + group2_count)/150. 90 | 91 | print('Accuracy: {:.2}'.format(accuracy)) 92 | 93 | # Also plot the output 94 | # First use PCA to transform the 4-dimensional data into 2-dimensions 95 | pca_model = PCA(n_components=2) 96 | reduced_data = pca_model.fit_transform(iris.data) 97 | # Transform centers 98 | reduced_centers = pca_model.transform(centers) 99 | 100 | # Step size of mesh for plotting 101 | h = .02 102 | 103 | # Plot the decision boundary. For that, we will assign a color to each 104 | x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1 105 | y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1 106 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) 107 | 108 | # Get k-means classifications for the grid points 109 | xx_pt = list(xx.ravel()) 110 | yy_pt = list(yy.ravel()) 111 | xy_pts = np.array([[x,y] for x,y in zip(xx_pt, yy_pt)]) 112 | mytree = cKDTree(reduced_centers) 113 | dist, indexes = mytree.query(xy_pts) 114 | 115 | # Put the result into a color plot 116 | indexes = indexes.reshape(xx.shape) 117 | plt.figure(1) 118 | plt.clf() 119 | plt.imshow(indexes, interpolation='nearest', 120 | extent=(xx.min(), xx.max(), yy.min(), yy.max()), 121 | cmap=plt.cm.Paired, 122 | aspect='auto', origin='lower') 123 | 124 | # Plot each of the true iris data groups 125 | symbols = ['o', '^', 'D'] 126 | label_name = ['Setosa', 'Versicolour', 'Virginica'] 127 | for i in range(3): 128 | temp_group = reduced_data[(i*50):(50)*(i+1)] 129 | plt.plot(temp_group[:, 0], temp_group[:, 1], symbols[i], markersize=10, label=label_name[i]) 130 | # Plot the centroids as a white X 131 | plt.scatter(reduced_centers[:, 0], reduced_centers[:, 1], 132 | marker='x', s=169, linewidths=3, 133 | color='w', zorder=10) 134 | plt.title('K-means clustering on Iris Dataset\n' 135 | 'Centroids are marked with white cross') 136 | plt.xlim(x_min, x_max) 137 | plt.ylim(y_min, y_max) 138 | plt.legend(loc='lower right') 139 | plt.show() -------------------------------------------------------------------------------- /Chapter 11/solving_ode_system.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Solving a Sytem of ODEs 3 | #---------------------------------- 4 | # 5 | # In this script, we use Tensorflow to solve a sytem 6 | # of ODEs. 7 | # 8 | # The system of ODEs we will solve is the Lotka-Volterra 9 | # predator-prey system. 10 | 11 | 12 | # Declaring Operations 13 | import matplotlib.pyplot as plt 14 | import tensorflow as tf 15 | from tensorflow.python.framework import ops 16 | ops.reset_default_graph() 17 | 18 | # Open interactive graph session 19 | sess = tf.Session() 20 | 21 | # Discrete Lotka-Volterra predator/prey equations 22 | # 23 | # X(t+1) = X(t) + (aX(t) + bX(t)Y(t)) * t_delta # Prey 24 | # 25 | # Y(t+1) = Y(t) + (cY(t) + dX(t)Y(t)) * t_delta # Predator 26 | 27 | # Declare constants and variables 28 | x_initial = tf.constant(1.0) 29 | y_initial = tf.constant(1.0) 30 | X_t1 = tf.Variable(x_initial) 31 | Y_t1 = tf.Variable(y_initial) 32 | 33 | # Make the placeholders 34 | t_delta = tf.placeholder(tf.float32, shape=()) 35 | a = tf.placeholder(tf.float32, shape=()) 36 | b = tf.placeholder(tf.float32, shape=()) 37 | c = tf.placeholder(tf.float32, shape=()) 38 | d = tf.placeholder(tf.float32, shape=()) 39 | 40 | # Discretized ODE update 41 | X_t2 = X_t1 + (a * X_t1 + b * X_t1 * Y_t1) * t_delta 42 | Y_t2 = Y_t1 + (c * Y_t1 + d * X_t1 * Y_t1) * t_delta 43 | 44 | # Update to New Population 45 | step = tf.group( 46 | X_t1.assign(X_t2), 47 | Y_t1.assign(Y_t2)) 48 | 49 | init = tf.global_variables_initializer() 50 | sess.run(init) 51 | 52 | # Run the ODE 53 | prey_values = [] 54 | predator_values = [] 55 | for i in range(1000): 56 | # Step simulation (using constants for a known cyclic solution) 57 | step.run({a: (2./3.), b: (-4./3.), c: -1.0, d: 1.0, t_delta: 0.01}, session=sess) 58 | # Store each outcome 59 | temp_prey, temp_pred = sess.run([X_t1, Y_t1]) 60 | prey_values.append(temp_prey) 61 | predator_values.append(temp_pred) 62 | 63 | # Visualize the output 64 | plt.plot(prey_values) 65 | plt.plot(predator_values) 66 | plt.legend(['Prey', 'Predator'], loc='upper right') 67 | plt.show() -------------------------------------------------------------------------------- /Chapter 11/using_tensorboard.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Using Tensorboard 3 | #---------------------------------- 4 | # 5 | # We illustrate the various ways to use 6 | # Tensorboard 7 | 8 | import os 9 | import io 10 | import time 11 | import numpy as np 12 | import matplotlib.pyplot as plt 13 | import tensorflow as tf 14 | 15 | # Initialize a graph session 16 | sess = tf.Session() 17 | 18 | # Create a visualizer object 19 | summary_writer = tf.train.SummaryWriter('tensorboard', tf.get_default_graph()) 20 | 21 | # Create tensorboard folder if not exists 22 | if not os.path.exists('tensorboard'): 23 | os.makedirs('tensorboard') 24 | print('Running a slowed down linear regression. ' 25 | 'Run the command: $tensorboard --logdir="tensorboard" ' 26 | ' Then navigate to http://127.0.0.0:6006') 27 | 28 | # You can also specify a port option with --port 6006 29 | 30 | # Wait a few seconds for user to run tensorboard commands 31 | time.sleep(3) 32 | 33 | # Some parameters 34 | batch_size = 50 35 | generations = 100 36 | 37 | # Create sample input data 38 | x_data = np.arange(1000)/10. 39 | true_slope = 2. 40 | y_data = x_data * true_slope + np.random.normal(loc=0.0, scale=25, size=1000) 41 | 42 | # Split into train/test 43 | train_ix = np.random.choice(len(x_data), size=int(len(x_data)*0.9), replace=False) 44 | test_ix = np.setdiff1d(np.arange(1000), train_ix) 45 | x_data_train, y_data_train = x_data[train_ix], y_data[train_ix] 46 | x_data_test, y_data_test = x_data[test_ix], y_data[test_ix] 47 | 48 | # Declare placeholders 49 | x_graph_input = tf.placeholder(tf.float32, [None]) 50 | y_graph_input = tf.placeholder(tf.float32, [None]) 51 | 52 | # Declare model variables 53 | m = tf.Variable(tf.random_normal([1], dtype=tf.float32), name='Slope') 54 | 55 | # Declare model 56 | output = tf.multiply(m, x_graph_input, name='Batch_Multiplication') 57 | 58 | # Declare loss function (L1) 59 | residuals = output - y_graph_input 60 | l2_loss = tf.reduce_mean(tf.abs(residuals), name="L2_Loss") 61 | 62 | # Declare optimization function 63 | my_optim = tf.train.GradientDescentOptimizer(0.01) 64 | train_step = my_optim.minimize(l2_loss) 65 | 66 | # Visualize a scalar 67 | with tf.name_scope('Slope_Estimate'): 68 | tf.scalar_summary('Slope_Estimate', tf.squeeze(m)) 69 | 70 | # Visualize a histogram (errors) 71 | with tf.name_scope('Loss_and_Residuals'): 72 | tf.histogram_summary('Histogram_Errors', l2_loss) 73 | tf.histogram_summary('Histogram_Residuals', residuals) 74 | 75 | 76 | 77 | # Declare summary merging operation 78 | summary_op = tf.merge_all_summaries() 79 | 80 | # Initialize Variables 81 | init = tf.global_variables_initializer() 82 | sess.run(init) 83 | 84 | for i in range(generations): 85 | batch_indices = np.random.choice(len(x_data_train), size=batch_size) 86 | x_batch = x_data_train[batch_indices] 87 | y_batch = y_data_train[batch_indices] 88 | _, train_loss, summary = sess.run([train_step, l2_loss, summary_op], 89 | feed_dict={x_graph_input: x_batch, 90 | y_graph_input: y_batch}) 91 | 92 | test_loss, test_resids = sess.run([l2_loss, residuals], feed_dict={x_graph_input: x_data_test, 93 | y_graph_input: y_data_test}) 94 | 95 | if (i+1)%10==0: 96 | print('Generation {} of {}. Train Loss: {:.3}, Test Loss: {:.3}.'.format(i+1, generations, train_loss, test_loss)) 97 | 98 | log_writer = tf.train.SummaryWriter('tensorboard') 99 | log_writer.add_summary(summary, i) 100 | time.sleep(0.5) 101 | 102 | #Create a function to save a protobuf bytes version of the graph 103 | def gen_linear_plot(slope): 104 | linear_prediction = x_data * slope 105 | plt.plot(x_data, y_data, 'b.', label='data') 106 | plt.plot(x_data, linear_prediction, 'r-', linewidth=3, label='predicted line') 107 | plt.legend(loc='upper left') 108 | buf = io.BytesIO() 109 | plt.savefig(buf, format='png') 110 | buf.seek(0) 111 | return(buf) 112 | 113 | # Add image to tensorboard (plot the linear fit!) 114 | slope = sess.run(m) 115 | plot_buf = gen_linear_plot(slope[0]) 116 | # Convert PNG buffer to TF image 117 | image = tf.image.decode_png(plot_buf.getvalue(), channels=4) 118 | # Add the batch dimension 119 | image = tf.expand_dims(image, 0) 120 | # Add image summary 121 | image_summary_op = tf.image_summary("Linear Plot", image) 122 | image_summary = sess.run(image_summary_op) 123 | log_writer.add_summary(image_summary, i) 124 | log_writer.close() -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Packt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # TensorFlow Machine Learning Cookbook 5 | This is the code repository for [TensorFlow Machine Learning Cookbook](https://www.packtpub.com/big-data-and-business-intelligence/tensorflow-machine-learning-cookbook?utm_source=github&utm_medium=repository&utm_content=9781786462169), published by Packt. It contains all the supporting project files necessary to work through the book from start to finish. 6 | 7 | ## About the book 8 | TensorFlow is an open source software library for Machine Intelligence. The independent recipes in this book will teach you how to use TensorFlow for complex data computations and will let you dig deeper and gain more insights into your data than ever before. You’ll work through recipes on training models, model evaluation, sentiment analysis, regression analysis, clustering analysis, artificial neural networks, and deep learning – each using Google’s machine learning library TensorFlow. 9 | 10 | This guide starts with the fundamentals of the TensorFlow library which includes variables, matrices, and various data sources. Moving ahead, you will get hands-on experience with Linear Regression techniques with TensorFlow. The next chapters cover important high-level concepts such as neural networks, CNN, RNN, and NLP. 11 | 12 | Once you are familiar and comfortable with the TensorFlow ecosystem, the last chapter will show you how to take it to production. 13 | 14 | ## Instructions and Navigations 15 | All of the code is organized into folders. Each folder starts with a number followed by the application name. For example, Chapter 03. 16 | 17 | The code will look like the following: 18 | 19 | import matplotlib.pyplot as plt 20 | import numpy as np 21 | import tensorflow as tf 22 | from sklearn import datasets 23 | from tensorflow.python.framework import ops 24 | ops.reset_default_graph() 25 | 26 | ### Software requirements: 27 | Python 3, with the following installed Python libraries: TensorFlow, Numpy, 28 | Scikit-Learn, Requests, and Jupyter. It is compatible in all three major operating 29 | systems, Mac, Windows, and Linux. It requires no special hardware to run the 30 | scripts. 31 | 32 | ## Related Products: 33 | * [Getting Started with TensorFlow](https://www.packtpub.com/big-data-and-business-intelligence/getting-started-tensorflow?utm_source=github&utm_medium=repository&utm_content=9781786468574) 34 | 35 | * [Deep Learning with TensorFlow [Video]](https://www.packtpub.com/big-data-and-business-intelligence/deep-learning-tensorflow-video?utm_source=github&utm_medium=repository&utm_content=9781786464491) 36 | 37 | * [Building Machine Learning Systems with TensorFlow [Video]](https://www.packtpub.com/big-data-and-business-intelligence/building-machine-learning-systems-tensorflow-video?utm_source=github&utm_medium=repository&utm_content=9781787281806) 38 | 39 | ### Suggestions and Feedback 40 | [Click here](https://docs.google.com/forms/d/e/1FAIpQLSe5qwunkGf6PUvzPirPDtuy1Du5Rlzew23UBp2S-P3wB-GcwQ/viewform) if you have any feedback or suggestions. 41 | 42 | ### Download a free PDF 43 | 44 | If you have already purchased a print or Kindle version of this book, you can get a DRM-free PDF version at no cost.
Simply click on the link to claim your free PDF. 45 |

https://packt.link/free-ebook/9781786462169

--------------------------------------------------------------------------------