├── Chapter04
    ├── weather.npz
    ├── DNNs
    │   ├── weather.npz
    │   ├── cnn.py
    │   ├── extracting_weights.py
    │   └── dnn.py
    ├── Exploring RNNs
    │   ├── rnn.odg
    │   ├── weather.npz
    │   └── rnn_tf.py
    └── TensorFlow learn
    │   ├── data_with_labels.npz
    │   └── learn_intro.py
├── Chapter03
    ├── Deep CNN
    │   ├── data_with_labels.npz
    │   └── conv.py
    ├── Wrapping up deep CNN
    │   ├── data_with_labels.npz
    │   └── conv_p2.py
    ├── Pooling layer application
    │   └── conv_simple.py
    ├── Convolutional layer application
    │   └── conv_simple.py
    └── Deeper CNN
    │   └── conv_p2.py
├── Chapter05
    └── Research evaluation
    │   ├── conv1.odg
    │   ├── conv2.odg
    │   ├── mlp.odg
    │   ├── logistic.odg
    │   ├── single_hidden.odg
    │   ├── data_with_labels.npz
    │   └── data_review.py
├── Chapter01
    ├── Logistic regression model building
    │   ├── data_with_labels.npz
    │   └── logistic.py
    ├── Installing TensorFlow
    │   └── install.sh
    ├── Simple Computations
    │   └── simple.py
    └── Logistic regression training
    │   └── logistic.py
├── LICENSE
├── Chapter02
    ├── Basic neural networks
    │   └── basic_nn.py
    ├── Single hidden layer explained
    │   └── single_hidden.py
    ├── Single hidden layer model
    │   └── single_hidden.py
    ├── Results of the multiple hidden layer
    │   └── mlp.py
    └── The multiple hidden layer model
    │   └── mlp.py
└── README.md


/Chapter04/weather.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter04/weather.npz


--------------------------------------------------------------------------------
/Chapter04/DNNs/weather.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter04/DNNs/weather.npz


--------------------------------------------------------------------------------
/Chapter04/Exploring RNNs/rnn.odg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter04/Exploring RNNs/rnn.odg


--------------------------------------------------------------------------------
/Chapter04/Exploring RNNs/weather.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter04/Exploring RNNs/weather.npz


--------------------------------------------------------------------------------
/Chapter03/Deep CNN/data_with_labels.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter03/Deep CNN/data_with_labels.npz


--------------------------------------------------------------------------------
/Chapter05/Research evaluation/conv1.odg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter05/Research evaluation/conv1.odg


--------------------------------------------------------------------------------
/Chapter05/Research evaluation/conv2.odg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter05/Research evaluation/conv2.odg


--------------------------------------------------------------------------------
/Chapter05/Research evaluation/mlp.odg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter05/Research evaluation/mlp.odg


--------------------------------------------------------------------------------
/Chapter05/Research evaluation/logistic.odg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter05/Research evaluation/logistic.odg


--------------------------------------------------------------------------------
/Chapter04/TensorFlow learn/data_with_labels.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter04/TensorFlow learn/data_with_labels.npz


--------------------------------------------------------------------------------
/Chapter05/Research evaluation/single_hidden.odg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter05/Research evaluation/single_hidden.odg


--------------------------------------------------------------------------------
/Chapter03/Wrapping up deep CNN/data_with_labels.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter03/Wrapping up deep CNN/data_with_labels.npz


--------------------------------------------------------------------------------
/Chapter05/Research evaluation/data_with_labels.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter05/Research evaluation/data_with_labels.npz


--------------------------------------------------------------------------------
/Chapter01/Logistic regression model building/data_with_labels.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/PacktPublishing/Hands-On-Deep-Learning-with-TensorFlow/HEAD/Chapter01/Logistic regression model building/data_with_labels.npz


--------------------------------------------------------------------------------
/Chapter01/Installing TensorFlow/install.sh:
--------------------------------------------------------------------------------
1 | # Python 3.4 installation
2 | sudo pip3 install --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.2.1-cp34-cp34m-linux_x86_64.whl
3 | 
4 | # Python 3.5 installation
5 | wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.2.1-cp34-cp34m-linux_x86_64.whl
6 | mv tensorflow-1.2.1-cp34-cp34m-linux_x86_64.whl tensorflow-1.2.1-cp35-cp35m-linux_x86_64.whl
7 | sudo pip3 install ./tensorflow-1.2.1-cp35-cp35m-linux_x86_64.whl


--------------------------------------------------------------------------------
/Chapter05/Research evaluation/data_review.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | import numpy as np
 3 | %autoindent
 4 | try:
 5 |     from tqdm import tqdm
 6 | except ImportError:
 7 |     def tqdm(x, *args, **kwargs):
 8 |         return x
 9 | 
10 | # Load data
11 | data = np.load('data_with_labels.npz')
12 | train = data['arr_0']/255.
13 | labels = data['arr_1']
14 | 
15 | # Look at some data
16 | print(train[0])
17 | print(labels[0])
18 | 
19 | # If you have matplotlib installed
20 | import matplotlib.pyplot as plt
21 | plt.ion()
22 | 
23 | # One look at a letter/digit from each font
24 | # Best to reshape as one large array, then plot
25 | all_letters = np.zeros([5*36,62*36])
26 | for font in range(5):
27 |     for letter in range(62):
28 |         all_letters[font*36:(font+1)*36,
29 |                 letter*36:(letter+1)*36] = \
30 |                 train[9*(font*62 + letter)]
31 | plt.pcolormesh(all_letters,
32 |         cmap=plt.cm.gray)
33 | 
34 | # Let's look at the jitters
35 | f, plts = plt.subplots(3,3, sharex=True, sharey=True)
36 | for i in range(3):
37 |     for j in range(3):
38 |         plts[i,j].pcolor(train[i + 3*j]) 
39 | 
40 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2017 Packt
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Chapter02/Basic neural networks/basic_nn.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | import numpy as np
 3 | import math
 4 | 
 5 | sess = tf.InteractiveSession()
 6 | 
 7 | # Some simple constants
 8 | x1 = tf.Variable(tf.truncated_normal([5],
 9 |                  mean=3, stddev=1./math.sqrt(5)))
10 | x2 = tf.Variable(tf.truncated_normal([5],
11 |                  mean=-1, stddev=1./math.sqrt(5)))
12 | x3 = tf.Variable(tf.truncated_normal([5],
13 |                  mean=0, stddev=1./math.sqrt(5)))
14 | 
15 | sess.run(tf.global_variables_initializer())
16 | 
17 | # Squaring makes large values extreme (but positive)
18 | # Be careful if you have negative values
19 | sqx2 = x2 * x2
20 | print(x2.eval())
21 | print(sqx2.eval())
22 | 
23 | # Logarithm makes small values more pronounced (and negative)
24 | # Be careful that your algorithm can handle negative numbers
25 | logx1 = tf.log(x1)
26 | print(x1.eval())
27 | print(logx1.eval())
28 | 
29 | # "sigmoid" is a common transformation in deep learning
30 | # Extreme values get flattened to +1 or 0
31 | # Inputs closer to zero stay similar, sigmoid(0) = 0.5
32 | sigx3 = tf.sigmoid(x3)
33 | print(x3.eval())
34 | print(sigx3.eval())
35 | 
36 | # We linearly combine multiple inputs, then transform
37 | w1 = tf.constant(0.1)
38 | w2 = tf.constant(0.2)
39 | sess.run(tf.global_variables_initializer())
40 | n1 = tf.sigmoid(w1*x1 + w2*x2)
41 | print((w1*x1).eval())
42 | print((w2*x2).eval())
43 | print(n1.eval())
44 | 


--------------------------------------------------------------------------------
/Chapter04/TensorFlow learn/learn_intro.py:
--------------------------------------------------------------------------------
 1 | # TF made EZ
 2 | import tensorflow.contrib.learn as learn
 3 | from tensorflow.contrib.learn.python.learn.estimators import estimator
 4 | 
 5 | # Some basics
 6 | import numpy as np
 7 | import math
 8 | import matplotlib.pyplot as plt
 9 | plt.ion()
10 | # Learn more sklearn
11 | # scikit-learn.org
12 | import sklearn
13 | from sklearn import metrics
14 | 
15 | # Seed the data
16 | np.random.seed(42)
17 | # Load data
18 | data = np.load('data_with_labels.npz')
19 | train = data['arr_0']/255.
20 | labels = data['arr_1']
21 | 
22 | # Split data into training and validation
23 | indices = np.random.permutation(train.shape[0])
24 | valid_cnt = int(train.shape[0] * 0.1)
25 | test_idx, training_idx = indices[:valid_cnt],\
26 |  indices[valid_cnt:]
27 | test, train = train[test_idx,:],\
28 |               train[training_idx,:]
29 | test_labels, train_labels = labels[test_idx],\
30 |                             labels[training_idx]
31 |  
32 | train = np.array(train,dtype=np.float32)
33 | test = np.array(test,dtype=np.float32)
34 | train_labels = np.array(train_labels,dtype=np.int32)
35 | test_labels = np.array(test_labels,dtype=np.int32)
36 | 
37 | # Convert features to learn style
38 | feature_columns = learn.infer_real_valued_columns_from_input(train.reshape([-1,36*36]))
39 | 
40 | # Logistic Regression
41 | classifier = estimator.SKCompat(learn.LinearClassifier(
42 | feature_columns = feature_columns,
43 | n_classes=5))
44 | 
45 | # One line training
46 | # steps is number of total batches
47 | # steps*batch_size/len(train) = num_epochs
48 | classifier.fit(train.reshape([-1,36*36]),
49 |  train_labels,
50 |  steps=1024,
51 |  batch_size=32)
52 |  
53 |  # sklearn compatible accuracy
54 | test_probs = classifier.predict(test.reshape([-1,36*36]))
55 | sklearn.metrics.accuracy_score(test_labels,
56 |  test_probs['classes'])


--------------------------------------------------------------------------------
/Chapter04/DNNs/cnn.py:
--------------------------------------------------------------------------------
 1 | # Access general TF functions
 2 | import tensorflow as tf
 3 | import tensorflow.contrib.layers as layers
 4 | 
 5 | def conv_learn(X, y, mode):
 6 |     # Ensure our images are 2d 
 7 |     X = tf.reshape(X, [-1, 36, 36, 1])
 8 |     # We'll need these in one-hot format
 9 |     y = tf.one_hot(tf.cast(y, tf.int32), 5, 1, 0)
10 | 
11 |     # conv layer will compute 4 kernels for each 5x5 patch
12 |     with tf.variable_scope('conv_layer'):
13 |         # 5x5 convolution, pad with zeros on edges
14 |         h1 = layers.convolution2d(X, num_outputs=4,
15 |                 kernel_size=[5, 5], 
16 |                 activation_fn=tf.nn.relu)
17 |         # 2x2 Max pooling, no padding on edges
18 |         p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
19 |                 strides=[1, 2, 2, 1], padding='VALID')
20 |                 
21 |         # Need to flatten conv output for use in dense layer
22 |     p1_size = np.product(
23 |               [s.value for s in p1.get_shape()[1:]])
24 |     p1f = tf.reshape(p1, [-1, p1_size ])
25 |     
26 |     # densely connected layer with 32 neurons and dropout
27 |     h_fc1 = layers.fully_connected(p1f,
28 |              5,
29 |              activation_fn=tf.nn.relu)
30 |     drop = layers.dropout(h_fc1, keep_prob=0.5, is_training=mode == tf.contrib.learn.ModeKeys.TRAIN)
31 |     
32 |     logits = layers.fully_connected(drop, 5, activation_fn=None)
33 |     loss = tf.losses.softmax_cross_entropy(y, logits)
34 |     # Setup the training function manually
35 |     train_op = layers.optimize_loss(
36 |         loss,
37 |         tf.contrib.framework.get_global_step(),
38 |         optimizer='Adam',
39 |         learning_rate=0.01)
40 |     return tf.argmax(logits, 1), loss, train_op
41 | # Use generic estimator with our function
42 | classifier = estimator.SKCompat(
43 |          learn.Estimator(
44 |          model_fn=conv_learn))
45 | 
46 | classifier.fit(train,train_labels,
47 |                 steps=1024,
48 |                 batch_size=32)
49 | 
50 | # simple accuracy
51 | metrics.accuracy_score(test_labels,classifier.predict(test))
52 | 
53 | 
54 | 
55 | 
56 | 


--------------------------------------------------------------------------------
/Chapter04/DNNs/extracting_weights.py:
--------------------------------------------------------------------------------
 1 | # Access general TF functions
 2 | import tensorflow as tf
 3 | import tensorflow.contrib.layers as layers
 4 | 
 5 | def conv_learn(X, y, mode):
 6 |     # Ensure our images are 2d 
 7 |     X = tf.reshape(X, [-1, 36, 36, 1])
 8 |     # We'll need these in one-hot format
 9 |     y = tf.one_hot(tf.cast(y, tf.int32), 5, 1, 0)
10 | 
11 |     # conv layer will compute 4 kernels for each 5x5 patch
12 |     with tf.variable_scope('conv_layer'):
13 |         # 5x5 convolution, pad with zeros on edges
14 |         h1 = layers.convolution2d(X, num_outputs=4,
15 |                 kernel_size=[5, 5], 
16 |                 activation_fn=tf.nn.relu)
17 |         # 2x2 Max pooling, no padding on edges
18 |         p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
19 |                 strides=[1, 2, 2, 1], padding='VALID')
20 |                 
21 |         # Need to flatten conv output for use in dense layer
22 |     p1_size = np.product(
23 |               [s.value for s in p1.get_shape()[1:]])
24 |     p1f = tf.reshape(p1, [-1, p1_size ])
25 |     
26 |     # densely connected layer with 32 neurons and dropout
27 |     h_fc1 = layers.fully_connected(p1f,
28 |              5,
29 |              activation_fn=tf.nn.relu)
30 |     drop = layers.dropout(h_fc1, keep_prob=0.5, is_training=mode == tf.contrib.learn.ModeKeys.TRAIN)
31 |     
32 |     logits = layers.fully_connected(drop, 5, activation_fn=None)
33 |     loss = tf.losses.softmax_cross_entropy(y, logits)
34 |     # Setup the training function manually
35 |     train_op = layers.optimize_loss(
36 |         loss,
37 |         tf.contrib.framework.get_global_step(),
38 |         optimizer='Adam',
39 |         learning_rate=0.01)
40 |     return tf.argmax(logits, 1), loss, train_op
41 | # Use generic estimator with our function
42 | classifier = estimator.SKCompat(
43 |          learn.Estimator(
44 |          model_fn=conv_learn))
45 | 
46 | classifier.fit(train,train_labels,
47 |                 steps=1024,
48 |                 batch_size=32)
49 | 
50 | # simple accuracy
51 | metrics.accuracy_score(test_labels,classifier.predict(test))
52 | 
53 | # See layer names
54 | print(classifier._estimator.get_variable_names())
55 | 
56 | 
57 | 
58 | 


--------------------------------------------------------------------------------
/Chapter04/DNNs/dnn.py:
--------------------------------------------------------------------------------
 1 | # TF made EZ
 2 | import tensorflow.contrib.learn as learn
 3 | from tensorflow.contrib.learn.python.learn.estimators import estimator
 4 | 
 5 | # Some basics
 6 | import numpy as np
 7 | import math
 8 | import matplotlib.pyplot as plt
 9 | plt.ion()
10 | # Learn more sklearn
11 | # scikit-learn.org
12 | import sklearn
13 | from sklearn import metrics
14 | 
15 | # Seed the data
16 | np.random.seed(42)
17 | # Load data
18 | data = np.load('data_with_labels.npz')
19 | train = data['arr_0']/255.
20 | labels = data['arr_1']
21 | 
22 | # Split data into training and validation
23 | indices = np.random.permutation(train.shape[0])
24 | valid_cnt = int(train.shape[0] * 0.1)
25 | test_idx, training_idx = indices[:valid_cnt],\
26 |  indices[valid_cnt:]
27 | test, train = train[test_idx,:],\
28 |               train[training_idx,:]
29 | test_labels, train_labels = labels[test_idx],\
30 |                             labels[training_idx]
31 |  
32 | train = np.array(train,dtype=np.float32)
33 | test = np.array(test,dtype=np.float32)
34 | train_labels = np.array(train_labels,dtype=np.int32)
35 | test_labels = np.array(test_labels,dtype=np.int32)
36 | 
37 | # Convert features to learn style
38 | feature_columns = learn.infer_real_valued_columns_from_input(train.reshape([-1,36*36]))
39 | 
40 | # Logistic Regression
41 | classifier = estimator.SKCompat(learn.LinearClassifier(
42 | feature_columns = feature_columns,
43 | n_classes=5))
44 | 
45 | # One line training
46 | # steps is number of total batches
47 | # steps*batch_size/len(train) = num_epochs
48 | classifier.fit(train.reshape([-1,36*36]),
49 |  train_labels,
50 |  steps=1024,
51 |  batch_size=32)
52 |  
53 | # sklearn compatible accuracy
54 | test_probs = classifier.predict(test.reshape([-1,36*36]))
55 | sklearn.metrics.accuracy_score(test_labels,
56 |  test_probs['classes'])
57 | 
58 | # Dense neural net
59 | classifier = estimator.SKCompat(learn.DNNClassifier(
60 |  feature_columns = feature_columns,
61 |  hidden_units=[10,5],
62 |  n_classes=5,
63 |  optimizer='Adam'))
64 | 
65 | # Same training call
66 | classifier.fit(train.reshape([-1,36*36]),
67 |  train_labels,
68 |  steps=1024,
69 |  batch_size=32)
70 | # simple accuracy
71 | test_probs = classifier.predict(test.reshape([-1,36*36]))
72 | sklearn.metrics.accuracy_score(test_labels,
73 |  test_probs['classes'])
74 | # confusion is easy
75 | train_probs = classifier.predict(train.reshape([-1,36*36]))
76 | conf = metrics.confusion_matrix(train_labels,
77 |  train_probs['classes'])
78 | print(conf)


--------------------------------------------------------------------------------
/Chapter01/Simple Computations/simple.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | 
 3 | # You can create constants in TF to
 4 | # hold specific values
 5 | a = tf.constant(1)
 6 | b = tf.constant(2)
 7 | 
 8 | # Of course you can add, multiply,
 9 | # and compute on these as you like
10 | c = a + b
11 | d = a * b
12 | 
13 | # TF numbers are stored in "tensors",
14 | # a fancy term for multidimensional arrays
15 | # If you pass TF a Python list, it can convert it
16 | V1 = tf.constant([1., 2.])   # Vector, 1-dimensional
17 | V2 = tf.constant([3., 4.])   # Vector, 1-dimensional
18 | M = tf.constant([[1., 2.]])             # Matrix, 2d
19 | N = tf.constant([[1., 2.],[3.,4.]])     # Matrix, 2d
20 | K = tf.constant([[[1., 2.],[3.,4.]]])   # Tensor, 3d+
21 | 
22 | # You can also compute on tensors
23 | # like you did scalars, but be careful of shape
24 | V3 = V1 + V2
25 | 
26 | # Operations are element-wise by default
27 | M2 = M * M
28 | 
29 | # True matrix multiplication requires a special call
30 | NN = tf.matmul(N,N)
31 | 
32 | # The above code only defines a TF "graph".
33 | # Nothing has been computed yet
34 | # For that, you first need to create a TF "session"
35 | sess = tf.Session()
36 | # Note the parallelism information TF
37 | # reports to you when starting a session
38 | 
39 | # Now you can run specific nodes of your graph,
40 | # i.e. the variables you've named
41 | output = sess.run(NN)
42 | print("NN is:")
43 | print(output)
44 | 
45 | # Remember to close your session
46 | # when you're done using it
47 | sess.close()
48 | 
49 | # Often, we work interactively,
50 | # it's convenient to use a simplified session
51 | sess = tf.InteractiveSession()
52 | 
53 | # Now we can compute any node
54 | print("M2 is:")
55 | print(M2.eval())
56 | 
57 | # TF "variables" can change value,
58 | # useful for updating model weights
59 | W = tf.Variable(0, name="weight")
60 | 
61 | # But variables must be initialized by TF before use
62 | init_op = tf.initialize_all_variables()
63 | sess.run(init_op)
64 | 
65 | print("W is:")
66 | print(W.eval())
67 | 
68 | W += a
69 | print("W after adding a:")
70 | print(W.eval())
71 | 
72 | W += a
73 | print("W after adding a again:")
74 | print(W.eval())
75 | 
76 | # You can return or supply arbitrary nodes,
77 | # i.e. check an intermediate value or
78 | # sub your value in the middle of a computation
79 | 
80 | E = d + b # 1*2 + 2 = 4
81 | 
82 | print("E as defined:")
83 | print(E.eval())
84 | 
85 | # Let's see what d was at the same time
86 | print("E and d:")
87 | print(sess.run([E,d]))
88 | 
89 | # Use a custom d by specifying a dictionary
90 | print("E with custom d=4:")
91 | print(sess.run(E, feed_dict = {d:4.}))
92 | 


--------------------------------------------------------------------------------
/Chapter03/Pooling layer application/conv_simple.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import math
  3 | import numpy as np
  4 | 
  5 | sess = tf.InteractiveSession()
  6 | 
  7 | # Make some fake data, 1 data points
  8 | image = np.random.randint(10,size=[1,10,10]) + np.eye(10)*10
  9 | 
 10 | # TensorFlow placeholder
 11 | # None is for batch processing 
 12 | # (-1 keeps same size)
 13 | # 10x10 is the shape
 14 | # 1 is the number of "channels" 
 15 | # (like RGB colors or gray)
 16 | x = tf.placeholder("float", [None, 10, 10])
 17 | x_im = tf.reshape(x, [-1,10,10,1])
 18 | 
 19 | ### Convolutional Layer
 20 | 
 21 | # Window size to use, 3x3 here
 22 | winx = 3
 23 | winy = 3
 24 | 
 25 | # How many features to compute on the window
 26 | num_filters = 2
 27 | 
 28 | # Weight shape should match window size
 29 | # The '1' represents the number of 
 30 | # input "channels" (colors)
 31 | W1 = tf.Variable(tf.truncated_normal(
 32 |     [winx, winy,1, num_filters],
 33 |     stddev=1./math.sqrt(winx*winy)))
 34 | b1 = tf.Variable(tf.constant(
 35 |     0.1,shape=[num_filters]))
 36 | 
 37 | # 3x3 convolution, Pad with zeros on edges
 38 | # Strides is how to step, here 1 pixel at a time
 39 | xw = tf.nn.conv2d(x_im, W1,
 40 |         strides=[1, 1, 1, 1],
 41 |         padding='SAME')
 42 | h1 = tf.nn.relu(xw + b1)
 43 | 
 44 | # Remember to initialize!
 45 | sess.run(tf.initialize_all_variables())
 46 | 
 47 | # Peek inside
 48 | H = h1.eval(feed_dict = {x: image})
 49 | 
 50 | # Let's take a look
 51 | import matplotlib.pyplot as plt
 52 | plt.ion()
 53 | 
 54 | # Original
 55 | plt.matshow(image[0])
 56 | plt.colorbar()
 57 | 
 58 | # Conv channel 1
 59 | plt.matshow(H[0,:,:,0])
 60 | plt.colorbar()
 61 | 
 62 | # Conv channel 2
 63 | plt.matshow(H[0,:,:,1])
 64 | plt.colorbar()
 65 | 
 66 | ### End Video 3.2
 67 | 
 68 | ### Begin Video 3.4
 69 | 
 70 | ### Pooling Layer
 71 | # "Max" pooling keeps best of 2x2 square
 72 | # in h1 output
 73 | # ksize defines size of this block
 74 | # "VALID" padding means incomplete squares are
 75 | # not used
 76 | # Stride of 2x2 means no overlap of 2x2 blocks
 77 | p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
 78 |           strides=[1, 2, 2, 1], padding='VALID')
 79 | 
 80 | # We automatically determine the size
 81 | p1_size = np.product([s.value for s in p1.get_shape()[1:]])
 82 | 
 83 | # Need to flatten convolutional output for use
 84 | # in a dense layer
 85 | # -1 chooses appropriate shape to keep overall
 86 | # size the same
 87 | p1f = tf.reshape(p1, [-1, p1_size ])
 88 | 
 89 | # Pooling Layer before flattening
 90 | # Note how it's only 5x5, because we took the
 91 | # best of every 2x2 window
 92 | P = p1.eval(feed_dict = {x: image})
 93 | plt.matshow(P[0,:,:,0])
 94 | plt.colorbar()
 95 | 
 96 | 
 97 | 
 98 | 
 99 | 
100 | 
101 | 
102 | 
103 | 


--------------------------------------------------------------------------------
/Chapter03/Convolutional layer application/conv_simple.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import math
  3 | import numpy as np
  4 | 
  5 | sess = tf.InteractiveSession()
  6 | 
  7 | # Make some fake data, 1 data points
  8 | image = np.random.randint(10,size=[1,10,10]) + np.eye(10)*10
  9 | 
 10 | # TensorFlow placeholder
 11 | # None is for batch processing 
 12 | # (-1 keeps same size)
 13 | # 10x10 is the shape
 14 | # 1 is the number of "channels" 
 15 | # (like RGB colors or gray)
 16 | x = tf.placeholder("float", [None, 10, 10])
 17 | x_im = tf.reshape(x, [-1,10,10,1])
 18 | 
 19 | ### Convolutional Layer
 20 | 
 21 | # Window size to use, 3x3 here
 22 | winx = 3
 23 | winy = 3
 24 | 
 25 | # How many features to compute on the window
 26 | num_filters = 2
 27 | 
 28 | # Weight shape should match window size
 29 | # The '1' represents the number of 
 30 | # input "channels" (colors)
 31 | W1 = tf.Variable(tf.truncated_normal(
 32 |     [winx, winy,1, num_filters],
 33 |     stddev=1./math.sqrt(winx*winy)))
 34 | b1 = tf.Variable(tf.constant(
 35 |     0.1,shape=[num_filters]))
 36 | 
 37 | # 3x3 convolution, Pad with zeros on edges
 38 | # Strides is how to step, here 1 pixel at a time
 39 | xw = tf.nn.conv2d(x_im, W1,
 40 |         strides=[1, 1, 1, 1],
 41 |         padding='SAME')
 42 | h1 = tf.nn.relu(xw + b1)
 43 | 
 44 | # Remember to initialize!
 45 | sess.run(tf.global_variables_initializer())
 46 | 
 47 | # Peek inside
 48 | H = h1.eval(feed_dict = {x: image})
 49 | 
 50 | # Let's take a look
 51 | import matplotlib.pyplot as plt
 52 | plt.ion()
 53 | 
 54 | # Original
 55 | plt.matshow(image[0])
 56 | plt.colorbar()
 57 | 
 58 | # Conv channel 1
 59 | plt.matshow(H[0,:,:,0])
 60 | plt.colorbar()
 61 | 
 62 | # Conv channel 2
 63 | plt.matshow(H[0,:,:,1])
 64 | plt.colorbar()
 65 | 
 66 | ### End Video 3.2
 67 | 
 68 | ### Begin Video 3.4
 69 | 
 70 | ### Pooling Layer
 71 | # "Max" pooling keeps best of 2x2 square
 72 | # in h1 output
 73 | # ksize defines size of this block
 74 | # "VALID" padding means incomplete squares are
 75 | # not used
 76 | # Stride of 2x2 means no overlap of 2x2 blocks
 77 | p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
 78 |           strides=[1, 2, 2, 1], padding='VALID')
 79 | 
 80 | # We automatically determine the size
 81 | p1_size = np.product([s.value for s in p1.get_shape()[1:]])
 82 | 
 83 | # Need to flatten convolutional output for use
 84 | # in a dense layer
 85 | # -1 chooses appropriate shape to keep overall
 86 | # size the same
 87 | p1f = tf.reshape(p1, [-1, p1_size ])
 88 | 
 89 | # Pooling Layer before flattening
 90 | # Note how it's only 5x5, because we took the
 91 | # best of every 2x2 window
 92 | P = p1.eval(feed_dict = {x: image})
 93 | plt.matshow(P[0,:,:,0])
 94 | plt.colorbar()
 95 | 
 96 | 
 97 | 
 98 | 
 99 | 
100 | 
101 | 
102 | 
103 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | 
 4 | # Hands-On Deep Learning with TensorFlow
 5 | This is the code repository for [Hands-On Deep Learning with TensorFlow](https://www.packtpub.com/big-data-and-business-intelligence/hands-deep-learning-tensorflow?utm_source=github&utm_medium=repository&utm_campaign=9781787282773), published by [Packt](https://www.packtpub.com/?utm_source=github). It contains all the supporting project files necessary to work through the book from start to finish.
 6 | ## About the Book
 7 | 
 8 | Dan Van Boxel’s Deep Learning with TensorFlow is based on Dan’s best-selling TensorFlow video course. With deep learning going mainstream, making sense of data and getting accurate results using deep networks is possible. Dan Van Boxel will be your guide to exploring the possibilities with deep learning; he will enable you to understand data like never before. With the efficiency and simplicity of TensorFlow, you will be able to process your data and gain insights that will change how you look at data.
 9 | 
10 | With Dan’s guidance, you will dig deeper into the hidden layers of abstraction using raw data. Dan then shows you various complex algorithms for deep learning and various examples that use these deep neural networks. You will also learn how to train your machine to craft new features to make sense of deeper layers of data.
11 | 
12 | In this book, Dan shares his knowledge across topics such as logistic regression, convolutional neural networks, recurrent neural networks, training deep networks, and high level interfaces. With the help of novel practical examples, you will become an ace at advanced multilayer networks, image recognition, and beyond.
13 | 
14 | ## Instructions and Navigation
15 | All of the code is organized into folders. Each folder starts with a number followed by the application name. For example, Chapter02.
16 | 
17 | 
18 | 
19 | The code will look like the following:
20 | ```
21 | import tensorflow as tf
22 | # You can create constants in TF to hold specific values
23 | a = tf.constant(1)
24 | b = tf.constant(2)
25 | ```
26 | 
27 | While this book will show you how to install TensorFlow, there are a few dependencies you need to be aware of. At a minimum, you need a recent version of Python 2 or 3 and NumPy. To get the most out of the book, you should also have Matplotlib and IPython.
28 | 
29 | ## Related Products
30 | * [Deep Learning with TensorFlow](https://www.packtpub.com/big-data-and-business-intelligence/deep-learning-tensorflow?utm_source=github&utm_medium=repository&utm_campaign=9781786469786)
31 | 
32 | * [Learning Computer Vision with TensorFlow [Video]](https://www.packtpub.com/all/learning-computer-vision-tensorflow-video?utm_source=github&utm_medium=repository&utm_campaign=9781788292573)
33 | 
34 | * [TensorFlow Machine Learning Cookbook](https://www.packtpub.com/big-data-and-business-intelligence/tensorflow-machine-learning-cookbook?utm_source=github&utm_medium=repository&utm_campaign=9781786462169)
35 | 
36 | 
37 | 
38 | ### Download a free PDF
39 | 
40 |  <i>If you have already purchased a print or Kindle version of this book, you can get a DRM-free PDF version at no cost.<br>Simply click on the link to claim your free PDF.</i>
41 | <p align="center"> <a href="https://packt.link/free-ebook/9781787282773">https://packt.link/free-ebook/9781787282773 </a> </p>


--------------------------------------------------------------------------------
/Chapter01/Logistic regression training/logistic.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | %autoindent
  4 | try:
  5 |     from tqdm import tqdm
  6 | except ImportError:
  7 |     def tqdm(x, *args, **kwargs):
  8 |         return x
  9 | 
 10 | # Set random seed
 11 | np.random.seed(0)	
 12 | 
 13 | # Load data
 14 | data = np.load('data_with_labels.npz')
 15 | train = data['arr_0']/255.
 16 | labels = data['arr_1']
 17 | 
 18 | # Look at some data
 19 | print(train[0])
 20 | print(labels[0])
 21 | 
 22 | # If you have matplotlib installed
 23 | import matplotlib.pyplot as plt
 24 | plt.ion()
 25 | 
 26 | # Let's look at a subplot of one of A in each font
 27 | f, plts = plt.subplots(5, sharex=True)
 28 | c = 91
 29 | for i in range(5):
 30 |     plts[i].pcolor(train[c + i * 558],
 31 |                    cmap=plt.cm.gray_r)
 32 | 
 33 | def to_onehot(labels,nclasses = 5):
 34 |     '''
 35 |     Convert labels to "one-hot" format.
 36 |     >>> a = [0,1,2,3]
 37 |     >>> to_onehot(a,5)
 38 |     array([[ 1.,  0.,  0.,  0.,  0.],
 39 |            [ 0.,  1.,  0.,  0.,  0.],
 40 |            [ 0.,  0.,  1.,  0.,  0.],
 41 |            [ 0.,  0.,  0.,  1.,  0.]])
 42 |     '''
 43 |     outlabels = np.zeros((len(labels),nclasses))
 44 |     for i,l in enumerate(labels):
 45 |         outlabels[i,l] = 1
 46 |     return outlabels
 47 | 
 48 | onehot = to_onehot(labels)
 49 | 
 50 | # Split data into training and validation
 51 | indices = np.random.permutation(train.shape[0])
 52 | valid_cnt = int(train.shape[0] * 0.1)
 53 | test_idx, training_idx = indices[:valid_cnt],\
 54 |                          indices[valid_cnt:]
 55 | test, train = train[test_idx,:],\
 56 |               train[training_idx,:]
 57 | onehot_test, onehot_train = onehot[test_idx,:],\
 58 |                         onehot[training_idx,:]
 59 | 
 60 | 
 61 | sess = tf.InteractiveSession()
 62 | 
 63 | 
 64 | # These will be inputs
 65 | ## Input pixels, flattened
 66 | x = tf.placeholder("float", [None, 1296])
 67 | ## Known labels
 68 | y_ = tf.placeholder("float", [None,5])
 69 | 
 70 | # Variables
 71 | W = tf.Variable(tf.zeros([1296,5]))
 72 | b = tf.Variable(tf.zeros([5]))
 73 | 
 74 | # Just initialize
 75 | sess.run(tf.initialize_all_variables())
 76 | 
 77 | # Define model
 78 | y = tf.nn.softmax(tf.matmul(x,W) + b)
 79 | 
 80 | ### End model specification, begin training code
 81 | 
 82 | 
 83 | # Climb on cross-entropy
 84 | cross_entropy = tf.reduce_mean(
 85 |         tf.nn.softmax_cross_entropy_with_logits(
 86 |         logits = y + 1e-50, labels = y_))
 87 | 
 88 | # How we train
 89 | train_step = tf.train.GradientDescentOptimizer(
 90 |                 0.02).minimize(cross_entropy)
 91 | 
 92 | # Define accuracy
 93 | correct_prediction = tf.equal(tf.argmax(y,1),
 94 |                      tf.argmax(y_,1))
 95 | accuracy = tf.reduce_mean(tf.cast(
 96 |            correct_prediction, "float"))
 97 | 
 98 | # Actually train
 99 | epochs = 1000
100 | train_acc = np.zeros(epochs//10)
101 | test_acc = np.zeros(epochs//10)
102 | for i in tqdm(range(epochs)):
103 |     # Record summary data, and the accuracy
104 |     if i % 10 == 0:
105 |         # Check accuracy on train set
106 |         A = accuracy.eval(feed_dict={
107 |             x: train.reshape([-1,1296]),
108 |             y_: onehot_train})
109 |         train_acc[i//10] = A
110 |         # And now the validation set
111 |         A = accuracy.eval(feed_dict={
112 |             x: test.reshape([-1,1296]),
113 |             y_: onehot_test})
114 |         test_acc[i//10] = A
115 |     train_step.run(feed_dict={
116 |         x: train.reshape([-1,1296]),
117 |         y_: onehot_train})
118 | 
119 | # Notice that accuracy flattens out
120 | print(train_acc[-1])
121 | print(test_acc[-1])
122 | 
123 | # Plot the accuracy curves
124 | plt.figure(figsize=(6,6))
125 | plt.plot(train_acc,'bo')
126 | plt.plot(test_acc,'rx')
127 | 
128 | # Look at a subplot of the weights for each font
129 | f, plts = plt.subplots(5, sharex=True)
130 | for i in range(5):
131 |     plts[i].pcolor(W.eval()[:,i].reshape([36,36]))
132 | 
133 | 
134 | 


--------------------------------------------------------------------------------
/Chapter01/Logistic regression model building/logistic.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | %autoindent
  4 | try:
  5 |     from tqdm import tqdm
  6 | except ImportError:
  7 |     def tqdm(x, *args, **kwargs):
  8 |         return x
  9 | 
 10 | # Set random seed
 11 | np.random.seed(0)		
 12 | 
 13 | # Load data
 14 | data = np.load('data_with_labels.npz')
 15 | train = data['arr_0']/255.
 16 | labels = data['arr_1']
 17 | 
 18 | # Look at some data
 19 | print(train[0])
 20 | print(labels[0])
 21 | 
 22 | # If you have matplotlib installed
 23 | import matplotlib.pyplot as plt
 24 | plt.ion()
 25 | 
 26 | # Let's look at a subplot of one of A in each font
 27 | f, plts = plt.subplots(5, sharex=True)
 28 | c = 91
 29 | for i in range(5):
 30 |     plts[i].pcolor(train[c + i * 558],
 31 |                    cmap=plt.cm.gray_r)
 32 | 
 33 | def to_onehot(labels,nclasses = 5):
 34 |     '''
 35 |     Convert labels to "one-hot" format.
 36 |     >>> a = [0,1,2,3]
 37 |     >>> to_onehot(a,5)
 38 |     array([[ 1.,  0.,  0.,  0.,  0.],
 39 |            [ 0.,  1.,  0.,  0.,  0.],
 40 |            [ 0.,  0.,  1.,  0.,  0.],
 41 |            [ 0.,  0.,  0.,  1.,  0.]])
 42 |     '''
 43 |     outlabels = np.zeros((len(labels),nclasses))
 44 |     for i,l in enumerate(labels):
 45 |         outlabels[i,l] = 1
 46 |     return outlabels
 47 | 
 48 | onehot = to_onehot(labels)
 49 | 
 50 | # Split data into training and validation
 51 | indices = np.random.permutation(train.shape[0])
 52 | valid_cnt = int(train.shape[0] * 0.1)
 53 | test_idx, training_idx = indices[:valid_cnt],\
 54 |                          indices[valid_cnt:]
 55 | test, train = train[test_idx,:],\
 56 |               train[training_idx,:]
 57 | onehot_test, onehot_train = onehot[test_idx,:],\
 58 |                         onehot[training_idx,:]
 59 | 
 60 | 
 61 | sess = tf.InteractiveSession()
 62 | 
 63 | 
 64 | # These will be inputs
 65 | ## Input pixels, flattened
 66 | x = tf.placeholder("float", [None, 1296])
 67 | ## Known labels
 68 | y_ = tf.placeholder("float", [None,5])
 69 | 
 70 | # Variables
 71 | W = tf.Variable(tf.zeros([1296,5]))
 72 | b = tf.Variable(tf.zeros([5]))
 73 | 
 74 | # Just initialize
 75 | sess.run(tf.global_variables_initializer())
 76 | 
 77 | # Define model
 78 | y = tf.nn.softmax(tf.matmul(x,W) + b)
 79 | 
 80 | ### End model specification, begin training code
 81 | 
 82 | 
 83 | # Climb on cross-entropy
 84 | cross_entropy = tf.reduce_mean(
 85 |         tf.nn.softmax_cross_entropy_with_logits(
 86 |         logits = y + 1e-50, labels = y_))
 87 | 
 88 | # How we train
 89 | train_step = tf.train.GradientDescentOptimizer(
 90 |                 0.02).minimize(cross_entropy)
 91 | 
 92 | # Define accuracy
 93 | correct_prediction = tf.equal(tf.argmax(y,1),
 94 |                      tf.argmax(y_,1))
 95 | accuracy = tf.reduce_mean(tf.cast(
 96 |            correct_prediction, "float"))
 97 | 
 98 | # Actually train
 99 | epochs = 1000
100 | train_acc = np.zeros(epochs//10)
101 | test_acc = np.zeros(epochs//10)
102 | for i in tqdm(range(epochs)):
103 |     # Record summary data, and the accuracy
104 |     if i % 10 == 0:
105 |         # Check accuracy on train set
106 |         A = accuracy.eval(feed_dict={
107 |             x: train.reshape([-1,1296]),
108 |             y_: onehot_train})
109 |         train_acc[i//10] = A
110 |         # And now the validation set
111 |         A = accuracy.eval(feed_dict={
112 |             x: test.reshape([-1,1296]),
113 |             y_: onehot_test})
114 |         test_acc[i//10] = A
115 |     train_step.run(feed_dict={
116 |         x: train.reshape([-1,1296]),
117 |         y_: onehot_train})
118 | 
119 | # Notice that accuracy flattens out
120 | print(train_acc[-1])
121 | print(test_acc[-1])
122 | 
123 | # Plot the accuracy curves
124 | plt.figure(figsize=(6,6))
125 | plt.plot(train_acc,'bo')
126 | plt.plot(test_acc,'rx')
127 | 
128 | # Look at a subplot of the weights for each font
129 | f, plts = plt.subplots(5, sharex=True)
130 | for i in range(5):
131 |     plts[i].pcolor(W.eval()[:,i].reshape([36,36]))
132 | 
133 | 
134 | 


--------------------------------------------------------------------------------
/Chapter02/Single hidden layer explained/single_hidden.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | import math
  4 | %autoindent
  5 | 
  6 | try:
  7 |     from tqdm import tqdm
  8 | except ImportError:
  9 |     def tqdm(x, *args, **kwargs):
 10 |         return x
 11 | 		
 12 | # Load data
 13 | data = np.load('data_with_labels.npz')
 14 | train = data['arr_0']/255.
 15 | labels = data['arr_1']
 16 | 
 17 | # Look at some data
 18 | print(train[0])
 19 | print(labels[0])
 20 | 
 21 | # If you have matplotlib installed
 22 | import matplotlib.pyplot as plt
 23 | plt.ion()
 24 | 
 25 | def to_onehot(labels,nclasses = 5):
 26 |     '''
 27 |     Convert labels to "one-hot" format.
 28 | 
 29 |     >>> a = [0,1,2,3]
 30 |     >>> to_onehot(a,5)
 31 |     array([[ 1.,  0.,  0.,  0.,  0.],
 32 |            [ 0.,  1.,  0.,  0.,  0.],
 33 |            [ 0.,  0.,  1.,  0.,  0.],
 34 |            [ 0.,  0.,  0.,  1.,  0.]])
 35 |     '''
 36 |     outlabels = np.zeros((len(labels),nclasses))
 37 |     for i,l in enumerate(labels):
 38 |         outlabels[i,l] = 1
 39 |     return outlabels
 40 | 
 41 | onehot = to_onehot(labels)
 42 | 
 43 | # Split data into training and validation
 44 | indices = np.random.permutation(train.shape[0])
 45 | valid_cnt = int(train.shape[0] * 0.1)
 46 | test_idx, training_idx = indices[:valid_cnt], indices[valid_cnt:]
 47 | test, train = train[test_idx,:], train[training_idx,:]
 48 | onehot_test, onehot_train = onehot[test_idx,:], onehot[training_idx,:]
 49 | 
 50 | sess = tf.InteractiveSession()
 51 | 
 52 | 
 53 | # These will be inputs
 54 | ## Input pixels, flattened
 55 | x = tf.placeholder("float", [None, 1296])
 56 | ## Known labels
 57 | y_ = tf.placeholder("float", [None,5])
 58 | 
 59 | # Hidden layer
 60 | num_hidden = 128
 61 | W1 = tf.Variable(tf.truncated_normal([1296, num_hidden],
 62 |                                         stddev=1./math.sqrt(1296)))
 63 | b1 = tf.Variable(tf.constant(0.1,shape=[num_hidden]))
 64 | h1 = tf.sigmoid(tf.matmul(x,W1) + b1)
 65 | 
 66 | # Output Layer
 67 | W2 = tf.Variable(tf.truncated_normal([num_hidden, 5],
 68 |                                         stddev=1./math.sqrt(5)))
 69 | b2 = tf.Variable(tf.constant(0.1,shape=[5]))
 70 | 
 71 | # Just initialize
 72 | sess.run(tf.global_variables_initializer())
 73 | 
 74 | # Define model
 75 | y = tf.nn.softmax(tf.matmul(h1,W2) + b2)
 76 | 
 77 | ### End model specification, begin training code
 78 | 
 79 | 
 80 | # Climb on cross-entropy
 81 | cross_entropy = tf.reduce_mean(
 82 |         tf.nn.softmax_cross_entropy_with_logits(logits = y + 1e-50, labels = y_))
 83 | 
 84 | # How we train
 85 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
 86 | 
 87 | # Define accuracy
 88 | correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
 89 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
 90 | 
 91 | # Actually train
 92 | epochs = 5000
 93 | train_acc = np.zeros(epochs//10)
 94 | test_acc = np.zeros(epochs//10)
 95 | for i in tqdm(range(epochs), ascii=True):
 96 |     if i % 10 == 0:  # Record summary data, and the accuracy
 97 |         # Check accuracy on train set
 98 |         A = accuracy.eval(feed_dict={x: train.reshape([-1,1296]), y_: onehot_train})
 99 |         train_acc[i//10] = A
100 | 
101 |         # And now the validation set
102 |         A = accuracy.eval(feed_dict={x: test.reshape([-1,1296]), y_: onehot_test})
103 |         test_acc[i//10] = A
104 |     train_step.run(feed_dict={x: train.reshape([-1,1296]), y_: onehot_train})
105 | 
106 | # Plot the accuracy curves
107 | plt.figure(figsize=(6,6))
108 | plt.plot(train_acc,'bo')
109 | plt.plot(test_acc,'rx')
110 | 
111 | # Look at the final testing confusion matrix
112 | pred = np.argmax(y.eval(feed_dict={x: test.reshape([-1,1296]), y_: onehot_test}), axis = 1)
113 | conf = np.zeros([5,5])
114 | for p,t in zip(pred,np.argmax(onehot_test,axis=1)):
115 |     conf[t,p] += 1
116 | 
117 | plt.matshow(conf)
118 | plt.colorbar()
119 | 
120 | 
121 | # Let's look at a subplot of some weights
122 | plt.figure(figsize(6,6))
123 | f, plts = plt.subplots(4,8, sharex=True)
124 | for i in range(32):
125 |     plts[i//8, i%8].pcolormesh(W1.eval()[:,i].reshape([36,36]))
126 | 
127 | 
128 | 


--------------------------------------------------------------------------------
/Chapter02/Single hidden layer model/single_hidden.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | import math
  4 | 
  5 | %autoindent
  6 | 
  7 | try:
  8 |     from tqdm import tqdm
  9 | except ImportError:
 10 |     def tqdm(x, *args, **kwargs):
 11 |         return x
 12 | 
 13 | # Load data
 14 | data = np.load('data_with_labels.npz')
 15 | train = data['arr_0']/255.
 16 | labels = data['arr_1']
 17 | 
 18 | # Look at some data
 19 | print(train[0])
 20 | print(labels[0])
 21 | 
 22 | # If you have matplotlib installed
 23 | import matplotlib.pyplot as plt
 24 | plt.ion()
 25 | 
 26 | def to_onehot(labels,nclasses = 5):
 27 |     '''
 28 |     Convert labels to "one-hot" format.
 29 | 
 30 |     >>> a = [0,1,2,3]
 31 |     >>> to_onehot(a,5)
 32 |     array([[ 1.,  0.,  0.,  0.,  0.],
 33 |            [ 0.,  1.,  0.,  0.,  0.],
 34 |            [ 0.,  0.,  1.,  0.,  0.],
 35 |            [ 0.,  0.,  0.,  1.,  0.]])
 36 |     '''
 37 |     outlabels = np.zeros((len(labels),nclasses))
 38 |     for i,l in enumerate(labels):
 39 |         outlabels[i,l] = 1
 40 |     return outlabels
 41 | 
 42 | onehot = to_onehot(labels)
 43 | 
 44 | # Split data into training and validation
 45 | indices = np.random.permutation(train.shape[0])
 46 | valid_cnt = int(train.shape[0] * 0.1)
 47 | test_idx, training_idx = indices[:valid_cnt], indices[valid_cnt:]
 48 | test, train = train[test_idx,:], train[training_idx,:]
 49 | onehot_test, onehot_train = onehot[test_idx,:], onehot[training_idx,:]
 50 | 
 51 | sess = tf.InteractiveSession()
 52 | 
 53 | 
 54 | # These will be inputs
 55 | ## Input pixels, flattened
 56 | x = tf.placeholder("float", [None, 1296])
 57 | ## Known labels
 58 | y_ = tf.placeholder("float", [None,5])
 59 | 
 60 | # Hidden layer
 61 | num_hidden = 128
 62 | W1 = tf.Variable(tf.truncated_normal([1296, num_hidden],
 63 |                                         stddev=1./math.sqrt(1296)))
 64 | b1 = tf.Variable(tf.constant(0.1,shape=[num_hidden]))
 65 | h1 = tf.sigmoid(tf.matmul(x,W1) + b1)
 66 | 
 67 | # Output Layer
 68 | W2 = tf.Variable(tf.truncated_normal([num_hidden, 5],
 69 |                                         stddev=1./math.sqrt(5)))
 70 | b2 = tf.Variable(tf.constant(0.1,shape=[5]))
 71 | 
 72 | # Just initialize
 73 | sess.run(tf.global_variables_initializer())
 74 | 
 75 | # Define model
 76 | y = tf.nn.softmax(tf.matmul(h1,W2) + b2)
 77 | 
 78 | ### End model specification, begin training code
 79 | 
 80 | 
 81 | # Climb on cross-entropy
 82 | cross_entropy = tf.reduce_mean(
 83 |         tf.nn.softmax_cross_entropy_with_logits(logits = y + 1e-50, labels = y_))
 84 | 
 85 | # How we train
 86 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
 87 | 
 88 | # Define accuracy
 89 | correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
 90 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
 91 | 
 92 | # Actually train
 93 | epochs = 5000
 94 | train_acc = np.zeros(epochs//10)
 95 | test_acc = np.zeros(epochs//10)
 96 | for i in tqdm(range(epochs), ascii=True):
 97 |     if i % 10 == 0:  # Record summary data, and the accuracy
 98 |         # Check accuracy on train set
 99 |         A = accuracy.eval(feed_dict={x: train.reshape([-1,1296]), y_: onehot_train})
100 |         train_acc[i//10] = A
101 | 
102 |         # And now the validation set
103 |         A = accuracy.eval(feed_dict={x: test.reshape([-1,1296]), y_: onehot_test})
104 |         test_acc[i//10] = A
105 |     train_step.run(feed_dict={x: train.reshape([-1,1296]), y_: onehot_train})
106 | 
107 | # Plot the accuracy curves
108 | plt.figure(figsize=(6,6))
109 | plt.plot(train_acc,'bo')
110 | plt.plot(test_acc,'rx')
111 | 
112 | # Look at the final testing confusion matrix
113 | pred = np.argmax(y.eval(feed_dict={x: test.reshape([-1,1296]), y_: onehot_test}), axis = 1)
114 | conf = np.zeros([5,5])
115 | for p,t in zip(pred,np.argmax(onehot_test,axis=1)):
116 |     conf[t,p] += 1
117 | 
118 | plt.matshow(conf)
119 | plt.colorbar()
120 | 
121 | 
122 | # Let's look at a subplot of some weights
123 | plt.figure(figsize=(6,6))
124 | f, plts = plt.subplots(4,8, sharex=True)
125 | for i in range(32):
126 |     plts[i//8, i%8].pcolormesh(W1.eval()[:,i].reshape([36,36]))
127 | 
128 | 
129 | 


--------------------------------------------------------------------------------
/Chapter02/Results of the multiple hidden layer/mlp.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | import math
  4 | from tqdm import tqdm
  5 | %autoindent
  6 | 
  7 | # Load data
  8 | data = np.load('data_with_labels.npz')
  9 | train = data['arr_0']/255.
 10 | labels = data['arr_1']
 11 | 
 12 | # Look at some data
 13 | print(train[0])
 14 | print(labels[0])
 15 | 
 16 | # If you have matplotlib installed
 17 | import matplotlib.pyplot as plt
 18 | plt.ion()
 19 | 
 20 | def to_onehot(labels,nclasses = 5):
 21 |     '''
 22 |     Convert labels to "one-hot" format.
 23 | 
 24 |     >>> a = [0,1,2,3]
 25 |     >>> to_onehot(a,5)
 26 |     array([[ 1.,  0.,  0.,  0.,  0.],
 27 |            [ 0.,  1.,  0.,  0.,  0.],
 28 |            [ 0.,  0.,  1.,  0.,  0.],
 29 |            [ 0.,  0.,  0.,  1.,  0.]])
 30 |     '''
 31 |     outlabels = np.zeros((len(labels),nclasses))
 32 |     for i,l in enumerate(labels):
 33 |         outlabels[i,l] = 1
 34 |     return outlabels
 35 | 
 36 | onehot = to_onehot(labels)
 37 | 
 38 | # Split data into training and validation
 39 | indices = np.random.permutation(train.shape[0])
 40 | valid_cnt = int(train.shape[0] * 0.1)
 41 | test_idx,training_idx=indices[:valid_cnt],indices[valid_cnt:]
 42 | test, train = train[test_idx,:], train[training_idx,:]
 43 | onehot_test, onehot_train = onehot[test_idx,:], onehot[training_idx,:]
 44 | 
 45 | sess = tf.InteractiveSession()
 46 | 
 47 | 
 48 | # These will be inputs
 49 | ## Input pixels, flattened
 50 | x = tf.placeholder("float", [None, 1296])
 51 | ## Known labels
 52 | y_ = tf.placeholder("float", [None,5])
 53 | 
 54 | # Hidden layer 1
 55 | num_hidden1 = 128
 56 | W1 = tf.Variable(tf.truncated_normal([1296,num_hidden1],
 57 |                                stddev=1./math.sqrt(1296)))
 58 | b1 = tf.Variable(tf.constant(0.1,shape=[num_hidden1]))
 59 | h1 = tf.sigmoid(tf.matmul(x,W1) + b1)
 60 | 
 61 | # Hidden Layer 2
 62 | num_hidden2 = 32
 63 | W2 = tf.Variable(tf.truncated_normal([num_hidden1,
 64 |             num_hidden2],stddev=2./math.sqrt(num_hidden1)))
 65 | b2 = tf.Variable(tf.constant(0.2,shape=[num_hidden2]))
 66 | h2 = tf.sigmoid(tf.matmul(h1,W2) + b2)
 67 | 
 68 | # Output Layer
 69 | W3 = tf.Variable(tf.truncated_normal([num_hidden2, 5],
 70 |                                    stddev=1./math.sqrt(5)))
 71 | b3 = tf.Variable(tf.constant(0.1,shape=[5]))
 72 | 
 73 | # Just initialize
 74 | sess.run(tf.initialize_all_variables())
 75 | 
 76 | # Define model
 77 | y = tf.nn.softmax(tf.matmul(h2,W3) + b3)
 78 | 
 79 | ### End model specification, begin training code
 80 | 
 81 | 
 82 | # Climb on cross-entropy
 83 | cross_entropy = tf.reduce_mean(
 84 |      tf.nn.softmax_cross_entropy_with_logits(y + 1e-50, y_))
 85 | 
 86 | # How we train
 87 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
 88 | 
 89 | # Define accuracy
 90 | correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
 91 | accuracy=tf.reduce_mean(tf.cast(correct_prediction, "float"))
 92 | 
 93 | # Actually train
 94 | epochs = 25000
 95 | train_acc = np.zeros(epochs//10)
 96 | test_acc = np.zeros(epochs//10)
 97 | for i in tqdm(range(epochs), ascii=True):
 98 |     if i % 10 == 0:  # Record summary data, and the accuracy
 99 |         # Check accuracy on train set
100 |         A = accuracy.eval(feed_dict={x: train.reshape([-1,1296]), y_: onehot_train})
101 |         train_acc[i//10] = A
102 | 
103 |         # And now the validation set
104 |         A = accuracy.eval(feed_dict={x: test.reshape([-1,1296]), y_: onehot_test})
105 |         test_acc[i//10] = A
106 |     train_step.run(feed_dict={x: train.reshape([-1,1296]), y_: onehot_train})
107 | 
108 | # Plot the accuracy curves
109 | plt.plot(train_acc,'bo')
110 | plt.plot(test_acc,'rx')
111 | 
112 | # Look at the final testing confusion matrix
113 | pred = np.argmax(y.eval(feed_dict={x: test.reshape([-1,1296]), y_: onehot_test}), axis = 1)
114 | conf = np.zeros([5,5])
115 | for p,t in zip(pred,np.argmax(onehot_test,axis=1)):
116 |     conf[t,p] += 1
117 | 
118 | plt.matshow(conf)
119 | plt.colorbar()
120 | 
121 | # Let's look at a subplot of some weights
122 | f, plts = plt.subplots(4,8, sharex=True)
123 | for i in range(32):
124 |     plts[i//8, i%8].matshow(W1.eval()[:,i].reshape([36,36]))
125 | 
126 | # Examine the output weights
127 | plt.matshow(W3.eval())
128 | plt.colorbar()
129 | 
130 | # Save the weights
131 | saver = tf.train.Saver()
132 | saver.save(sess, "mpl.ckpt")
133 | 
134 | # Restore
135 | saver.restore(sess, "mlp.ckpt")
136 | 
137 | 
138 | 


--------------------------------------------------------------------------------
/Chapter02/The multiple hidden layer model/mlp.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | import math
  4 | %autoindent
  5 | 
  6 | try:
  7 |     from tqdm import tqdm
  8 | except ImportError:
  9 |     def tqdm(x, *args, **kwargs):
 10 |         return x
 11 | 
 12 | # Load data
 13 | data = np.load('data_with_labels.npz')
 14 | train = data['arr_0']/255.
 15 | labels = data['arr_1']
 16 | 
 17 | # Look at some data
 18 | print(train[0])
 19 | print(labels[0])
 20 | 
 21 | # If you have matplotlib installed
 22 | import matplotlib.pyplot as plt
 23 | plt.ion()
 24 | 
 25 | def to_onehot(labels,nclasses = 5):
 26 |     '''
 27 |     Convert labels to "one-hot" format.
 28 | 
 29 |     >>> a = [0,1,2,3]
 30 |     >>> to_onehot(a,5)
 31 |     array([[ 1.,  0.,  0.,  0.,  0.],
 32 |            [ 0.,  1.,  0.,  0.,  0.],
 33 |            [ 0.,  0.,  1.,  0.,  0.],
 34 |            [ 0.,  0.,  0.,  1.,  0.]])
 35 |     '''
 36 |     outlabels = np.zeros((len(labels),nclasses))
 37 |     for i,l in enumerate(labels):
 38 |         outlabels[i,l] = 1
 39 |     return outlabels
 40 | 
 41 | onehot = to_onehot(labels)
 42 | 
 43 | # Split data into training and validation
 44 | indices = np.random.permutation(train.shape[0])
 45 | valid_cnt = int(train.shape[0] * 0.1)
 46 | test_idx,training_idx=indices[:valid_cnt],indices[valid_cnt:]
 47 | test, train = train[test_idx,:], train[training_idx,:]
 48 | onehot_test, onehot_train = onehot[test_idx,:], onehot[training_idx,:]
 49 | 
 50 | sess = tf.InteractiveSession()
 51 | 
 52 | 
 53 | # These will be inputs
 54 | ## Input pixels, flattened
 55 | x = tf.placeholder("float", [None, 1296])
 56 | ## Known labels
 57 | y_ = tf.placeholder("float", [None,5])
 58 | 
 59 | # Hidden layer 1
 60 | num_hidden1 = 128
 61 | W1 = tf.Variable(tf.truncated_normal([1296,num_hidden1],
 62 |                                stddev=1./math.sqrt(1296)))
 63 | b1 = tf.Variable(tf.constant(0.1,shape=[num_hidden1]))
 64 | h1 = tf.sigmoid(tf.matmul(x,W1) + b1)
 65 | 
 66 | # Hidden Layer 2
 67 | num_hidden2 = 32
 68 | W2 = tf.Variable(tf.truncated_normal([num_hidden1,
 69 |             num_hidden2],stddev=2./math.sqrt(num_hidden1)))
 70 | b2 = tf.Variable(tf.constant(0.2,shape=[num_hidden2]))
 71 | h2 = tf.sigmoid(tf.matmul(h1,W2) + b2)
 72 | 
 73 | # Output Layer
 74 | W3 = tf.Variable(tf.truncated_normal([num_hidden2, 5],
 75 |                                    stddev=1./math.sqrt(5)))
 76 | b3 = tf.Variable(tf.constant(0.1,shape=[5]))
 77 | 
 78 | # Just initialize
 79 | sess.run(tf.initialize_all_variables())
 80 | 
 81 | # Define model
 82 | y = tf.nn.softmax(tf.matmul(h2,W3) + b3)
 83 | 
 84 | ### End model specification, begin training code
 85 | 
 86 | 
 87 | # Climb on cross-entropy
 88 | cross_entropy = tf.reduce_mean(
 89 |      tf.nn.softmax_cross_entropy_with_logits(logits = y + 1e-50, labels = y_))
 90 | 
 91 | # How we train
 92 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
 93 | 
 94 | # Define accuracy
 95 | correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
 96 | accuracy=tf.reduce_mean(tf.cast(correct_prediction, "float"))
 97 | 
 98 | # Actually train
 99 | epochs = 25000
100 | train_acc = np.zeros(epochs//10)
101 | test_acc = np.zeros(epochs//10)
102 | for i in tqdm(range(epochs), ascii=True):
103 |     if i % 10 == 0:  # Record summary data, and the accuracy
104 |         # Check accuracy on train set
105 |         A = accuracy.eval(feed_dict={x: train.reshape([-1,1296]), y_: onehot_train})
106 |         train_acc[i//10] = A
107 | 
108 |         # And now the validation set
109 |         A = accuracy.eval(feed_dict={x: test.reshape([-1,1296]), y_: onehot_test})
110 |         test_acc[i//10] = A
111 |     train_step.run(feed_dict={x: train.reshape([-1,1296]), y_: onehot_train})
112 | 
113 | # Plot the accuracy curves
114 | plt.figure(figsize=(6,6))
115 | plt.plot(train_acc,'bo')
116 | plt.plot(test_acc,'rx')
117 | 
118 | # Look at the final testing confusion matrix
119 | pred = np.argmax(y.eval(feed_dict={x: test.reshape([-1,1296]), y_: onehot_test}), axis = 1)
120 | conf = np.zeros([5,5])
121 | for p,t in zip(pred,np.argmax(onehot_test,axis=1)):
122 |     conf[t,p] += 1
123 | 
124 | plt.matshow(conf)
125 | plt.colorbar()
126 | 
127 | # Let's look at a subplot of some weights
128 | f, plts = plt.subplots(4,8, sharex=True)
129 | for i in range(32):
130 |     plts[i//8, i%8].matshow(W1.eval()[:,i].reshape([36,36]))
131 | 
132 | # Examine the output weights
133 | plt.matshow(W3.eval())
134 | plt.colorbar()
135 | 
136 | # Save the weights
137 | saver = tf.train.Saver()
138 | saver.save(sess, "mpl.ckpt")
139 | 
140 | # Restore
141 | saver.restore(sess, "mlp.ckpt")
142 | 
143 | 
144 | 


--------------------------------------------------------------------------------
/Chapter04/Exploring RNNs/rnn_tf.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | import datetime
  4 | import math
  5 | %autoindent
  6 | try:
  7 |     from tqdm import tqdm
  8 | except ImportError:
  9 |     def tqdm(x, *args, **kwargs):
 10 |         return x
 11 | 
 12 | # read in data
 13 | filename = 'weather.npz'
 14 | data = np.load(filename)
 15 | daily = data['daily']
 16 | weekly = data['weekly']
 17 | 
 18 | num_weeks = len(weekly)
 19 | dates = np.array([datetime.datetime.strptime(str(int(d)),
 20 |                 '%Y%m%d') for d in weekly[:,0]])
 21 | 
 22 | def assign_season(date):
 23 |     ''' Assign season based on meteorological season.
 24 |         Spring - from Mar 1 to May 31
 25 |         Summer - from Jun 1 to Aug 31
 26 |         Autumn - from Sep 1 to Nov 30
 27 |         Winter - from Dec 1 to Feb 28 (Feb 29 in a leap year)
 28 |     '''
 29 |     month = date.month
 30 |     # spring = 0
 31 |     if 3 <= month < 6:
 32 |         season = 0
 33 |     # summer = 1
 34 |     elif 6 <= month < 9:
 35 |         season = 1
 36 |     # autumn = 2
 37 |     elif 9 <= month < 12:
 38 |         season = 2
 39 |     # winter = 3
 40 |     elif month == 12 or month < 3:
 41 |         season = 3
 42 |     return season
 43 | 
 44 | 
 45 | # There are 4 seasons
 46 | num_classes = 4
 47 | 
 48 | # and 5 variables
 49 | num_inputs = 5
 50 | 
 51 | # And a state of 11 numbers
 52 | state_size = 11
 53 | 
 54 | labels = np.zeros([num_weeks,num_classes])
 55 | # read and convert to one-hot
 56 | for i,d in enumerate(dates):
 57 |     labels[i,assign_season(d)] = 1
 58 | 
 59 | # extract and scale training data
 60 | train = weekly[:,1:]
 61 | train = train - np.average(train,axis=0)
 62 | train = train / train.std(axis=0)
 63 | 
 64 | # Startup TensorFlow
 65 | sess = tf.InteractiveSession()
 66 | 
 67 | # These will be inputs
 68 | x = tf.placeholder("float", [None, num_inputs])
 69 | # TF likes a funky input to RNN
 70 | x_ = tf.reshape(x, [1, num_weeks, num_inputs])
 71 | 
 72 | ## Known labels
 73 | # None works during variable creation to be
 74 | # unspecified size
 75 | y_ = tf.placeholder("float", [None,num_classes])
 76 | 
 77 | cell = tf.nn.rnn_cell.BasicRNNCell(state_size)
 78 | outputs, states = tf.nn.dynamic_rnn(cell,x_,
 79 |             dtype=tf.nn.dtypes.float32, initial_state=None)
 80 | 
 81 | W1 = tf.Variable(tf.truncated_normal([state_size,num_classes],
 82 |                           stddev=1./math.sqrt(num_inputs)))
 83 | b1 = tf.Variable(tf.constant(0.1,shape=[num_classes]))
 84 | 
 85 | # reshape the output for traditional usage
 86 | h1 = tf.reshape(outputs,[-1,state_size])
 87 | 
 88 | # Just initialize
 89 | sess.run(tf.initialize_all_variables())
 90 | 
 91 | # Logistic regression as usual
 92 | y = tf.nn.softmax(tf.matmul(h1, W1) + b1)
 93 | 
 94 | # Climb on cross-entropy
 95 | cross_entropy = tf.reduce_mean(
 96 |      tf.nn.softmax_cross_entropy_with_logits(y + 1e-50, y_))
 97 | 
 98 | # How we train
 99 | train_step = tf.train.GradientDescentOptimizer(0.01
100 |                     ).minimize(cross_entropy)
101 | 
102 | # Define accuracy
103 | correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
104 | accuracy=tf.reduce_mean(tf.cast(correct_prediction, "float"))
105 | 
106 | # Actually train
107 | epochs = 100
108 | train_acc = np.zeros(epochs//10)
109 | for i in tqdm(range(epochs), ascii=True):
110 |     if i % 10 == 0:  # Record summary data, and the accuracy
111 |         # Check accuracy on train set
112 |         A = accuracy.eval(feed_dict={x: train, y_: labels})
113 |         train_acc[i//10] = A
114 |     train_step.run(feed_dict={x: train, y_: labels})
115 | 
116 | import matplotlib.pyplot as plt
117 | plt.ion()
118 | plt.figure(figsize=(6, 6))
119 | plt.plot(train_acc)
120 | 
121 | 
122 | 
123 | 
124 | 
125 | 
126 | 
127 | 
128 | 
129 | 
130 | 
131 | 
132 | 
133 | 
134 | ####
135 | 
136 | # can it be done with skflow?
137 | import tensorflow.contrib.skflow as skflow
138 | from sklearn import metrics
139 | 
140 | # To pass in data correctly
141 | def listify(x):
142 |     return [x]
143 | 
144 | # Undo the one_hot encoding
145 | classes = [assign_season(d) for d in dates]
146 | 
147 | # One line model
148 | classifier = skflow.TensorFlowRNNClassifier(rnn_size=11, 
149 |     n_classes=4, cell_type='rnn', input_op_fn = listify,
150 |     num_layers=8,
151 |     steps=1000, optimizer='Adam',
152 |     learning_rate=0.01, continue_training=True)
153 | 
154 | # Train model
155 | classifier.fit(train, classes )
156 | 
157 | # simple accuracy
158 | metrics.accuracy_score(classes,classifier.predict(train))
159 | 
160 | # confusion is easy in skflow
161 | conf = metrics.confusion_matrix(classes,
162 |             classifier.predict(train))
163 | print(conf)
164 | 
165 | 
166 | 
167 | 
168 | 
169 | 
170 | 
171 | 
172 | 
173 | 
174 | 
175 | 


--------------------------------------------------------------------------------
/Chapter03/Deep CNN/conv.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | import math
  4 | %autoindent
  5 | try:
  6 |     from tqdm import tqdm
  7 | except ImportError:
  8 |     def tqdm(x, *args, **kwargs):
  9 |         return x
 10 | 
 11 | # Set random seed
 12 | np.random.seed(0)
 13 | 
 14 | # Load data
 15 | data = np.load('data_with_labels.npz')
 16 | train = data['arr_0']/255.
 17 | labels = data['arr_1']
 18 | 
 19 | # Look at some data
 20 | print(train[0])
 21 | print(labels[0])
 22 | 
 23 | # If you have matplotlib installed
 24 | import matplotlib.pyplot as plt
 25 | plt.ion()
 26 | 
 27 | def to_onehot(labels,nclasses = 5):
 28 |     '''
 29 |     Convert labels to "one-hot" format.
 30 |     >>> a = [0,1,2,3]
 31 |     >>> to_onehot(a,5)
 32 |     array([[ 1.,  0.,  0.,  0.,  0.],
 33 |            [ 0.,  1.,  0.,  0.,  0.],
 34 |            [ 0.,  0.,  1.,  0.,  0.],
 35 |            [ 0.,  0.,  0.,  1.,  0.]])
 36 |     '''
 37 |     outlabels = np.zeros((len(labels),nclasses))
 38 |     for i,l in enumerate(labels):
 39 |         outlabels[i,l] = 1
 40 |     return outlabels
 41 | 
 42 | onehot = to_onehot(labels)
 43 | 
 44 | # Split data into training and validation
 45 | indices = np.random.permutation(train.shape[0])
 46 | valid_cnt = int(train.shape[0] * 0.1)
 47 | test_idx, training_idx = indices[:valid_cnt],\
 48 |                          indices[valid_cnt:]
 49 | test, train = train[test_idx,:],\
 50 |               train[training_idx,:]
 51 | onehot_test, onehot_train = onehot[test_idx,:],\
 52 |                         onehot[training_idx,:]
 53 | 
 54 | sess = tf.InteractiveSession()
 55 | 
 56 | 
 57 | # These will be inputs
 58 | ## Input pixels, image with one channel (gray)
 59 | x = tf.placeholder("float", [None, 36, 36])
 60 | # Note that -1 is for reshaping
 61 | x_im = tf.reshape(x, [-1,36,36,1])
 62 | ## Known labels
 63 | # None works during variable creation to be
 64 | # unspecified size
 65 | y_ = tf.placeholder("float", [None,5])
 66 | 
 67 | # Conv layer 1
 68 | num_filters = 4
 69 | winx = 5
 70 | winy = 5
 71 | W1 = tf.Variable(tf.truncated_normal(
 72 |     [winx, winy, 1 , num_filters],
 73 |     stddev=1./math.sqrt(winx*winy)))
 74 | b1 = tf.Variable(tf.constant(0.1,
 75 |                 shape=[num_filters]))
 76 | # 5x5 convolution, pad with zeros on edges
 77 | xw = tf.nn.conv2d(x_im, W1,
 78 |                   strides=[1, 1, 1, 1],
 79 |                   padding='SAME')
 80 | h1 = tf.nn.relu(xw + b1)
 81 | # 2x2 Max pooling, no padding on edges
 82 | p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
 83 |         strides=[1, 2, 2, 1], padding='VALID')
 84 | 
 85 | # Need to flatten convolutional output for use in dense layer
 86 | p1_size = np.product(
 87 |           [s.value for s in p1.get_shape()[1:]])
 88 | p1f = tf.reshape(p1, [-1, p1_size ])
 89 | 
 90 | # Dense layer
 91 | num_hidden = 32
 92 | W2 = tf.Variable(tf.truncated_normal(
 93 |      [p1_size, num_hidden],
 94 |      stddev=2./math.sqrt(p1_size)))
 95 | b2 = tf.Variable(tf.constant(0.2,
 96 |      shape=[num_hidden]))
 97 | h2 = tf.nn.relu(tf.matmul(p1f,W2) + b2)
 98 | 
 99 | # Output Layer
100 | W3 = tf.Variable(tf.truncated_normal(
101 |      [num_hidden, 5],
102 |      stddev=1./math.sqrt(num_hidden)))
103 | b3 = tf.Variable(tf.constant(0.1,shape=[5]))
104 | 
105 | keep_prob = tf.placeholder("float")
106 | h2_drop = tf.nn.dropout(h2, keep_prob)
107 | 
108 | # Just initialize
109 | sess.run(tf.global_variables_initializer())
110 | 
111 | # Define model
112 | y = tf.nn.softmax(tf.matmul(h2_drop,W3) + b3)
113 | 
114 | ### End model specification, begin training code
115 | 
116 | 
117 | # Climb on cross-entropy
118 | cross_entropy = tf.reduce_mean(
119 |         tf.nn.softmax_cross_entropy_with_logits(
120 |         logits = y + 1e-50, labels = y_))
121 | 
122 | # How we train
123 | train_step = tf.train.GradientDescentOptimizer(
124 |              0.01).minimize(cross_entropy)
125 | 
126 | # Define accuracy
127 | correct_prediction = tf.equal(tf.argmax(y,1),
128 |                               tf.argmax(y_,1))
129 | accuracy = tf.reduce_mean(tf.cast(
130 |            correct_prediction, "float"))
131 | 
132 | # Actually train
133 | epochs = 5000
134 | train_acc = np.zeros(epochs//10)
135 | test_acc = np.zeros(epochs//10)
136 | for i in tqdm(range(epochs), ascii=True):
137 |     # Record summary data, and the accuracy
138 |     if i % 10 == 0:  
139 |         # Check accuracy on train set
140 |         A = accuracy.eval(feed_dict={x: train,
141 |             y_: onehot_train, keep_prob: 1.0})
142 |         train_acc[i//10] = A
143 |         # And now the validation set
144 |         A = accuracy.eval(feed_dict={x: test,
145 |             y_: onehot_test, keep_prob: 1.0})
146 |         test_acc[i//10] = A
147 |     train_step.run(feed_dict={x: train,
148 |         y_: onehot_train, keep_prob: 0.5})
149 | 
150 | # Plot the accuracy curves
151 | plt.figure(figsize=(6, 6))
152 | plt.plot(train_acc,'bo')
153 | plt.plot(test_acc,'rx')
154 | 
155 | # Look at the final testing confusion matrix
156 | pred = np.argmax(y.eval(
157 |        feed_dict={x: test, keep_prob: 1.0,
158 |        y_: onehot_test}), axis = 1)
159 | conf = np.zeros([5,5])
160 | for p,t in zip(pred,np.argmax(onehot_test,
161 |                               axis=1)):
162 |     conf[t,p] += 1
163 | 
164 | plt.matshow(conf)
165 | plt.colorbar()
166 | 
167 | # Let's look at a subplot of some weights
168 | f, plts = plt.subplots(4)
169 | for i in range(4):
170 |     plts[i].matshow(
171 |             W1.eval()[:,:,0,i])
172 | 
173 | # Examine the output weights
174 | plt.matshow(W3.eval())
175 | plt.colorbar()
176 | 
177 | # Save the weights
178 | saver = tf.train.Saver()
179 | saver.save(sess, "conv1.ckpt")
180 | 
181 | # Restore
182 | saver.restore(sess, "conv1.ckpt")
183 | 
184 | # Or use Numpy manually
185 | def save_all(name = 'conv1'):
186 |     np.savez_compressed(name, W1.eval(),
187 |             b1.eval(), W2.eval(), b2.eval(),
188 |             W3.eval(), b3.eval())
189 | 
190 | save_all()
191 | 
192 | def load_all(name = 'conv1.npz'):
193 |     data = np.load(name)
194 |     sess.run(W1.assign(data['arr_0']))
195 |     sess.run(b1.assign(data['arr_1']))
196 |     sess.run(W2.assign(data['arr_2']))
197 |     sess.run(b2.assign(data['arr_3']))
198 |     sess.run(W3.assign(data['arr_4']))
199 |     sess.run(b3.assign(data['arr_5']))
200 | 
201 | load_all()
202 | 


--------------------------------------------------------------------------------
/Chapter03/Deeper CNN/conv_p2.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | import math
  4 | %autoindent
  5 | try:
  6 |     from tqdm import tqdm
  7 | except ImportError:
  8 |     def tqdm(x, *args, **kwargs):
  9 |         return x
 10 | 
 11 | # Set random seed
 12 | np.random.seed(0)
 13 | 
 14 | # Load data
 15 | data = np.load('data_with_labels.npz')
 16 | train = data['arr_0']/255.
 17 | labels = data['arr_1']
 18 | 
 19 | # Look at some data
 20 | print(train[0])
 21 | print(labels[0])
 22 | 
 23 | # If you have matplotlib installed
 24 | import matplotlib.pyplot as plt
 25 | plt.ion()
 26 | 
 27 | def to_onehot(labels,nclasses = 5):
 28 |     '''
 29 |     Convert labels to "one-hot" format.
 30 |     >>> a = [0,1,2,3]
 31 |     >>> to_onehot(a,5)
 32 |     array([[ 1.,  0.,  0.,  0.,  0.],
 33 |            [ 0.,  1.,  0.,  0.,  0.],
 34 |            [ 0.,  0.,  1.,  0.,  0.],
 35 |            [ 0.,  0.,  0.,  1.,  0.]])
 36 |     '''
 37 |     outlabels = np.zeros((len(labels),nclasses))
 38 |     for i,l in enumerate(labels):
 39 |         outlabels[i,l] = 1
 40 |     return outlabels
 41 | 
 42 | onehot = to_onehot(labels)
 43 | 
 44 | # Split data into training and validation
 45 | indices = np.random.permutation(train.shape[0])
 46 | valid_cnt = int(train.shape[0] * 0.1)
 47 | test_idx, training_idx = indices[:valid_cnt],\
 48 |                          indices[valid_cnt:]
 49 | test, train = train[test_idx,:],\
 50 |               train[training_idx,:]
 51 | onehot_test, onehot_train = onehot[test_idx,:],\
 52 |                         onehot[training_idx,:]
 53 | 
 54 | sess = tf.InteractiveSession()
 55 | 
 56 | 
 57 | # These will be inputs
 58 | ## Input pixels, image with one channel (gray)
 59 | x = tf.placeholder("float", [None, 36, 36])
 60 | # Note that -1 is for reshaping
 61 | x_im = tf.reshape(x, [-1,36,36,1])
 62 | ## Known labels
 63 | # None works during variable creation to be
 64 | # unspecified size
 65 | y_ = tf.placeholder("float", [None,5])
 66 | 
 67 | # Conv layer 1
 68 | num_filters1 = 16
 69 | winx1 = 3
 70 | winy1 = 3
 71 | W1 = tf.Variable(tf.truncated_normal(
 72 |     [winx1, winy1, 1 , num_filters1],
 73 |     stddev=1./math.sqrt(winx1*winy1)))
 74 | b1 = tf.Variable(tf.constant(0.1,
 75 |                 shape=[num_filters1]))
 76 | # 5x5 convolution, pad with zeros on edges
 77 | xw = tf.nn.conv2d(x_im, W1,
 78 |                   strides=[1, 1, 1, 1],
 79 |                   padding='SAME')
 80 | h1 = tf.nn.relu(xw + b1)
 81 | # 2x2 Max pooling, no padding on edges
 82 | p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
 83 |         strides=[1, 2, 2, 1], padding='VALID')
 84 | 
 85 | # Conv layer 2
 86 | num_filters2 = 4
 87 | winx2 = 3
 88 | winy2 = 3
 89 | W2 = tf.Variable(tf.truncated_normal(
 90 |     [winx2, winy2, num_filters1, num_filters2],
 91 |     stddev=1./math.sqrt(winx2*winy2)))
 92 | b2 = tf.Variable(tf.constant(0.1,
 93 |      shape=[num_filters2]))
 94 | # 3x3 convolution, pad with zeros on edges
 95 | p1w2 = tf.nn.conv2d(p1, W2,
 96 |        strides=[1, 1, 1, 1], padding='SAME')
 97 | h1 = tf.nn.relu(p1w2 + b2)
 98 | # 2x2 Max pooling, no padding on edges
 99 | p2 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
100 |      strides=[1, 2, 2, 1], padding='VALID')
101 | 
102 | # Need to flatten convolutional output
103 | p2_size = np.product(
104 |         [s.value for s in p2.get_shape()[1:]])
105 | p2f = tf.reshape(p2, [-1, p2_size ])
106 | 
107 | # Dense layer
108 | num_hidden = 32
109 | W3 = tf.Variable(tf.truncated_normal(
110 |      [p2_size, num_hidden],
111 |      stddev=2./math.sqrt(p2_size)))
112 | b3 = tf.Variable(tf.constant(0.2,
113 |      shape=[num_hidden]))
114 | h3 = tf.nn.relu(tf.matmul(p2f,W3) + b3)
115 | 
116 | # Drop out training
117 | keep_prob = tf.placeholder("float")
118 | h3_drop = tf.nn.dropout(h3, keep_prob)
119 | 
120 | # Output Layer
121 | W4 = tf.Variable(tf.truncated_normal(
122 |      [num_hidden, 5],
123 |      stddev=1./math.sqrt(num_hidden)))
124 | b4 = tf.Variable(tf.constant(0.1,shape=[5]))
125 | 
126 | # Just initialize
127 | sess.run(tf.initialize_all_variables())
128 | 
129 | # Define model
130 | y = tf.nn.softmax(tf.matmul(h3_drop,W4) + b4)
131 | 
132 | ### End model specification, begin training code
133 | 
134 | 
135 | # Climb on cross-entropy
136 | cross_entropy = tf.reduce_mean(
137 |         tf.nn.softmax_cross_entropy_with_logits(
138 |         logits = y + 1e-50, labels = y_))
139 | 
140 | # How we train
141 | train_step = tf.train.GradientDescentOptimizer(
142 |              0.01).minimize(cross_entropy)
143 | 
144 | # Define accuracy
145 | correct_prediction = tf.equal(tf.argmax(y,1),
146 |                               tf.argmax(y_,1))
147 | accuracy = tf.reduce_mean(tf.cast(
148 |            correct_prediction, "float"))
149 | 
150 | # Actually train
151 | epochs = 6000
152 | train_acc = np.zeros(epochs//10)
153 | test_acc = np.zeros(epochs//10)
154 | for i in tqdm(range(epochs), ascii=True):
155 |     # Record summary data, and the accuracy
156 |     if i % 10 == 0:  
157 |         # Check accuracy on train set
158 |         A = accuracy.eval(feed_dict={x: train,
159 |             y_: onehot_train, keep_prob: 1.0})
160 |         train_acc[i//10] = A
161 |         # And now the validation set
162 |         A = accuracy.eval(feed_dict={x: test,
163 |             y_: onehot_test, keep_prob: 1.0})
164 |         test_acc[i//10] = A
165 |     train_step.run(feed_dict={x: train,\
166 |         y_: onehot_train, keep_prob: 0.5})
167 | 
168 | # Plot the accuracy curves
169 | plt.figure(figsize=(6, 6))
170 | plt.plot(train_acc,'bo')
171 | plt.plot(test_acc,'rx')
172 | 
173 | # Look at the final testing confusion matrix
174 | pred = np.argmax(y.eval(
175 |        feed_dict={x: test, keep_prob: 1.0,
176 |        y_: onehot_test}), axis = 1)
177 | conf = np.zeros([5,5])
178 | for p,t in zip(pred,np.argmax(onehot_test,
179 |                               axis=1)):
180 |     conf[t,p] += 1
181 | 
182 | plt.matshow(conf)
183 | plt.colorbar()
184 | 
185 | # Let's look at a subplot of some weights
186 | f, plts = plt.subplots(4,4)
187 | for i in range(16):
188 |     plts[i//4,i%4].matshow(W1.eval()[:,:,0,i],
189 |             cmap = plt.cm.gray_r)
190 | 
191 | # Examine the output weights
192 | plt.matshow(W4.eval().T)
193 | plt.colorbar()
194 | 
195 | # Save the weights
196 | saver = tf.train.Saver()
197 | saver.save(sess, "conv2a.ckpt")
198 | 
199 | # Restore
200 | saver.restore(sess, "conv2a.ckpt")
201 | 
202 | # Or use Numpy manually
203 | def save_all(name = 'conv2'):
204 |     np.savez_compressed(name, W1.eval(),
205 |             b1.eval(), W2.eval(), b2.eval(),
206 |             W3.eval(), b3.eval(), W4.eval(),
207 |             b4.eval())
208 | 
209 | save_all()
210 | 
211 | def load_all(name = 'conv2.npz'):
212 |     data = np.load(name)
213 |     sess.run(W1.assign(data['arr_0']))
214 |     sess.run(b1.assign(data['arr_1']))
215 |     sess.run(W2.assign(data['arr_2']))
216 |     sess.run(b2.assign(data['arr_3']))
217 |     sess.run(W3.assign(data['arr_4']))
218 |     sess.run(b3.assign(data['arr_5']))
219 |     sess.run(W4.assign(data['arr_6']))
220 |     sess.run(b4.assign(data['arr_7']))
221 | 
222 | load_all()
223 | 


--------------------------------------------------------------------------------
/Chapter03/Wrapping up deep CNN/conv_p2.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import numpy as np
  3 | import math
  4 | %autoindent
  5 | try:
  6 |     from tqdm import tqdm
  7 | except ImportError:
  8 |     def tqdm(x, *args, **kwargs):
  9 |         return x
 10 | 
 11 | # Set random seed
 12 | np.random.seed(0)
 13 | 
 14 | # Load data
 15 | data = np.load('data_with_labels.npz')
 16 | train = data['arr_0']/255.
 17 | labels = data['arr_1']
 18 | 
 19 | # Look at some data
 20 | print(train[0])
 21 | print(labels[0])
 22 | 
 23 | # If you have matplotlib installed
 24 | import matplotlib.pyplot as plt
 25 | plt.ion()
 26 | 
 27 | def to_onehot(labels,nclasses = 5):
 28 |     '''
 29 |     Convert labels to "one-hot" format.
 30 |     >>> a = [0,1,2,3]
 31 |     >>> to_onehot(a,5)
 32 |     array([[ 1.,  0.,  0.,  0.,  0.],
 33 |            [ 0.,  1.,  0.,  0.,  0.],
 34 |            [ 0.,  0.,  1.,  0.,  0.],
 35 |            [ 0.,  0.,  0.,  1.,  0.]])
 36 |     '''
 37 |     outlabels = np.zeros((len(labels),nclasses))
 38 |     for i,l in enumerate(labels):
 39 |         outlabels[i,l] = 1
 40 |     return outlabels
 41 | 
 42 | onehot = to_onehot(labels)
 43 | 
 44 | # Split data into training and validation
 45 | indices = np.random.permutation(train.shape[0])
 46 | valid_cnt = int(train.shape[0] * 0.1)
 47 | test_idx, training_idx = indices[:valid_cnt],\
 48 |                          indices[valid_cnt:]
 49 | test, train = train[test_idx,:],\
 50 |               train[training_idx,:]
 51 | onehot_test, onehot_train = onehot[test_idx,:],\
 52 |                         onehot[training_idx,:]
 53 | 
 54 | sess = tf.InteractiveSession()
 55 | 
 56 | 
 57 | # These will be inputs
 58 | ## Input pixels, image with one channel (gray)
 59 | x = tf.placeholder("float", [None, 36, 36])
 60 | # Note that -1 is for reshaping
 61 | x_im = tf.reshape(x, [-1,36,36,1])
 62 | ## Known labels
 63 | # None works during variable creation to be
 64 | # unspecified size
 65 | y_ = tf.placeholder("float", [None,5])
 66 | 
 67 | # Conv layer 1
 68 | num_filters1 = 16
 69 | winx1 = 3
 70 | winy1 = 3
 71 | W1 = tf.Variable(tf.truncated_normal(
 72 |     [winx1, winy1, 1 , num_filters1],
 73 |     stddev=1./math.sqrt(winx1*winy1)))
 74 | b1 = tf.Variable(tf.constant(0.1,
 75 |                 shape=[num_filters1]))
 76 | # 5x5 convolution, pad with zeros on edges
 77 | xw = tf.nn.conv2d(x_im, W1,
 78 |                   strides=[1, 1, 1, 1],
 79 |                   padding='SAME')
 80 | h1 = tf.nn.relu(xw + b1)
 81 | # 2x2 Max pooling, no padding on edges
 82 | p1 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
 83 |         strides=[1, 2, 2, 1], padding='VALID')
 84 | 
 85 | # Conv layer 2
 86 | num_filters2 = 4
 87 | winx2 = 3
 88 | winy2 = 3
 89 | W2 = tf.Variable(tf.truncated_normal(
 90 |     [winx2, winy2, num_filters1, num_filters2],
 91 |     stddev=1./math.sqrt(winx2*winy2)))
 92 | b2 = tf.Variable(tf.constant(0.1,
 93 |      shape=[num_filters2]))
 94 | # 3x3 convolution, pad with zeros on edges
 95 | p1w2 = tf.nn.conv2d(p1, W2,
 96 |        strides=[1, 1, 1, 1], padding='SAME')
 97 | h1 = tf.nn.relu(p1w2 + b2)
 98 | # 2x2 Max pooling, no padding on edges
 99 | p2 = tf.nn.max_pool(h1, ksize=[1, 2, 2, 1],
100 |      strides=[1, 2, 2, 1], padding='VALID')
101 | 
102 | # Need to flatten convolutional output
103 | p2_size = np.product(
104 |         [s.value for s in p2.get_shape()[1:]])
105 | p2f = tf.reshape(p2, [-1, p2_size ])
106 | 
107 | # Dense layer
108 | num_hidden = 32
109 | W3 = tf.Variable(tf.truncated_normal(
110 |      [p2_size, num_hidden],
111 |      stddev=2./math.sqrt(p2_size)))
112 | b3 = tf.Variable(tf.constant(0.2,
113 |      shape=[num_hidden]))
114 | h3 = tf.nn.relu(tf.matmul(p2f,W3) + b3)
115 | 
116 | # Drop out training
117 | keep_prob = tf.placeholder("float")
118 | h3_drop = tf.nn.dropout(h3, keep_prob)
119 | 
120 | # Output Layer
121 | W4 = tf.Variable(tf.truncated_normal(
122 |      [num_hidden, 5],
123 |      stddev=1./math.sqrt(num_hidden)))
124 | b4 = tf.Variable(tf.constant(0.1,shape=[5]))
125 | 
126 | # Just initialize
127 | sess.run(tf.initialize_all_variables())
128 | 
129 | # Define model
130 | y = tf.nn.softmax(tf.matmul(h3_drop,W4) + b4)
131 | 
132 | ### End model specification, begin training code
133 | 
134 | 
135 | # Climb on cross-entropy
136 | cross_entropy = tf.reduce_mean(
137 |         tf.nn.softmax_cross_entropy_with_logits(
138 |         logits = y + 1e-50, labels = y_))
139 | 
140 | # How we train
141 | train_step = tf.train.GradientDescentOptimizer(
142 |              0.01).minimize(cross_entropy)
143 | 
144 | # Define accuracy
145 | correct_prediction = tf.equal(tf.argmax(y,1),
146 |                               tf.argmax(y_,1))
147 | accuracy = tf.reduce_mean(tf.cast(
148 |            correct_prediction, "float"))
149 | 
150 | # Actually train
151 | epochs = 6000
152 | train_acc = np.zeros(epochs//10)
153 | test_acc = np.zeros(epochs//10)
154 | for i in tqdm(range(epochs), ascii=True):
155 |     # Record summary data, and the accuracy
156 |     if i % 10 == 0:  
157 |         # Check accuracy on train set
158 |         A = accuracy.eval(feed_dict={x: train,
159 |             y_: onehot_train, keep_prob: 1.0})
160 |         train_acc[i//10] = A
161 |         # And now the validation set
162 |         A = accuracy.eval(feed_dict={x: test,
163 |             y_: onehot_test, keep_prob: 1.0})
164 |         test_acc[i//10] = A
165 |     train_step.run(feed_dict={x: train,\
166 |         y_: onehot_train, keep_prob: 0.5})
167 | 
168 | # Plot the accuracy curves
169 | plt.figure(figsize=(6, 6))
170 | plt.plot(train_acc,'bo')
171 | plt.plot(test_acc,'rx')
172 | 
173 | # Look at the final testing confusion matrix
174 | pred = np.argmax(y.eval(
175 |        feed_dict={x: test, keep_prob: 1.0,
176 |        y_: onehot_test}), axis = 1)
177 | conf = np.zeros([5,5])
178 | for p,t in zip(pred,np.argmax(onehot_test,
179 |                               axis=1)):
180 |     conf[t,p] += 1
181 | 
182 | plt.matshow(conf)
183 | plt.colorbar()
184 | 
185 | # Let's look at a subplot of some weights
186 | f, plts = plt.subplots(4,4)
187 | for i in range(16):
188 |     plts[i//4,i%4].matshow(W1.eval()[:,:,0,i],
189 |             cmap = plt.cm.gray_r)
190 | 
191 | # Examine the output weights
192 | plt.matshow(W4.eval().T)
193 | plt.colorbar()
194 | 
195 | # Save the weights
196 | saver = tf.train.Saver()
197 | saver.save(sess, "conv2a.ckpt")
198 | 
199 | # Restore
200 | saver.restore(sess, "conv2a.ckpt")
201 | 
202 | # Or use Numpy manually
203 | def save_all(name = 'conv2'):
204 |     np.savez_compressed(name, W1.eval(),
205 |             b1.eval(), W2.eval(), b2.eval(),
206 |             W3.eval(), b3.eval(), W4.eval(),
207 |             b4.eval())
208 | 
209 | save_all()
210 | 
211 | def load_all(name = 'conv2.npz'):
212 |     data = np.load(name)
213 |     sess.run(W1.assign(data['arr_0']))
214 |     sess.run(b1.assign(data['arr_1']))
215 |     sess.run(W2.assign(data['arr_2']))
216 |     sess.run(b2.assign(data['arr_3']))
217 |     sess.run(W3.assign(data['arr_4']))
218 |     sess.run(b3.assign(data['arr_5']))
219 |     sess.run(W4.assign(data['arr_6']))
220 |     sess.run(b4.assign(data['arr_7']))
221 | 
222 | load_all()
223 | 


--------------------------------------------------------------------------------