├── .gitignore
├── 1_Introduction
    ├── Introduction to Deep Learning D1.pdf
    └── README.md
├── 2_Perceptron
    ├── README.md
    ├── perceptronMNIST1.py
    ├── perceptronMNIST2.py
    ├── perceptronMNIST3.py
    ├── perceptronMNIST4.py
    ├── perceptronMNIST5.py
    └── perceptronMNISTv1.py
├── 3_ConvolutionalNetworks
    ├── Deep Learning Slides.pdf
    └── convolutionalMNIST.py
├── 4_FullyConvolutionalNetworks
    ├── README.md
    ├── fc.py
    └── fullyConvolutional.py
├── 5_Autoencoders
    ├── TensorFlowInterface.py
    ├── autoencoderMNIST.py
    ├── digitOriginal.jpg
    ├── digitOriginalRecon.jpg
    ├── digitSNR10.jpg
    ├── digitSNR10Recon.jpg
    └── input_data.py
├── 6_MRISegmentation
    ├── ccseg.py
    ├── corpusCallosum.tar.gz
    └── support
    │   ├── ccseg.py
    │   └── prepareData.py
├── LICENSE
├── README.md
└── tfs
    ├── setup.py
    └── tfs
        ├── TensorFlowInterface.py
        ├── __init__.py
        └── input_data.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.pyc
 2 | *.egg-info
 3 | logs
 4 | logs/*
 5 | data
 6 | data/*
 7 | *.nii
 8 | *.nii.gz
 9 | MNIST_data
10 | MNIST_data/*
11 | .DS_Store
12 | __pycache__
13 | __pycache__/*


--------------------------------------------------------------------------------
/1_Introduction/Introduction to Deep Learning D1.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/robb-brown/IntroToDeepLearning/ac6398427d0ac829e119122dafba1981bd68ff98/1_Introduction/Introduction to Deep Learning D1.pdf


--------------------------------------------------------------------------------
/1_Introduction/README.md:
--------------------------------------------------------------------------------
1 | #An Intuitive Introduction to Artificial Neural Networks and Deep Learning
2 | These notes are an introduction to artificial neural networks, deep learning, and graph-based computation.  Code that was formerly located here has been moved to Session 2.


--------------------------------------------------------------------------------
/2_Perceptron/README.md:
--------------------------------------------------------------------------------
 1 | # Setup Test
 2 | This session introduced TensorFlow for fully connected artificial neural networks.
 3 | 
 4 | The code for this session requires Python, Numpy, Matplotlib and TensorFlow.  Python can be obtained from <a href="python.org">python.org</a>.  We recommend pip (distributed with Python) for installation of the others.
 5 | 
 6 | Once you have downloaded the .py files in this directory, you can test your installation using:
 7 | 
 8 | ```python
 9 | python mnistExample.py
10 | ```
11 | 
12 | If you see the following output, your installation of TensorFlow is completed.
13 | ```
14 | Accuracy at step 0: train: 0.12
15 | Accuracy at step 100: train: 0.78
16 | Accuracy at step 200: train: 0.82
17 | Accuracy at step 300: train: 0.91
18 | Accuracy at step 400: train: 0.71
19 | Accuracy at step 500: train: 0.88
20 | Accuracy at step 600: train: 0.84
21 | Accuracy at step 700: train: 0.84
22 | Accuracy at step 800: train: 0.84
23 | Accuracy at step 900: train: 0.85
24 | Test accuracy: 0.8522
25 | ```
26 | 


--------------------------------------------------------------------------------
/2_Perceptron/perceptronMNIST1.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The MIT License (MIT)
  3 | 
  4 | Copyright (c) 2015 Robert A. Brown (www.robbtech.com)
  5 | 
  6 | Permission is hereby granted, free of charge, to any person obtaining a copy
  7 | of this software and associated documentation files (the "Software"), to deal
  8 | in the Software without restriction, including without limitation the rights
  9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10 | copies of the Software, and to permit persons to whom the Software is
 11 | furnished to do so, subject to the following conditions:
 12 | 
 13 | The above copyright notice and this permission notice shall be included in all
 14 | copies or substantial portions of the Software.
 15 | 
 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 22 | SOFTWARE.
 23 | """
 24 | 
 25 | """ 
 26 | This program demonstrates how to create a simple model in Tensorflow and perform
 27 | an iteration of training.
 28 | """
 29 | 
 30 | 
 31 | import tensorflow as tf
 32 | import math
 33 | from tfs import input_data
 34 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
 35 | 
 36 | # Grab an MNIST example. x is the input, y_ is the true label
 37 | x,y_ = mnist.train.next_batch(1)
 38 | 
 39 | 
 40 | # Set up the variables for our model. This is a basic logistic regression model, where
 41 | # W is a tensorflow variable that holds the weights and
 42 | # b holds the biases. Note their shapes.
 43 | W = tf.Variable(tf.zeros([784,10]),name='W_logistic')
 44 | b = tf.Variable(tf.zeros([10]),name='b_logistic')
 45 | 
 46 | # For the next two steps we need to tell Tensorflow to keep track of what we're doing
 47 | # so it can compute gradients later.
 48 | with tf.GradientTape() as tape:
 49 | 	
 50 | 	# y is the model's prediction, compare to the true labels, y_
 51 | 	y = tf.nn.softmax(tf.matmul(x,W) + b)
 52 | 	
 53 | 	# Calculate the cost, comparing y to y_
 54 | 	crossEntropy = -tf.reduce_sum(y_*tf.math.log(y))
 55 | 
 56 | 
 57 | # Compute accuracy, for our interest only
 58 | correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
 59 | accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
 60 | 
 61 | # notice that our cost is high and accuracy is low, because the model is not trained.
 62 | print(f'Cost: {crossEntropy};  Accuracy: {accuracy}')
 63 | 
 64 | # we can train the model by choosing an optimizer,
 65 | optimizer = tf.optimizers.Adam(learning_rate=1e-3)
 66 | 
 67 | # computing the gradient (that's what that tape thing was for)
 68 | # The gradient function computes the partial derivatives of 'target' with
 69 | # respect to 'sources'. The resulting vector is the gradient.
 70 | # Note that you only get to call this function once!
 71 | gradient = tape.gradient(target=crossEntropy,sources=[W,b])
 72 | 
 73 | # We can then give our optimizer the gradient and ask it to change our variables
 74 | # to move our model's prediction, y closer to the output we desire, y_
 75 | # The zip function rearranges our list of gradients and our list of variables into a 
 76 | # list of gradient-variable tuples.
 77 | optimizer.apply_gradients(zip(gradient, [W, b]))
 78 | 
 79 | # We can re-evaluate the cost and accuracy:
 80 | y = tf.nn.softmax(tf.matmul(x,W) + b)
 81 | newCrossEntropy = -tf.reduce_sum(y_*tf.math.log(y))
 82 | correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
 83 | newAccuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
 84 | 
 85 | # and notice that the cost has decreased a little, and accuracy has increased
 86 | print(f'Old Cost: {crossEntropy}; Old Accuracy: {accuracy}')
 87 | print(f'New Cost: {newCrossEntropy};  New Accuracy: {newAccuracy}')
 88 | 
 89 | # but if we take a different example
 90 | x,y_ = mnist.train.next_batch(1)
 91 | 
 92 | # and recalculate
 93 | y = tf.nn.softmax(tf.matmul(x,W) + b)
 94 | newExampleCrossEntropy = -tf.reduce_sum(y_*tf.math.log(y))
 95 | correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
 96 | newExampleAccuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
 97 | 
 98 | # The cost is high for the new example, and accuracy is low again.
 99 | print(f'Original Example Cost: {newCrossEntropy},  Accuracy: {newAccuracy}')
100 | print(f'New Example Cost: {newExampleCrossEntropy},  Accuracy: {newExampleAccuracy}')
101 | 
102 | 
103 | """
104 | We'd like the model to do well on all examples, so we need to train it on a
105 | representative sample of them.
106 | Repeatedly re-creating our model every time we want to run a training iteration
107 | or evaluate our result is inelegant and inefficient. Fortunately, we can clean
108 | this up using functions. See perceptronMNIST2.py.
109 | """
110 | 


--------------------------------------------------------------------------------
/2_Perceptron/perceptronMNIST2.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The MIT License (MIT)
  3 | 
  4 | Copyright (c) 2015 Robert A. Brown (www.robbtech.com)
  5 | 
  6 | Permission is hereby granted, free of charge, to any person obtaining a copy
  7 | of this software and associated documentation files (the "Software"), to deal
  8 | in the Software without restriction, including without limitation the rights
  9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10 | copies of the Software, and to permit persons to whom the Software is
 11 | furnished to do so, subject to the following conditions:
 12 | 
 13 | The above copyright notice and this permission notice shall be included in all
 14 | copies or substantial portions of the Software.
 15 | 
 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 22 | SOFTWARE.
 23 | """
 24 | 
 25 | """ 
 26 | This program is very much like perceptronMNIST1 but demonstrates how to clean up
 27 | the code and make it ready for training in a loop with a variety of samples by
 28 | putting the model creation and cost function calculations in functions.
 29 | """
 30 | 
 31 | import tensorflow as tf
 32 | import math
 33 | from tfs import input_data
 34 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
 35 | 
 36 | # Define a function that creates our model
 37 | def logisticModel(x,W,b):
 38 | 	y = tf.nn.softmax(tf.matmul(x,W) + b)
 39 | 	return y
 40 | 
 41 | # And a function that calculates the cost
 42 | def crossEntropyCost(y,y_):
 43 | 	crossEntropy = -tf.reduce_sum(y_*tf.math.log(y))
 44 | 	return crossEntropy
 45 | 
 46 | # finally a function that calculates accuracy
 47 | def accuracyMetric(y,y_):
 48 | 	correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
 49 | 	accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
 50 | 	return accuracy
 51 | 
 52 | 
 53 | # Now we can create a function that performs an optimization step
 54 | def optimizationStep(optimizer,x,y_,W,b):
 55 | 	with tf.GradientTape() as tape:
 56 | 		tape.watch(W); tape.watch(b)
 57 | 		y = logisticModel(x,W,b)
 58 | 		cost = crossEntropyCost(y,y_)
 59 | 	gradient = tape.gradient(target=cost,sources=[W,b])
 60 | 	optimizer.apply_gradients(zip(gradient, [W, b]))
 61 | 
 62 | 
 63 | # And a training function that runs the optimization step repeatedly with a 
 64 | # varity of examples
 65 | def train(mnist,x,y_,W,b,trainingIterations=3000):
 66 | 	optimizer = tf.optimizers.Adam(learning_rate=1e-3)
 67 | 	for iteration in range(trainingIterations):
 68 | 		# Grab 100 examples instead of just 1. This is called a "minibatch"
 69 | 		x,y_ = mnist.train.next_batch(100)
 70 | 		optimizationStep(optimizer=optimizer,x=x,y_=y_,W=W,b=b)
 71 | 		# For every 100th iteration, print out the training accuracy
 72 | 		if iteration % 100 == 0:
 73 | 			y = logisticModel(x,W,b)
 74 | 			accuracy = accuracyMetric(y,y_)
 75 | 			print(f'Training accuracy at step {iteration}: {accuracy}')
 76 | 	
 77 | 
 78 | # ---------------  Main Part --------------------
 79 | # Now we can use our functions instead of repeating code.
 80 | # First, we set up our model variables
 81 | W = tf.Variable(tf.zeros([784,10]),name='W_logistic')
 82 | b = tf.Variable(tf.zeros([10]),name='b_logistic')
 83 | 
 84 | # Check our starting cost and accuracy:
 85 | x,y_ = mnist.train.next_batch(1)
 86 | y = logisticModel(x,W,b)
 87 | crossEntropy = crossEntropyCost(y,y_)
 88 | accuracy = accuracyMetric(y,y_)
 89 | print(f'Starting Cost: {crossEntropy};  Accuracy: {accuracy}')
 90 | 
 91 | # Then we can train
 92 | train(mnist=mnist,x=x,y_=y_,W=W,b=b)
 93 | 
 94 | # And check our trained cost and accuracy on a batch of held out test examples
 95 | x,y_ = mnist.test.next_batch(1)
 96 | y = logisticModel(x,W,b)
 97 | crossEntropy = crossEntropyCost(y,y_)
 98 | accuracy = accuracyMetric(y,y_)
 99 | print(f'Trained Cost On Test Set: {crossEntropy};  Accuracy: {accuracy}')
100 | 
101 | 
102 | """
103 | This is better, but our functions are special purpose. If we want to change our
104 | model, especially if we use one with more parameters, we have to rewrite all our
105 | functions. To fix this, we have to create some software infrastructure. Tensorflow
106 | includes a module called Keras which does this, but it also hides a lot of details
107 | inside Tensorflow. We think it's important for learning that you see these details,
108 | so we're going to take the time to create our own minimal system that lets us see
109 | all the parts. See perceptronMNIST3.py.
110 | 
111 | """
112 | 
113 | 
114 | 


--------------------------------------------------------------------------------
/2_Perceptron/perceptronMNIST3.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The MIT License (MIT)
  3 | 
  4 | Copyright (c) 2015 Robert A. Brown (www.robbtech.com)
  5 | 
  6 | Permission is hereby granted, free of charge, to any person obtaining a copy
  7 | of this software and associated documentation files (the "Software"), to deal
  8 | in the Software without restriction, including without limitation the rights
  9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10 | copies of the Software, and to permit persons to whom the Software is
 11 | furnished to do so, subject to the following conditions:
 12 | 
 13 | The above copyright notice and this permission notice shall be included in all
 14 | copies or substantial portions of the Software.
 15 | 
 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 22 | SOFTWARE.
 23 | """
 24 | 
 25 | """ 
 26 | This program is very much like perceptronMNIST1 but demonstrates how to clean up
 27 | the code and make it ready for training in a loop with a variety of samples by
 28 | putting the model creation and cost function calculations in functions.
 29 | """
 30 | 
 31 | import tensorflow as tf
 32 | import math
 33 | from tfs import input_data
 34 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
 35 | 
 36 | 
 37 | # This is a base class that defines the basic interface for our objects.
 38 | # This one is very basic: an init function that stores the object's
 39 | # properties, a setup function that is called when the object is created,
 40 | # and an output function that can be called to get the object's output.
 41 | 
 42 | class Base(object):
 43 | 	
 44 | 	def __init__(self,name):
 45 | 		self.name = name
 46 | 		self.setup()
 47 | 	
 48 | 	def setup(self):
 49 | 		pass
 50 | 	
 51 | 	def output(self):
 52 | 		return None
 53 | 		
 54 | 
 55 | # We'll create a 'concrete class' implementing our logistic regression model.
 56 | # We have to tell it how big our input and output will be, then it creates
 57 | # and stores the W and b variables for us. The output function actually
 58 | # computes the logistic regression and returns the result.
 59 | # We add a variables function that returns the model's trainable variables because
 60 | # our training function will need those.
 61 | 
 62 | class LogisticModel(Base):
 63 | 	
 64 | 	def __init__(self,inputShape,outputShape,name):
 65 | 		self.inputShape = inputShape
 66 | 		self.outputShape = outputShape
 67 | 		self.name = name
 68 | 		self.setup()
 69 | 	
 70 | 	def setup(self):
 71 | 		self.W = tf.Variable(tf.zeros([self.inputShape,self.outputShape]),name=self.name+'W_logistic')
 72 | 		self.b = tf.Variable(tf.zeros([self.outputShape]),name=self.name+'b_logistic')
 73 | 	
 74 | 	def variables(self):
 75 | 		return [self.W,self.b]
 76 | 	
 77 | 	def output(self,x):
 78 | 		return tf.nn.softmax(tf.matmul(x,self.W) + self.b)
 79 | 	
 80 | 
 81 | # Similarly for our cost function. We don't need to create an init function
 82 | # because this one will be the same as in the base class.  This object "inherits"
 83 | # the init from it's ancestor.
 84 | class CrossEntropyCost(Base):
 85 | 
 86 | 	def output(self,y,y_):
 87 | 		return -tf.reduce_sum(y_*tf.math.log(y))
 88 | 
 89 | # And similarly for our accuracy metric.
 90 | class AccuracyMetric(Base):
 91 | 
 92 | 	def output(self,y,y_):
 93 | 		correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
 94 | 		accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
 95 | 		return accuracy
 96 | 
 97 | # Our optimizationStep and train function both go into a Trainer object
 98 | class Trainer(Base):
 99 | 	
100 | 	def __init__(self,model,cost,metrics,name):
101 | 		self.model = model
102 | 		self.cost = cost
103 | 		self.metrics = metrics
104 | 		self.name = name
105 | 		self.setup()
106 | 		
107 | 	def gradient(self,x,y_):
108 | 		variables = self.model.variables()
109 | 		with tf.GradientTape() as tape:
110 | 			y = self.model.output(x=x)
111 | 			cost = self.cost.output(y=y,y_=y_)
112 | 		gradient = tape.gradient(target=cost,sources=variables)
113 | 		return gradient
114 | 		
115 | 	def train(self,data,iterations=3000,learning_rate=1e-3,batchSize=100):
116 | 		optimizer = tf.optimizers.Adam(learning_rate=learning_rate)
117 | 		variables = self.model.variables()
118 | 		for iteration in range(iterations):
119 | 			x,y_ = mnist.train.next_batch(batchSize)
120 | 			optimizer.apply_gradients(zip(self.gradient(x,y_),variables))
121 | 			if iteration % 100 == 0:
122 | 				y = self.model.output(x=x)
123 | 				print(f'Training metrics at step {iteration}:')
124 | 				print(f'  {self.cost.name} = {self.cost.output(y=y,y_=y_)}')
125 | 				for metric in self.metrics:
126 | 					print(f'  {metric.name} = {metric.output(y=y,y_=y_)}')
127 | 		
128 | 
129 | # ---------------  Main Part --------------------
130 | # Now we no longer need to create our model by hand in the main part,
131 | # we just create our objects and pass them to the trainer.
132 | 
133 | model = LogisticModel(inputShape=784,outputShape=10,name='LogisticModel')
134 | cost = CrossEntropyCost(name='CrossEntropy')
135 | metrics = [AccuracyMetric(name='Accuracy')]
136 | trainer = Trainer(model=model,cost=cost,metrics=metrics,name='Trainer')
137 | 
138 | # Check our starting cost and accuracy:
139 | x,y_ = mnist.train.next_batch(1)
140 | y = model.output(x)
141 | print(f'Starting Cost: {cost.output(y,y_)};  Accuracy: {metrics[0].output(y,y_)}')
142 | 
143 | # Then we can train
144 | trainer.train(data=mnist)
145 | 
146 | # And check our trained cost and accuracy on a batch of held out test examples
147 | x,y_ = mnist.test.next_batch(100)
148 | y = model.output(x)
149 | print(f'Trained Cost On Test Set: {cost.output(y=y,y_=y_)};  Accuracy: {metrics[0].output(y=y,y_=y_)}')
150 | 


--------------------------------------------------------------------------------
/2_Perceptron/perceptronMNIST4.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The MIT License (MIT)
  3 | 
  4 | Copyright (c) 2015 Robert A. Brown (www.robbtech.com)
  5 | 
  6 | Permission is hereby granted, free of charge, to any person obtaining a copy
  7 | of this software and associated documentation files (the "Software"), to deal
  8 | in the Software without restriction, including without limitation the rights
  9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10 | copies of the Software, and to permit persons to whom the Software is
 11 | furnished to do so, subject to the following conditions:
 12 | 
 13 | The above copyright notice and this permission notice shall be included in all
 14 | copies or substantial portions of the Software.
 15 | 
 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 22 | SOFTWARE.
 23 | """
 24 | 
 25 | """ 
 26 | This program is very much like perceptronMNIST3 but demonstrates how to implement
 27 | different models by first creating the necessary layers objects and then passing those
 28 | to a model object.
 29 | """
 30 | 
 31 | import tensorflow as tf
 32 | import math
 33 | from tfs import input_data
 34 | import numpy
 35 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
 36 | 
 37 | 
 38 | # We'll keep our base class from step 3
 39 | class Base(object):
 40 | 	
 41 | 	def __init__(self,name):
 42 | 		self.name = name
 43 | 		self.setup()
 44 | 	
 45 | 	def setup(self):
 46 | 		pass
 47 | 	
 48 | 	def output(self):
 49 | 		return None
 50 | 
 51 | # The "deep" part of deep learning refers to the use of sequential computation,
 52 | # where the result of one calculation is used as input to another, etc., in a
 53 | # a chain. In mathematics, this is function composition: y = h(g(f(x))).
 54 | # It's convenient to treat each function as a "layer" of computation, then we
 55 | # can connect relatively simple layers together to produce a deep (multi-layered) 
 56 | # computational network.
 57 | 
 58 | # Our logistic regression model only had one layer of computation. Now we'll introduce
 59 | # a layer class, and a model class that will assemble multiple layers. Our 
 60 | # old logist regression model will now become a layer, and quite an important one:
 61 | # logistic regression is often the final layer in a deep learning classifier.
 62 | 
 63 | # This is a 'concrete class' which will be used for creating fairly generic
 64 | # fully connected deep learning layer objects. You'll see in the next section
 65 | # alternatives to fully connected layers, but for now our models will be
 66 | # entirely composed of them.
 67 | # A fully connected layer generally has weights (W) and biases (b), and an 
 68 | # activation function a, and computes the function:
 69 | # y = a(W @ x + b), where x is the input, y is the output and @ is matrix
 70 | # multiplication. Our logistic regression layer does exactly this, using 
 71 | # the softmax (logistic) function for a. The activation function is important in 
 72 | # deep learning because it can be nonlinear. Without it, a model, no matter how big,
 73 | # can only learn linear functions, like linear regression does. With a nonlinear
 74 | # activation function, a model that is sufficiently large and more than two layers
 75 | # deep can learn any function.
 76 | 
 77 | # We have to tell out layer how big our input and output will be, and what activation
 78 | # function to use. It then creates and stores whatever variables it needs. 
 79 | # The output function will assemble the computation for the layer and return it.
 80 | # We can also pass in an initialization function, but we'll allow it to default to
 81 | # None so that the layer can make a good choice if we don't specify a value. 
 82 | # As before, we have a "variables" function that returns the layer's 
 83 | # trainable variables, which the model object will collect and pass to 
 84 | # the trainer function.
 85 | # Since activation functions are very well standardized in Tensorflow, we can actually
 86 | # pass the Tensorflow function itself, e.g. tf.identity for a function that does 
 87 | # nothing (a linear layer, like linear regression), tf.softmax, which we've seen
 88 | # gives us logistic regression, and tf.relu, which is a rectified linear function, the
 89 | # workhorse of deep learning.
 90 | 
 91 | # For now we'll put all of these classes right in our program so they're easy to
 92 | # see, but since we're going to be using them so much, in the next step we'll 
 93 | # see how to make our own deep learning module out of them.
 94 | 
 95 | class Layer(Base):
 96 | 
 97 | 	def __init__(self,inputShape,outputShape,activation,name,initialization=None):
 98 | 		self.inputShape = inputShape
 99 | 		self.outputShape = outputShape
100 | 		self.activation = activation
101 | 		self.name = name
102 | 		if initialization is None:
103 | 			initialization = 'random'
104 | 		self.initialization = initialization
105 | 		
106 | 		self.setup()
107 | 
108 | 	def setup(self):
109 | 		if self.initialization == 'random':
110 | 			self.W = tf.Variable(tf.random.truncated_normal([self.inputShape,self.outputShape],stddev= 1.0 / math.sqrt(self.outputShape)),name=self.name+'W')
111 | 		elif self.initialization == 'zeros':
112 | 			self.W = tf.Variable(tf.zeros([self.inputShape,self.outputShape]),name=self.name+'W')
113 | 		self.b = tf.Variable(tf.zeros([self.outputShape]),name=self.name+'b')
114 | 
115 | 	def variables(self):
116 | 		return [self.W,self.b]
117 | 
118 | 	def output(self,x):
119 | 		return self.activation(tf.matmul(x,self.W) + self.b)
120 | 
121 | # This is our new model class.
122 | # We pass in a list of our layer objects and its job is to connect them for us.
123 | # It also needs to collect the variables from each layer when we ask for them.
124 | # The model doesn't actually need a setup function, but we'll call the ancestor's
125 | # function (which currently does nothing) anyway, because it's good practice.
126 | 
127 | class Model(Base):
128 | 
129 | 	def __init__(self,layers,name):
130 | 		self.layers = layers
131 | 		self.name = name
132 | 		self.setup()
133 | 	
134 | 	def variables(self):
135 | 		variables = []
136 | 		for layer in self.layers:
137 | 			variables = variables + layer.variables()
138 | 		return variables
139 | 
140 | 	def output(self, x):
141 | 		for layer in self.layers:
142 | 			x = layer.output(x)
143 | 		return x		
144 | 
145 | # The rest of our functions don't need to change, so we just copy them from
146 | # perceptronMNIST3. In the next step we'll make a module so we don't have to
147 | # do this copying every time!
148 | 
149 | # ------------------  Unchanged Stuff from before ----------------------
150 | class CrossEntropyCost(Base):
151 | 
152 | 	def output(self,y,y_):
153 | 		return -tf.reduce_sum(y_*tf.math.log(y))
154 | 
155 | 
156 | class AccuracyMetric(Base):
157 | 
158 | 	def output(self,y,y_):
159 | 		correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
160 | 		accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
161 | 		return accuracy
162 | 
163 | 
164 | class Trainer(Base):
165 | 	
166 | 	def __init__(self,model,cost,metrics,name):
167 | 		self.model = model
168 | 		self.cost = cost
169 | 		self.metrics = metrics
170 | 		self.name = name
171 | 		self.setup()
172 | 		
173 | 	def gradient(self,x,y_):
174 | 		variables = self.model.variables()
175 | 		with tf.GradientTape() as tape:
176 | 			y = self.model.output(x=x)
177 | 			cost = self.cost.output(y=y,y_=y_)
178 | 		gradient = tape.gradient(target=cost,sources=variables)
179 | 		return gradient
180 | 		
181 | 	def train(self,data,iterations=3000,learning_rate=1e-3,batchSize=100):
182 | 		optimizer = tf.optimizers.Adam(learning_rate=learning_rate)
183 | 		variables = self.model.variables()
184 | 		for iteration in range(iterations):
185 | 			x,y_ = data.train.next_batch(batchSize)
186 | 			optimizer.apply_gradients(zip(self.gradient(x,y_),variables))
187 | 			if iteration % 100 == 0:
188 | 				y = self.model.output(x=x)
189 | 				print(f'Training metrics at step {iteration}:')
190 | 				print(f'  {self.cost.name} = {self.cost.output(y=y,y_=y_)}')
191 | 				for metric in self.metrics:
192 | 					print(f'  {metric.name} = {metric.output(y=y,y_=y_)}')
193 | 
194 | # ---------------------  End of unchanged stuff -------------------------
195 | 		
196 | 
197 | # ---------------  Main Part --------------------
198 | # We can now implement different models by creating layer objects
199 | # and passing them to our model object. We then pass the necessary 
200 | # objects to the trainer.
201 | 
202 | # Here's our logistic regression model from before:
203 | logistic = Layer(	inputShape=784,outputShape=10,initialization='zeros',
204 | 					activation=tf.nn.softmax,name='logistic')
205 | # Notice that even though we only have one layer, we put it in a list.
206 | logisticModel = Model([logistic],name='LogisticModel')
207 | 
208 | 
209 | # But now we can just as easily make new models. This one has two layers, a linear one
210 | # then a logistic one. This is like a logistic regression model with interactions
211 | # between variables. We'll just call the layers 1 and 2, but you might see
212 | # terminology calling them "hidden" and "output". The "input" layer is really the
213 | # input to layer 1. The output of layer 1 is "hidden" because it's not data
214 | # we observe, or the end result of the computation, it's an intermediate result.
215 | # Of course, we can see it just fine, and we'll look at it later.
216 | 
217 | # Two-layer logistic Regression Model:
218 | layer1 = Layer(	inputShape=784,outputShape=200,initialization='random',
219 | 				activation=tf.identity,name='linear')
220 | 
221 | layer2 = Layer(	inputShape=200,outputShape=10,initialization='zeros',
222 | 				activation=tf.nn.softmax,name='logistic')
223 | 
224 | twoLayerRegressionModel = Model([layer1,layer2],name='TwoLayerRegressionModel')
225 | 
226 | 
227 | # This one is called a multilayer perceptron (MLP). It has two layers, just like the
228 | # previous one, but notice that the first layer has a proper activation function;
229 | # it's nonlinear! Our model is very small, but, if scaled up by increasing
230 | # the size of outputShape on the first layer and inputShape on the second, 
231 | # this model has everything it needs to learn *any* function.
232 | 
233 | # Two-layer Perceptron Model:
234 | layer1 = Layer(	inputShape=784,outputShape=200,initialization='random',
235 | 				activation=tf.nn.relu,name='input')
236 | 
237 | layer2 = Layer(	inputShape=200,outputShape=10,initialization='zeros',
238 | 				activation=tf.nn.softmax,name='output')
239 | 
240 | twoLayerPerceptronModel = Model([layer1,layer2],name='TwoLayerPerceptron')
241 | 
242 | 
243 | # Finally, this is pretty much the same thing, but it's got an extra layer in the
244 | # middle, for three total! We're officially doing deep learning now! Most machine
245 | # learning models, like decision trees, SVM, regression, 
246 | # older neural networks, etc. are one or two layers of computation deep.
247 | 
248 | # Three-Layer Deep Network Model:
249 | layer1 = Layer(	inputShape=784,outputShape=100,initialization='random',
250 | 				activation=tf.nn.relu,name='input')
251 | 
252 | layer2 = Layer(	inputShape=100,outputShape=100,initialization='random',
253 | 				activation=tf.nn.relu,name='hidden')
254 | 
255 | layer3 = Layer(	inputShape=100,outputShape=10,initialization='zeros',
256 | 				activation=tf.nn.softmax,name='output')
257 | 
258 | threeLayerPerceptronModel = Model([layer1,layer2,layer3],name='ThreeLayerDeepModel')
259 | 
260 | 
261 | # Now we can set up our cost function and trainer as we did before.
262 | cost = CrossEntropyCost(name='CrossEntropy')
263 | metrics = [AccuracyMetric(name='Accuracy')]
264 | trainer = Trainer(model=logisticModel,cost=cost,metrics=metrics,name='Trainer')
265 | 
266 | 
267 | # We can train our model just as we did before 
268 | # (remember, we didn't change the training function at all)
269 | # but let's do a little experiment. Since we now have multiple models,
270 | # let's train them all and see which one works best!
271 | 
272 | # We'll create a dictionary to hold our results, then loop over 
273 | # our models to train each one in succession.
274 | 
275 | results = dict()
276 | for model in [logisticModel,twoLayerRegressionModel,twoLayerPerceptronModel,threeLayerPerceptronModel]:
277 | 	print(f'Training model {model.name}')
278 | 
279 | 	# We'll create another dictionary to store the result just for this model
280 | 	result = dict()
281 | 	
282 | 	# we need to (re)create our trainer inside the loop with the current model
283 | 	# we don't need to recreate the cost or metrics, we can reuse those.
284 | 	trainer = Trainer(model=model,cost=cost,metrics=metrics,name='Trainer')
285 | 	
286 | 	# Check our starting cost and accuracy:
287 | 	x,y_ = mnist.train.next_batch(100)
288 | 	y = model.output(x)
289 | 	result['startingCost'] = cost.output(y,y_)
290 | 	result['startingAccuracy'] = metrics[0].output(y,y_)
291 | 	print(f'Starting Cost: {result["startingCost"]};  Accuracy: {result["startingAccuracy"]}')	
292 | 	
293 | 	# Then we can train
294 | 	trainer.train(data=mnist)
295 | 
296 | 	# And check our trained cost and accuracy on a batch of held out test examples
297 | 	x,y_ = mnist.test.next_batch(100)
298 | 	y = model.output(x)
299 | 	result['testCost'] = cost.output(y,y_)
300 | 	result['testAccuracy'] = metrics[0].output(y,y_)
301 | 	print(f'Trained Cost On Test Set: {result["testCost"]};  Accuracy: {result["testAccuracy"]}')
302 | 
303 | 	# finally, we'll put our result dictionary into our results dictionary so
304 | 	# we can analyze it later.
305 | 	results[model.name] = result
306 | 
307 | 
308 | # Now that we have all our results, let's print them out so they're easier to see
309 | # all together. If you look closely, I've added :0.4 after each variable we're
310 | # printing. This tells python to use four digits of precision. It makes the printout
311 | # a little easier to read.
312 | 
313 | for modelName in results.keys():
314 | 	result = results[modelName]
315 | 	print()
316 | 	print(modelName)
317 | 	print(f'Starting Cost: {result["startingCost"]:0.4};  Accuracy: {result["startingAccuracy"]:0.4}')	
318 | 	print(f'Trained Cost On Test Set: {result["testCost"]:0.4};  Accuracy: {result["testAccuracy"]:0.4}')
319 | 
320 | 
321 | # Congratulations, you have just completed a deep learning experiment. So, which
322 | # model performed the best?  Here's what I got:
323 | 
324 | # LogisticModel
325 | # Starting Cost: 230.3;  Accuracy: 0.12
326 | # Trained Cost On Test Set: 20.74;  Accuracy: 0.97
327 | #
328 | # TwoLayerRegressionModel
329 | # Starting Cost: 230.3;  Accuracy: 0.1
330 | # Trained Cost On Test Set: 15.77;  Accuracy: 0.96
331 | #
332 | # TwoLayerPerceptron
333 | # Starting Cost: 230.3;  Accuracy: 0.13
334 | # Trained Cost On Test Set: 9.695;  Accuracy: 0.98
335 | #
336 | # ThreeLayerDeepModel
337 | # Starting Cost: 230.3;  Accuracy: 0.11
338 | # Trained Cost On Test Set: 6.637;  Accuracy: 0.97
339 | 
340 | # Notice that all the models did pretty well on accuracy, but remember, we only
341 | # tested with a batch of 100 examples. The cost value takes into account
342 | # how confident each of the predictions was, and shows that the two layer models
343 | # were better than one layer, and the three layer model was better than two, and
344 | # the nonlinearity in the perceptrons also helped. If you were paying attention you
345 | # might have noticed that, although the one-layer logistic model's size is fixed
346 | # by the input and output, all the two layer models had 200 hidden artificial neurons
347 | # and the three layer model also had 200 hidden neurons, 100 in each hidden layer. 
348 | # This actually means the three layer model had fewer parameters, because W is a
349 | # function of inputSize * outputSize. But the three layer model did the best!
350 | # There's a proof that shows that a deeper model can use it's capacity more
351 | # efficiently (sometimes exponentially more efficiently) than a shallower model.
352 | # That's why deep learning works so well.
353 | 
354 | # Three layers doesn't seem like much, but without ReLU and functions like it, three or
355 | # more layer models are very hard to train. ReLU is really the magic that enables deep
356 | # learning, and the efficiency of deep models is the reason deep learning works so well.
357 | 
358 | # **An important note!**
359 | # You might have noticed that the starting costs can be a bit different, and the
360 | # starting accuracies can be quite different. If you run this program again, you might
361 | # get somewhat different results, and you probably got different results than I did.
362 | # There are a lot of random numbers in machine learning. We used random numbers to
363 | # intialize many of our parameters, and also chose random examples, in a random
364 | # order, to train and test our models. To be good scientists, we should run our program
365 | # many times, keep track of the results for each run, and compute statistics that
366 | # tell us which models were significantly better or worse. Never believe a number
367 | # or that doesn't have some kind of confidence interval! You may also notice that
368 | # lots of machine learning websites, competitions and even scientific papers
369 | # don't do this. This is really an area where the machine learning field needs
370 | # to improve. If you'd like to help, try modifying this program to train the
371 | # models multiple times, collect the results, and do a statistical comparison. Don't
372 | # forget to recreate your models each time so they're untrained! Do our initial results
373 | # still hold up?
374 | 
375 | 
376 | 
377 | 
378 | 


--------------------------------------------------------------------------------
/2_Perceptron/perceptronMNIST5.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | import math
 3 | from tfs import input_data
 4 | import numpy
 5 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
 6 | 
 7 | # We've been importing input_data from the tfs module in order to provide
 8 | # the mnist data. This module also contains the classes we've been writing
 9 | # to support our model building, so instead of including this support code
10 | # in every program, we can just import it:
11 | 
12 | from tfs import Layer, Model, CrossEntropyCost, AccuracyMetric, Trainer
13 | 
14 | 
15 | # Throughout this tutorial we'll introduce new tools in the main program
16 | # on first use so you can always see all the details, then subsequently 
17 | # import them from tfs.
18 | 
19 | 
20 | # ---------------  Main Part --------------------
21 | 
22 | # Logistic regression model
23 | logistic = Layer(	inputShape=784,outputShape=10,initialization='zeros',
24 | 					activation=tf.nn.softmax,name='logistic')
25 | logisticModel = Model([logistic],name='LogisticModel')
26 | 
27 | 
28 | # Two-layer logistic Regression Model:
29 | layer1 = Layer(	inputShape=784,outputShape=200,initialization='random',
30 | 				activation=tf.identity,name='linear')
31 | layer2 = Layer(	inputShape=200,outputShape=10,initialization='zeros',
32 | 				activation=tf.nn.softmax,name='logistic')
33 | twoLayerRegressionModel = Model([layer1,layer2],name='TwoLayerRegressionModel')
34 | 
35 | 
36 | # Two-layer Perceptron Model:
37 | layer1 = Layer(	inputShape=784,outputShape=200,initialization='random',
38 | 				activation=tf.nn.relu,name='input')
39 | layer2 = Layer(	inputShape=200,outputShape=10,initialization='zeros',
40 | 				activation=tf.nn.softmax,name='output')
41 | twoLayerPerceptronModel = Model([layer1,layer2],name='TwoLayerPerceptron')
42 | 
43 | 
44 | # Three-Layer Deep Network Model:
45 | layer1 = Layer(	inputShape=784,outputShape=100,initialization='random',
46 | 				activation=tf.nn.relu,name='input')
47 | layer2 = Layer(	inputShape=100,outputShape=100,initialization='random',
48 | 				activation=tf.nn.relu,name='hidden')
49 | layer3 = Layer(	inputShape=100,outputShape=10,initialization='zeros',
50 | 				activation=tf.nn.softmax,name='output')
51 | threeLayerPerceptronModel = Model([layer1,layer2,layer3],name='ThreeLayerDeepModel')
52 | 
53 | 
54 | # Cost and trainer
55 | cost = CrossEntropyCost(name='CrossEntropy')
56 | metrics = [AccuracyMetric(name='Accuracy')]
57 | trainer = Trainer(model=logisticModel,cost=cost,metrics=metrics,name='Trainer')
58 | 
59 | 
60 | 
61 | # Train and evaluate
62 | results = dict()
63 | for model in [logisticModel,twoLayerRegressionModel,twoLayerPerceptronModel,threeLayerPerceptronModel]:
64 | 	print(f'Training model {model.name}')
65 | 	result = dict()
66 | 	trainer = Trainer(model=model,cost=cost,metrics=metrics,name='Trainer')
67 | 	x,y_ = mnist.train.next_batch(100)
68 | 	y = model.output(x)
69 | 	result['startingCost'] = cost.output(y,y_)
70 | 	result['startingAccuracy'] = metrics[0].output(y,y_)
71 | 	print(f'Starting Cost: {result["startingCost"]};  Accuracy: {result["startingAccuracy"]}')	
72 | 	trainer.train(data=mnist)
73 | 	x,y_ = mnist.test.next_batch(100)
74 | 	y = model.output(x)
75 | 	result['testCost'] = cost.output(y,y_)
76 | 	result['testAccuracy'] = metrics[0].output(y,y_)
77 | 	print(f'Trained Cost On Test Set: {result["testCost"]};  Accuracy: {result["testAccuracy"]}')
78 | 	results[model.name] = result
79 | 
80 | 
81 | for modelName in results.keys():
82 | 	result = results[modelName]
83 | 	print()
84 | 	print(modelName)
85 | 	print(f'Starting Cost: {result["startingCost"]:0.4};  Accuracy: {result["startingAccuracy"]:0.4}')	
86 | 	print(f'Trained Cost On Test Set: {result["testCost"]:0.4};  Accuracy: {result["testAccuracy"]:0.4}')
87 | 


--------------------------------------------------------------------------------
/2_Perceptron/perceptronMNISTv1.py:
--------------------------------------------------------------------------------
 1 | """
 2 | The MIT License (MIT)
 3 | 
 4 | Copyright (c) 2015 Robert A. Brown (www.robbtech.com)
 5 | 
 6 | Permission is hereby granted, free of charge, to any person obtaining a copy
 7 | of this software and associated documentation files (the "Software"), to deal
 8 | in the Software without restriction, including without limitation the rights
 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 | copies of the Software, and to permit persons to whom the Software is
11 | furnished to do so, subject to the following conditions:
12 | 
13 | The above copyright notice and this permission notice shall be included in all
14 | copies or substantial portions of the Software.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 | SOFTWARE.
23 | """
24 | 
25 | import tensorflow as tf
26 | import math
27 | from tfs import input_data
28 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
29 | 
30 | # Common setup
31 | sess = tf.InteractiveSession()
32 | x = tf.placeholder('float',shape=[None,784],name='input')		# Input tensor
33 | y_ = tf.placeholder('float', shape=[None,10],name='correctLabels') 		# Correct labels
34 | 
35 | 
36 | # MODEL DEFINITION
37 | 
38 | # Softmax regression
39 | W = tf.Variable(tf.zeros([784,10]),name='W_logistic')
40 | b = tf.Variable(tf.zeros([10]),name='b_logistic')
41 | y = tf.nn.softmax(tf.matmul(x,W) + b)
42 | 
43 | # # Two-layer softmax regression
44 | # Wlin = tf.Variable(tf.truncated_normal([784,784],stddev= 1.0 / math.sqrt(784)),name='W_linear')
45 | # blin = tf.Variable(tf.zeros([784]),name='b_linear')
46 | # W = tf.Variable(tf.zeros([784,10]),name='W_logistic')
47 | # b = tf.Variable(tf.zeros([10]),name='b_logistic')
48 | # y_intermediate = tf.matmul(x,Wlin) + blin
49 | # y = tf.nn.softmax(tf.matmul(y_intermediate,W) + b)
50 | 
51 | # # Two-layer perceptron
52 | # hiddenUnitN = 200
53 | # W1 = tf.Variable(tf.truncated_normal([784,hiddenUnitN],stddev= 1.0 / math.sqrt(hiddenUnitN)),name='W1')
54 | # b1 = tf.Variable(tf.zeros([hiddenUnitN]),name='b1')
55 | # y_intermediate = tf.nn.relu(tf.matmul(x,W1) + b1)
56 | # 
57 | # W = tf.Variable(tf.zeros([hiddenUnitN,10]),name='W_logistic')
58 | # b = tf.Variable(tf.zeros([10]),name='b_logistic')
59 | # y = tf.nn.softmax(tf.matmul(y_intermediate,W) + b)
60 | 
61 | # # Three-layer Deep Network
62 | # hiddenUnitN1 = 200; hiddenUnitN2 = 200
63 | # W1 = tf.Variable(tf.truncated_normal([784,hiddenUnitN1],stddev= 1.0 / math.sqrt(hiddenUnitN1)),name='W1')
64 | # b1 = tf.Variable(tf.zeros([hiddenUnitN1]),name='b1')
65 | # hiddenLayer1 = tf.nn.relu(tf.matmul(x,W1) + b1)
66 | # 
67 | # W2 = tf.Variable(tf.truncated_normal([hiddenUnitN1,hiddenUnitN2],stddev= 1.0 / math.sqrt(hiddenUnitN2)),name='W2')
68 | # b2 = tf.Variable(tf.zeros([hiddenUnitN2]),name='b2')
69 | # hiddenLayer2 = tf.nn.relu(tf.matmul(hiddenLayer1,W2) + b2)
70 | # 
71 | # WSoftmax = tf.Variable(tf.zeros([hiddenUnitN2,10]),name='W_softmax')
72 | # bSoftmax = tf.Variable(tf.zeros([10]),name='b_softmax')
73 | # y = tf.nn.softmax(tf.matmul(hiddenLayer2,WSoftmax) + bSoftmax)
74 | 
75 | 
76 | # END MODEL DEFINITION
77 | 
78 | cross_entropy = -tf.reduce_sum(y_*tf.log(y))
79 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
80 | train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
81 | 
82 | correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
83 | accuracy = tf.reduce_mean(tf.cast(correct_prediction,'float'))
84 | 
85 | tf.initialize_all_variables().run()		# Take initial values and actually put them in variables
86 | 
87 | for i in range(3000):						# Do some training
88 | 	batch = mnist.train.next_batch(100)
89 | 	if i%100 == 0:
90 | 		train_accuracy = accuracy.eval(feed_dict={x:batch[0],y_:batch[1]})		
91 | 		print('Accuracy at step {}: train: {}'.format(i,train_accuracy))
92 | 	
93 | 	train_step.run(feed_dict={x:batch[0],y_:batch[1]})
94 | 
95 | print('Test accuracy: {}'.format(accuracy.eval(feed_dict={x:mnist.test.images, y_:mnist.test.labels})))
96 | 
97 | 


--------------------------------------------------------------------------------
/3_ConvolutionalNetworks/Deep Learning Slides.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/robb-brown/IntroToDeepLearning/ac6398427d0ac829e119122dafba1981bd68ff98/3_ConvolutionalNetworks/Deep Learning Slides.pdf


--------------------------------------------------------------------------------
/3_ConvolutionalNetworks/convolutionalMNIST.py:
--------------------------------------------------------------------------------
 1 | from tfs import *
 2 | from tfs import input_data
 3 | from pylab import *
 4 | from numpy import *
 5 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
 6 | matplotlib.interactive(True)
 7 | session = tf.InteractiveSession()
 8 | 
 9 | x = tf.placeholder('float',shape=[None,784],name='input')		# Input tensor
10 | y_ = tf.placeholder('float', shape=[None,10],name='correctLabels') 		# Correct labels
11 | 
12 | # A four layer fully connected net for comparison
13 | # trainingIterations = 10000
14 | # L1 = ReLu(x,512,'relu1')
15 | # L2 = ReLu(L1.output,128,'relu2')
16 | # L3 = ReLu(L2.output,64,'relu3')
17 | # L4 = SoftMax(x,10,'softmax')
18 | # y = L4.output
19 | # trainDict = {}; testDict = trainDict
20 | ## logName = 'logs/4LayerFullyConnected'
21 | 
22 | 
23 | xImage = tf.reshape(x,[-1,28,28,1])		# Reshape samples to 28x28x1 images
24 | trainingIterations = 1000
25 | 
26 | # Standard conv net
27 | L1 = Conv2D(xImage,[5,5,1,32],'Conv1')
28 | L2 = MaxPool2x2(L1.output,'MaxPool1')
29 | L0do = Dropout(L2.output,'dropout')
30 | L3 = Conv2D(L0do.output,[5,5,32,64],'Conv2')
31 | L4 = MaxPool2x2(L3.output,'MaxPool2')
32 | L5 = ReLu(L4.output,128,'relu1')
33 | L6 = SoftMax(L5.output,10,'softmax')
34 | y = L6.output
35 | kp = 0.5; trainDict = {L0do.keepProb:kp}
36 | kp = 1.0; testDict = {L0do.keepProb:kp}
37 | logName = None #logName = 'logs/Conv'
38 | 
39 | 
40 | 
41 | # Training and evaluation
42 | crossEntropy = -tf.reduce_sum(y_*tf.log(y))		# cost function
43 | trainStep = tf.train.GradientDescentOptimizer(0.01).minimize(crossEntropy)
44 | trainStep = tf.train.AdamOptimizer(1e-4).minimize(crossEntropy)
45 | correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
46 | accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
47 | train(session=session,trainingData=mnist.train,testingData=mnist.test,truth=y_,input=x,cost=crossEntropy,trainingStep=trainStep,accuracy=accuracy,iterations=trainingIterations,miniBatch=100,trainDict=trainDict,testDict=testDict,logName=logName)
48 | 
49 | # Mini viewer
50 | plotFields(L1,cmap=cm.coolwarm)
51 | example = mnist.test.next_batch(1); image = reshape(example[0][0],(28,28))
52 | feed_dict = {x:example[0],y_:example[1],L0do.keepProb:1.0}
53 | figure(1); clf(); subplot(231); imshow(image,cmap=cm.gray); title('Correct is %d' % where(example[1]>0)[1][0])
54 | subplot(232); plotOutput(L1,feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
55 | subplot(233); plotOutput(L2,feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
56 | subplot(235); plotOutput(L3,feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
57 | subplot(236); plotOutput(L4,feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
58 | subplot(234); plotOutput(L6,fieldShape=[10,1],feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
59 | 
60 | # Individual interesting fields
61 | # plotOutput(L1,feed_dict={x:example[0],y_:example[1]},cmap=cm.inferno); figure(2); clf(); imshow(image,cmap=cm.gray)
62 | # plotOutput(L3,fieldShape=[11,12],feed_dict=feed_dict,cmap=cm.inferno); figure(2); clf(); imshow(image,cmap=cm.gray)
63 | # plotOutput(L6,fieldShape=[10,1],feed_dict=feed_dict,cmap=cm.inferno); figure(2); clf(); imshow(image,cmap=cm.gray)
64 | 
65 | 
66 | 


--------------------------------------------------------------------------------
/4_FullyConvolutionalNetworks/README.md:
--------------------------------------------------------------------------------
 1 | # Session 4 - Machine Learning With Uncertain Bounds
 2 | This workshop introduces fully convolutional neural networks and illustrates the importance of using machine learning itself to do as much of the work as possible.  As an example problem, we will be trying to locate and identify handwritten characters randomly placed on an arbitrary sized page.
 3 | 
 4 | The code for this session requires Python, Numpy, Matplotlib and TensorFlow.  Python can be obtained from <a href="python.org">python.org</a>.  We recommend pip (distributed with Python) for installation of the others.
 5 | 
 6 | Once you have Python installed, you can install the required modules with the following commands:
 7 | 
 8 | ```pip install Numpy```
 9 | 
10 | ```pip install Matplotlib```
11 | 
12 | ```pip install TensorFlow```
13 | 
14 | If you installed TensorFlow prior to July 1, 2016, use the command:
15 | 
16 | ```pip install --upgrade TensorFlow```
17 | 
18 | to grab the latest version. 
19 | 
20 | Also make sure you download the .py files in this directory.  If you've got a copy of TensorFlowInterface.py from a previous session, make sure you grab this one as it has some new bits.
21 | 
22 | 


--------------------------------------------------------------------------------
/4_FullyConvolutionalNetworks/fc.py:
--------------------------------------------------------------------------------
 1 | from tfs import *
 2 | from tfs import input_data
 3 | from pylab import *
 4 | from numpy import *
 5 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
 6 | matplotlib.interactive(True)
 7 | session = tf.InteractiveSession()
 8 | 
 9 | 
10 | # Hacky class to add a null option to the MNIST one hot vector (position 10)
11 | class MNISTModifier(object):
12 | 	
13 | 	def __init__(self,data):
14 | 		self.data = data
15 | 	
16 | 	def next_batch(self,miniBatch):
17 | 		batch = list(self.data.next_batch(miniBatch))
18 | 		batch[1] = np.hstack((batch[1],zeros([shape(batch[1])[0]]+[1])))
19 | 		return batch
20 | 
21 | class Container(object):
22 | 	
23 | 	def __init__(self,mnist):
24 | 		self.train = MNISTModifier(mnist.train)
25 | 		self.test = MNISTModifier(mnist.test)
26 | 		
27 | 		
28 | def makeBigImage(data,Ndigits=5,width=100,height=100,overlap=False,digitShape=[28,28],maxTries=1000):
29 | 	newImage = zeros((height,width),float32)
30 | 	truthImage = zeros(list(shape(newImage))+[11],float32)
31 | 	batch = data.next_batch(Ndigits)
32 | 	tries = 0
33 | 	while Ndigits > 0:
34 | 		y,x = np.random.randint(0,height-digitShape[0],2)
35 | 		if not np.any(truthImage[y:y+digitShape[0],x:x+digitShape[0]]) or overlap:
36 | 			newImage[y:y+digitShape[0],x:x+digitShape[0]] = batch[0][Ndigits-1].reshape(digitShape)
37 | 			truthImage[y:y+digitShape[0],x:x+digitShape[0]] = batch[1][Ndigits-1]
38 | 			Ndigits -= 1; tries = 0
39 | 		else:
40 | 			tries += 1
41 | 		if tries > maxTries:
42 | 			break
43 | 	truthImage[:,:,10] += np.all(truthImage<0.5,axis=-1)
44 | 	return (newImage,truthImage)
45 | 		
46 | 
47 | 
48 | # Hack our MNIST dataset so it has a null option
49 | mnist = Container(mnist)
50 | 
51 | # x = tf.placeholder('float',shape=[None,784],name='input')		# Input tensor
52 | # y_ = tf.placeholder('float', shape=[None,11],name='correctLabels') 		# Correct labels
53 | 
54 | trainingIterations = 1000
55 | height = width = 100
56 | x2 = tf.placeholder('float',shape=[None,height,width],name='input2')		# Input tensor; we shouldn't need to specify dimensions except TensorFlow....
57 | y_2 = tf.placeholder('float', shape=[None,height,width,11],name='correctLabels2') # Correct labels
58 | 
59 | 
60 | inLayer = tf.expand_dims(x2,-1)
61 | L1 = Conv2D(inLayer,[5,5,1,32],'Conv1')
62 | L2 = MaxPool2x2(L1.output,'MaxPool1')
63 | L3 = Conv2D(L2.output,[5,5,32,64],'Conv2')
64 | L4 = MaxPool2x2(L3.output,'MaxPool2')
65 | L5 = Conv2D(L4.output,[7,7,64,128],'relu1-conv')
66 | L6 = ConvSoftMax(L5.output,[1,1,128,11],'softmax-conv')
67 | #L7 = Resample(L6.output,L1.input.get_shape().as_list()[1:3],'upsample')
68 | y = L6.output
69 | 
70 | trainDict = {}
71 | testDict = {}
72 | 
73 | 
74 | truthDownsampleLayer = Resample(y_2,L6.output.get_shape().as_list()[1:3],'ydownsample')
75 | y2 = L6.output; y_2b = truthDownsampleLayer.output
76 | 
77 | crossEntropy = -tf.reduce_sum(y_2b*tf.log(y2))		# cost function
78 | trainStep = tf.train.AdamOptimizer(1e-4).minimize(crossEntropy)
79 | correctPrediction = tf.equal(tf.argmax(y2,3), tf.argmax(y_2b,3))
80 | accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
81 | 
82 | print("Training Image recognizer")
83 | iterations = trainingIterations; batchSize = 2
84 | lastTime = 0; lastIterations = 0; trainingStep=trainStep; accuracy=accuracy;
85 | tf.global_variables_initializer().run()
86 | 
87 | for i in range(iterations):						# Do some more training
88 | 	# batchIndices = random.choice(range(0,len(trainingData)),batchSize)
89 | 	# batch = [trainingData[batchIndices],truthData[batchIndices]]
90 | 	batch = list(zip(*[makeBigImage(data=mnist.train,Ndigits=5,width=width,height=height,overlap=False) for j in range(0,3)]))
91 | 	if (i%100 == 0) or (time.time()-lastTime > 5):
92 | 		testDict.update({x2:batch[0],y_2:batch[1]})
93 | 		testAccuracy = session.run([accuracy],feed_dict=testDict)[0]
94 | 		print('Accuracy at batch {}: {} ({} samples/s)'.format(i,testAccuracy,(i-lastIterations)/(time.time()-lastTime)*batchSize))
95 | 		lastTime = time.time(); lastIterations = i
96 | 	
97 | 	trainDict.update({x2:batch[0],y_2:batch[1]})
98 | 	trainingStep.run(feed_dict=trainDict)
99 | 


--------------------------------------------------------------------------------
/4_FullyConvolutionalNetworks/fullyConvolutional.py:
--------------------------------------------------------------------------------
  1 | from tfs import *
  2 | from tfs import input_data
  3 | from pylab import *
  4 | from numpy import *
  5 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
  6 | matplotlib.interactive(True)
  7 | session = tf.InteractiveSession()
  8 | 
  9 | # Hacky class to add a null option to the MNIST one hot vector (position 10)
 10 | class MNISTModifier(object):
 11 | 	
 12 | 	def __init__(self,data):
 13 | 		self.data = data
 14 | 	
 15 | 	def next_batch(self,miniBatch):
 16 | 		batch = list(self.data.next_batch(miniBatch))
 17 | 		batch[1] = np.hstack((batch[1],zeros([shape(batch[1])[0]]+[1])))
 18 | 		return batch
 19 | 
 20 | class Container(object):
 21 | 	
 22 | 	def __init__(self,mnist):
 23 | 		self.train = MNISTModifier(mnist.train)
 24 | 		self.test = MNISTModifier(mnist.test)
 25 | 
 26 | 
 27 | # Hack our MNIST dataset so it has a null option
 28 | mnist = Container(mnist)
 29 | 
 30 | x = tf.placeholder('float',shape=[None,784],name='input')		# Input tensor
 31 | y_ = tf.placeholder('float', shape=[None,11],name='correctLabels') 		# Correct labels
 32 | 
 33 | xImage = tf.reshape(x,[-1,28,28,1])		# Reshape samples to 28x28x1 images
 34 | trainingIterations = 1000
 35 | 
 36 | # Standard conv net from Session 3
 37 | L1 = Conv2D(xImage,[5,5,1,32],'Conv1')
 38 | L2 = MaxPool2x2(L1.output,'MaxPool1')
 39 | L0do = Dropout(L2.output,'dropout')
 40 | L3 = Conv2D(L2.output,[5,5,32,64],'Conv2')
 41 | L4 = MaxPool2x2(L3.output,'MaxPool2')
 42 | L5 = ReLu(L4.output,128,'relu1')
 43 | L6 = SoftMax(L5.output,11,'softmax')
 44 | y = L6.output
 45 | kp = 0.5; trainDict = {L0do.keepProb:kp}
 46 | kp = 1.0; testDict = {L0do.keepProb:kp}
 47 | logName = None #logName = 'logs/Conv'
 48 | 
 49 | 
 50 | # Training and evaluation
 51 | crossEntropy = -tf.reduce_sum(y_*tf.log(y))		# cost function
 52 | trainStep = tf.train.GradientDescentOptimizer(0.01).minimize(crossEntropy)
 53 | trainStep = tf.train.AdamOptimizer(1e-4).minimize(crossEntropy)
 54 | correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
 55 | accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
 56 | train(session=session,trainingData=mnist.train,testingData=mnist.test,truth=y_,input=x,cost=crossEntropy,trainingStep=trainStep,accuracy=accuracy,iterations=trainingIterations,miniBatch=100,trainDict=trainDict,testDict=testDict,logName=logName)
 57 | 
 58 | 
 59 | 
 60 | # Now turn our trained conv net into a fully convolutional network by replacing the fully
 61 | # connected layers (L5 and L6) with convoluational layers
 62 | # Note that I'm copying the L5 and L6 weights and biases into the new layers
 63 | 
 64 | L5fc = Conv2D(L4.output,[7,7,64,128],'relu1-conv')
 65 | L5fc.W = tf.reshape(L5.W,[7,7,64,128])
 66 | L5fc.b = L5.b
 67 | L5fc.setupOutput()
 68 | 
 69 | L6fc = ConvSoftMax(L5fc.output,[1,1,128,11],'softmax-conv')
 70 | L6fc.W = tf.reshape(L6.W,[1,1,128,11])
 71 | L6fc.b = L6.b
 72 | L6fc.setupOutput()
 73 | 
 74 | # This last layer upsamples the output back up to the input size for comparison
 75 | L7 = Resample(L6fc.output,L1.input.get_shape().as_list()[1:3],'upsample')
 76 | 
 77 | 
 78 | # Choose an example and show it for comparison
 79 | example = mnist.test.next_batch(1); image = reshape(example[0][0],(28,28))
 80 | feed_dict = {x:example[0],y_:example[1],L0do.keepProb:1.0}
 81 | 
 82 | # Regular convolutional network
 83 | figure('Convolutional Network'); clf(); subplot(231); imshow(image,cmap=cm.gray); title('Correct is %d' % where(example[1]>0)[1][0])
 84 | subplot(232); plotOutput(L1,feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
 85 | subplot(233); plotOutput(L2,feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
 86 | subplot(235); plotOutput(L3,feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
 87 | subplot(236); plotOutput(L4,feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
 88 | subplot(234); plotOutput(L6,fieldShape=[11,1],feed_dict=feed_dict,cmap=cm.inferno,figOffset=None);
 89 | 
 90 | # Fully convolutional network
 91 | figure('Fully Convolutional Network'); clf(); subplot(121); imshow(np.argmax(L7.output.eval(feed_dict=feed_dict),axis=-1)[0],vmin=0,vmax=10); colorbar(); title('Digit Identification')
 92 | contour(image,[0.5],colors=['k'])
 93 | subplot(122); imshow(np.max(L7.output.eval(feed_dict=feed_dict),axis=-1)[0],cmap=cm.gray); colorbar(); title('Strength of Identification')
 94 | contour(image,[0.5],colors=['b'])
 95 | 
 96 | 
 97 | 
 98 | # Do a bit more training but stick a null image in each batch to teach the network about
 99 | # places where there is no value.  This code copied from TensorFlowInterface
100 | nullImage = [zeros(28*28,float32)]; nullTruth = array([zeros(11,float32)]); nullTruth[:,0] = 1.
101 | trainingData = mnist.train; iterations = trainingIterations; miniBatch = 100
102 | lastTime = 0; lastIterations = 0; trainingStep=trainStep; accuracy=accuracy
103 | for i in range(iterations):						# Do some more training
104 | 	batch = list(trainingData.next_batch(miniBatch))
105 | 	batch[0] = np.concatenate((batch[0],nullImage)); batch[1] = concatenate((batch[1],nullTruth))
106 | 	if (i%100 == 0) or (time.time()-lastTime > 5):
107 | 		testDict.update({x:batch[0],y_:batch[1]})
108 | 		testAccuracy = session.run([accuracy],feed_dict=testDict)[0]
109 | 		print('Accuracy at batch {}: {} ({} samples/s)'.format(i,testAccuracy,(i-lastIterations)/(time.time()-lastTime)*miniBatch))
110 | 		lastTime = time.time(); lastIterations = i
111 | 	
112 | 	trainDict.update({x:batch[0],y_:batch[1]})
113 | 	trainingStep.run(feed_dict=trainDict)
114 | 
115 | 
116 | # Plot again and see the difference
117 | figure('Fully Convolutional Network With Null Images'); clf(); subplot(121); imshow(np.argmax(L7.output.eval(feed_dict=feed_dict),axis=-1)[0],vmin=0,vmax=10); colorbar(); title('Digit Identification')
118 | contour(image,[0.5],colors=['k'])
119 | subplot(122); imshow(np.max(L7.output.eval(feed_dict=feed_dict),axis=-1)[0],cmap=cm.gray); colorbar(); title('Strength of Identification')
120 | contour(image,[0.5],colors=['b'])
121 | 
122 | 
123 | 
124 | # Make a bigger image with several numbers in it and see if we can find their location
125 | # AND identify them!
126 | 
127 | def makeBigImage(data,Ndigits=5,width=100,height=100,overlap=False,digitShape=[28,28],maxTries=1000):
128 | 	newImage = zeros((height,width),float32)
129 | 	truthImage = zeros(list(shape(newImage))+[11],float32)
130 | 	batch = data.next_batch(Ndigits)
131 | 	tries = 0
132 | 	while Ndigits > 0:
133 | 		y,x = np.random.randint(0,height-digitShape[0],2)
134 | 		if not np.any(truthImage[y:y+digitShape[0],x:x+digitShape[0]]) or overlap:
135 | 			newImage[y:y+digitShape[0],x:x+digitShape[0]] = batch[0][Ndigits-1].reshape(digitShape)
136 | 			truthImage[y:y+digitShape[0],x:x+digitShape[0]] = batch[1][Ndigits-1]
137 | 			Ndigits -= 1; tries = 0
138 | 		else:
139 | 			tries += 1
140 | 		if tries > maxTries:
141 | 			break
142 | 	truthImage[:,:,10] += np.all(truthImage<0.5,axis=-1)
143 | 	return (newImage,truthImage)
144 | 
145 | height = width = 100
146 | newImage,truthImage = makeBigImage(data = mnist.train,Ndigits=5,width=width,height=height,overlap=False)
147 | 
148 | # What's it look like?
149 | figure('Big Image and Truth'); clf(); subplot(121); imshow(newImage,cmap=cm.gray); subplot(122); imshow(np.argmax(truthImage,axis=-1),vmin=0,vmax=10); colorbar()
150 | 
151 | # We need new placeholders to hold the bigger image
152 | x2 = tf.placeholder('float',shape=[None,height,width],name='input2')		# Input tensor; we shouldn't need to specify dimensions except TensorFlow....
153 | y_2 = tf.placeholder('float', shape=[None,height,width,11],name='correctLabels2') # Correct labels
154 | 
155 | # We now need to rebuild our network to work with the new input
156 | lastLayerOutput = tf.expand_dims(x2,-1)			# Add the extra dummy dimension
157 | for layer in [L1,L2,L3,L4,L5fc,L6fc]:
158 | 	layer.input = lastLayerOutput; 
159 | 	layer.setupOutput()
160 | 	lastLayerOutput = layer.output
161 | 
162 | # Need to make a new L7 to match the input
163 | L7 = Resample(L6fc.output,L1.input.get_shape().as_list()[1:3],'upsample2')
164 | 
165 | 
166 | 
167 | # Forward prop
168 | feed_dict = {x2:[newImage],y_2:[truthImage],L0do.keepProb:1.0}
169 | figure('Before We Train The FC Net'); clf(); subplot(121); imshow(np.argmax(L7.output.eval(feed_dict=feed_dict),axis=-1)[0],vmin=0,vmax=10); colorbar(); title('Digit Identification')
170 | contour(newImage,[0.5],colors=['k'])
171 | subplot(122); imshow(np.max(L7.output.eval(feed_dict=feed_dict),axis=-1)[0],cmap=cm.gray); colorbar(); title('Strength of Identification')
172 | contour(newImage,[0.5],colors=['b'])
173 | 
174 | 
175 | # Train again, on the big image specifically
176 | # Tensorflow doesn't know how to do gradients on interpolated images (!) so lets downsample
177 | # the truth to match the L6 output
178 | 
179 | truthDownsampleLayer = Resample(y_2,L6fc.output.get_shape().as_list()[1:3],'ydownsample')
180 | 
181 | y2 = L6fc.output; y_2b = truthDownsampleLayer.output
182 | crossEntropy = -tf.reduce_sum(y_2b*tf.log(y2))		# cost function
183 | trainStep = tf.train.AdamOptimizer(1e-4).minimize(crossEntropy)
184 | correctPrediction = tf.equal(tf.argmax(y2,3), tf.argmax(y_2b,3))
185 | accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
186 | 
187 | tf.initialize_all_variables().run()
188 | 
189 | print("Training Image recognizer")
190 | iterations = trainingIterations; batchSize = 2
191 | lastTime = 0; lastIterations = 0; trainingStep=trainStep; accuracy=accuracy;
192 | for i in range(iterations):						# Do some more training
193 | 	# batchIndices = random.choice(range(0,len(trainingData)),batchSize)
194 | 	# batch = [trainingData[batchIndices],truthData[batchIndices]]
195 | 	batch = zip(*[makeBigImage(data=mnist.train,Ndigits=5,width=width,height=height,overlap=False) for j in range(0,3)])
196 | 	if (i%100 == 0) or (time.time()-lastTime > 5):
197 | 		testDict.update({x2:batch[0],y_2:batch[1]})
198 | 		testAccuracy = session.run([accuracy],feed_dict=testDict)[0]
199 | 		print('Accuracy at batch {}: {} ({} samples/s)'.format(i,testAccuracy,(i-lastIterations)/(time.time()-lastTime)*batchSize))
200 | 		lastTime = time.time(); lastIterations = i
201 | 	
202 | 	trainDict.update({x2:batch[0],y_2:batch[1]})
203 | 	trainingStep.run(feed_dict=trainDict)
204 | 
205 | 
206 | feed_dict = {x2:[newImage],y_2:[truthImage],L0do.keepProb:1.0}
207 | classification = np.argmax(L7.output.eval(feed_dict=feed_dict),axis=-1)[0]
208 | confidence = np.max(L7.output.eval(feed_dict=feed_dict),axis=-1)[0]
209 | figure('After FC Net Trained'); clf(); subplot(121); imshow(classification,vmin=0,vmax=10); colorbar(); title('Digit Identification')
210 | contour(newImage,[0.5],colors=['k'])
211 | subplot(122); imshow(confidence,cmap=cm.gray); colorbar(); title('Strength of Identification')
212 | contour(newImage,[0.5],colors=['b'])
213 | 
214 | figure('Final Output'); clf(); imshow(confidence*10,cmap=cm.gray); imshow(classification,vmin=0,vmax=10,alpha=0.3); colorbar()
215 | contour(newImage,[0.5],colors=['k'])
216 | 


--------------------------------------------------------------------------------
/5_Autoencoders/TensorFlowInterface.py:
--------------------------------------------------------------------------------
  1 | """The MIT License (MIT)
  2 | 
  3 | Copyright (c) 2016 Robert A. Brown (www.robbtech.com)
  4 | 
  5 | Permission is hereby granted, free of charge, to any person obtaining a copy
  6 | of this software and associated documentation files (the "Software"), to deal
  7 | in the Software without restriction, including without limitation the rights
  8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9 | copies of the Software, and to permit persons to whom the Software is
 10 | furnished to do so, subject to the following conditions:
 11 | 
 12 | The above copyright notice and this permission notice shall be included in all
 13 | copies or substantial portions of the Software.
 14 | 
 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 21 | SOFTWARE.
 22 | """
 23 | 
 24 | 
 25 | import tensorflow as tf
 26 | import math
 27 | import pylab as mpl
 28 | import numpy as np
 29 | import time
 30 | from collections import OrderedDict
 31 | 
 32 | 
 33 | def weightVariable(shape,std=1.0,name=None):
 34 | 	# Create a set of weights initialized with truncated normal random values
 35 | 	name = 'weights' if name is None else name
 36 | 	return tf.get_variable(name,shape,initializer=tf.truncated_normal_initializer(stddev=std/math.sqrt(shape[0])))
 37 | 
 38 | def biasVariable(shape,bias=0.1,name=None):
 39 | 	# create a set of bias nodes initialized with a constant 0.1
 40 | 	name = 'biases' if name is None else name
 41 | 	return tf.get_variable(name,shape,initializer=tf.constant_initializer(bias))
 42 | 
 43 | def conv2d(x,W,name=None):
 44 | 	# return an op that convolves x with W
 45 | 	return tf.nn.conv2d(x,W,strides=[1,1,1,1],padding='SAME',name=name)
 46 | 
 47 | def conv3d(x,W,name=None):
 48 | 	# return an op that convolves x with W
 49 | 	return tf.nn.conv3d(x,W,strides=[1,1,1,1,1],padding='SAME',name=name)
 50 | 
 51 | def max_pool_2x2(x,name=None):
 52 | 	# return an op that performs max pooling across a 2D image
 53 | 	return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME',name=name)
 54 | 
 55 | def max_pool(x,shape,name=None):
 56 | 	# return an op that performs max pooling across a 2D image
 57 | 	return tf.nn.max_pool(x,ksize=[1]+shape+[1],strides=[1]+shape+[1],padding='SAME',name=name)
 58 | 
 59 | def max_pool3d(x,shape,name=None):
 60 | 	# return an op that performs max pooling across a 2D image
 61 | 	return tf.nn.max_pool3d(x,ksize=[1]+shape+[1],strides=[1]+shape+[1],padding='SAME',name=name)
 62 | 
 63 | 
 64 | def plotFields(layer,fieldShape=None,channel=None,maxFields=25,figName='ReceptiveFields',cmap=None,padding=0.01):
 65 | 	# Receptive Fields Summary
 66 | 	W = layer.W
 67 | 	wp = W.eval().transpose();
 68 | 	if len(np.shape(wp)) < 4:		# Fully connected layer, has no shape
 69 | 		fields = np.reshape(wp,list(wp.shape[0:-1])+fieldShape)
 70 | 	else:			# Convolutional layer already has shape
 71 | 		features, channels, iy, ix = np.shape(wp)
 72 | 		if channel is not None:
 73 | 			fields = wp[:,channel,:,:]
 74 | 		else:
 75 | 			fields = np.reshape(wp,[features*channels,iy,ix])
 76 | 
 77 | 	fieldsN = min(fields.shape[0],maxFields)
 78 | 	perRow = int(math.floor(math.sqrt(fieldsN)))
 79 | 	perColumn = int(math.ceil(fieldsN/float(perRow)))
 80 | 
 81 | 	fig = mpl.figure(figName); mpl.clf()
 82 | 
 83 | 	# Using image grid
 84 | 	from mpl_toolkits.axes_grid1 import ImageGrid
 85 | 	grid = ImageGrid(fig,111,nrows_ncols=(perRow,perColumn),axes_pad=padding,cbar_mode='single')
 86 | 	for i in range(0,fieldsN):
 87 | 		im = grid[i].imshow(fields[i],cmap=cmap);
 88 | 
 89 | 	grid.cbar_axes[0].colorbar(im)
 90 | 	mpl.title('%s Receptive Fields' % layer.name)
 91 | 
 92 | 	# old way
 93 | 	# fields2 = np.vstack([fields,np.zeros([perRow*perColumn-fields.shape[0]] + list(fields.shape[1:]))])
 94 | 	# tiled = []
 95 | 	# for i in range(0,perColumn*perRow,perColumn):
 96 | 	# 	tiled.append(np.hstack(fields2[i:i+perColumn]))
 97 | 	#
 98 | 	# tiled = np.vstack(tiled)
 99 | 	# mpl.figure(figOffset); mpl.clf(); mpl.imshow(tiled,cmap=cmap); mpl.title('%s Receptive Fields' % layer.name); mpl.colorbar();
100 | 	mpl.figure(figName+' Total'); mpl.clf(); mpl.imshow(np.sum(np.abs(fields),0),cmap=cmap); mpl.title('%s Total Absolute Input Dependency' % layer.name); mpl.colorbar()
101 | 
102 | 
103 | 
104 | def plotOutput(layer,feed_dict,fieldShape=None,channel=None,figOffset=1,cmap=None):
105 | 	# Output summary
106 | 	W = layer.output
107 | 	wp = W.eval(feed_dict=feed_dict);
108 | 	if len(np.shape(wp)) < 4:		# Fully connected layer, has no shape
109 | 		temp = np.zeros(np.product(fieldShape)); temp[0:np.shape(wp.ravel())[0]] = wp.ravel()
110 | 		fields = np.reshape(temp,[1]+fieldShape)
111 | 	else:			# Convolutional layer already has shape
112 | 		wp = np.rollaxis(wp,3,0)
113 | 		features, channels, iy,ix = np.shape(wp)
114 | 		if channel is not None:
115 | 			fields = wp[:,channel,:,:]
116 | 		else:
117 | 			fields = np.reshape(wp,[features*channels,iy,ix])
118 | 
119 | 	perRow = int(math.floor(math.sqrt(fields.shape[0])))
120 | 	perColumn = int(math.ceil(fields.shape[0]/float(perRow)))
121 | 	fields2 = np.vstack([fields,np.zeros([perRow*perColumn-fields.shape[0]] + list(fields.shape[1:]))])
122 | 	tiled = []
123 | 	for i in range(0,perColumn*perRow,perColumn):
124 | 		tiled.append(np.hstack(fields2[i:i+perColumn]))
125 | 
126 | 	tiled = np.vstack(tiled)
127 | 	if figOffset is not None:
128 | 		mpl.figure(figOffset); mpl.clf();
129 | 
130 | 	mpl.imshow(tiled,cmap=cmap); mpl.title('%s Output' % layer.name); mpl.colorbar();
131 | 
132 | 
133 | 
134 | def train(session,trainingData,testingData,input,truth,cost,trainingStep,accuracy=None,iterations=5000,miniBatch=100,trainDict={},testDict=None,logName=None,initialize=True,addSummaryOps=True,plot=True):
135 | 	testDict = trainDict if testDict is None else testDict
136 | 
137 | 	metrics = OrderedDict(cost=cost) if accuracy is None else OrderedDict(cost=cost,accuracy=accuracy)
138 | 	history = []
139 | 
140 | 	if addSummaryOps:
141 | 		costSummary = tf.scalar_summary("Cost Function", cost)
142 | 		if accuracy is not None:
143 | 			accuracySummary = tf.scalar_summary("accuracy", accuracy)
144 | 		mergedSummary = tf.merge_all_summaries()
145 | 		if logName is not None:
146 | 			writer = tf.train.SummaryWriter(logName, session.graph_def)
147 | 
148 | 	if initialize:
149 | 		tf.initialize_all_variables().run()		# Take initial values and actually put them in variables
150 | 
151 | 	startTime = lastTime = time.time(); lastIterations = 0
152 | 	if iterations < 0:
153 | 		print "Training for %0.2f hours" % -iterations
154 | 	else:
155 | 		print "Doing %d iterations" % iterations
156 | 	done = False; i = 0;
157 | 	while not done:						# Do some training
158 | 		batch = trainingData.next_batch(miniBatch)
159 | 
160 | 		now = time.time()
161 | 		# Print some summary stats
162 | 		if (i%100 == 0) or (now-lastTime > 5):
163 | 			testDict.update({input:batch[0],truth:batch[1]})
164 | 			# Training accuracy for TensorBoard
165 | 			if addSummaryOps:
166 | 				testResults = session.run([mergedSummary]+metrics.values(),feed_dict=testDict)
167 | 				testResults = dict(zip(['summary']+metrics.keys(),testResults))
168 | 				if logName is not None:
169 | 					writer.add_summary(testResults['summary'],i)
170 | 			else:
171 | 				if accuracy is not None:
172 | 					testResults = session.run(metrics.values(),feed_dict=testDict)
173 | 					testResults = dict(zip(metrics.keys(),testResults))
174 | 
175 | 			testResults.pop('summary',None)
176 | 
177 | 			print 'Training Performance at batch %d: (%g samples/s)' % (i,(i-lastIterations)/(now-lastTime)*miniBatch)
178 | 			for k,v in testResults.items():
179 | 				if not k == 'summary':
180 | 					print '	%s: %g' % (k,v)
181 | 
182 | 			lastTime = time.time(); lastIterations = i
183 | 
184 | 			testResults['iteration'] = i
185 | 			testResults['time'] = now
186 | 			history.append(testResults)
187 | 			if plot:
188 | 				timecourses = OrderedDict(zip(history[0].keys(),zip(*[hist.values() for hist in history])))
189 | 				elapsed = np.array(timecourses['time']) - timecourses['time'][0]
190 | 				for k,v in timecourses.items():
191 | 					if not k in ['iteration','time']:
192 | 						mpl.figure('Training %s' % k); mpl.clf()
193 | 						mpl.plot(elapsed,v,alpha=0.5,marker='o')
194 | 						mpl.pause(0.000000001)
195 | 
196 | 
197 | 		trainDict.update({input:batch[0],truth:batch[1]})
198 | 		trainingStep.run(feed_dict=trainDict)
199 | 		i += 1
200 | 		if iterations > 0:
201 | 			if i > iterations:
202 | 				done = True
203 | 		else:
204 | 			if (now - startTime) / 3600. > -iterations:
205 | 				done = True
206 | 
207 | 
208 | 	# try:
209 | 	# 	# Only works with mnist-type data object
210 | 	# 	testDict.update({input:testingData.images, truth:testingData.labels})
211 | 	# 	print 'Test accuracy: %g' % accuracy.eval(feed_dict=testDict)
212 | 	# except:
213 | 	# 	pass
214 | 
215 | 
216 | 
217 | 
218 | 
219 | class Layer(object):
220 | 
221 | 	def __init__(self,input,units,name,std=1.0,bias=0.1):
222 | 		self.input = input
223 | 		self.units = units
224 | 		self.name = name
225 | 		self.initialize(std=std,bias=bias)
226 | 		self.setupOutput()
227 | 		self.setupSummary()
228 | 
229 | 	def initialize(self):
230 | 		pass
231 | 
232 | 	def setupOutput(self):
233 | 		pass
234 | 
235 | 	def setupSummary(self):
236 | 		pass
237 | 
238 | 
239 | class UtilityLayer(Layer):
240 | 
241 | 	def __init__(self,input,name):
242 | 		self.input = input
243 | 		self.name = name
244 | 		self.initialize()
245 | 		self.setupOutput()
246 | 		self.setupSummary()
247 | 
248 | 
249 | class Linear(Layer):
250 | 
251 | 	def initialize(self,std=1.0,bias=0.1):
252 | 		with tf.variable_scope(self.name):
253 | 			self.inputShape = np.product([i.value for i in self.input.get_shape()[1:] if i.value is not None])
254 | 			self.W = weightVariable([self.inputShape,self.units],std=std)
255 | 			self.b = biasVariable([self.units],bias=bias)
256 | 
257 | 	def setupOutput(self):
258 | 		if len(self.input.get_shape()) > 2:
259 | 			input = tf.reshape(self.input,[-1,self.inputShape])	# flatten reduced image into a vector
260 | 		else:
261 | 			input = self.input
262 | 		self.output = tf.matmul(input,self.W)
263 | 
264 | 	def setupSummary(self):
265 | 		self.WHist = tf.histogram_summary("%s/weights" % self.name, self.W)
266 | 		self.BHist = tf.histogram_summary("%s/biases" % self.name, self.b)
267 | 		self.outputHist = tf.histogram_summary("%s/output" % self.name, self.output)
268 | 
269 | 
270 | 
271 | class SoftMax(Layer):
272 | 
273 | 	def initialize(self,std=1.0,bias=0.1):
274 | 		with tf.variable_scope(self.name):
275 | 			self.inputShape = np.product([i.value for i in self.input.get_shape()[1:] if i.value is not None])
276 | 			self.W = weightVariable([self.inputShape,self.units],std=std)
277 | 			self.b = biasVariable([self.units],bias=bias)
278 | 
279 | 	def setupOutput(self):
280 | 		if len(self.input.get_shape()) > 2:
281 | 			input = tf.reshape(self.input,[-1,self.inputShape])	# flatten reduced image into a vector
282 | 		else:
283 | 			input = self.input
284 | 		self.output = tf.nn.softmax(tf.matmul(input,self.W) + self.b)
285 | 
286 | 	def setupSummary(self):
287 | 		self.WHist = tf.histogram_summary("%s/weights" % self.name, self.W)
288 | 		self.BHist = tf.histogram_summary("%s/biases" % self.name, self.b)
289 | 		self.outputHist = tf.histogram_summary("%s/output" % self.name, self.output)
290 | 
291 | 
292 | class ReLu(SoftMax):
293 | 
294 | 	def setupOutput(self):
295 | 		if len(self.input.get_shape()) > 2:
296 | 			input = tf.reshape(self.input,[-1,self.inputShape])	# flatten reduced image into a vector
297 | 		else:
298 | 			input = self.input
299 | 		self.output = tf.nn.relu(tf.matmul(input,self.W) + self.b)
300 | 
301 | 
302 | class Conv2D(SoftMax):
303 | 
304 | 	def __init__(self,input,shape,name,std=1.0,bias=0.1):
305 | 		self.input = input
306 | 		self.units = shape[-1]
307 | 		self.shape = shape
308 | 		self.name = name
309 | 		self.initialize(std=std,bias=bias)
310 | 		self.setupOutput()
311 | 		self.setupSummary()
312 | 
313 | 
314 | 	def initialize(self,std=1.0,bias=0.1):
315 | 		with tf.variable_scope(self.name):
316 | 			self.W = weightVariable(self.shape,std=std)		# YxX patch, Z contrast, outputs to N neurons
317 | 			self.b = biasVariable([self.shape[-1]],bias=bias)	# N bias variables to go with the N neurons
318 | 
319 | 	def setupOutput(self):
320 | 		self.output = tf.nn.relu(conv2d(self.input,self.W) + self.b)
321 | 
322 | 
323 | class ConvSoftMax(Conv2D):
324 | 
325 | 	def setupOutput(self):
326 | 		inputShape = self.input.get_shape()
327 | 		convResult = conv2d(self.input,self.W) + self.b
328 | 
329 | 		convResult = tf.reshape(convResult,[-1,self.units])	# flatten reduced image into a vector
330 | 		softMaxed = tf.nn.softmax(convResult)
331 | 		self.output = tf.reshape(softMaxed,[-1] + inputShape[1:3].as_list() + [self.units])
332 | 
333 | 
334 | class Conv3D(Conv2D):
335 | 
336 | 	def setupOutput(self):
337 | 		self.output = tf.nn.relu(conv3d(self.input,self.W) + self.b)
338 | 
339 | 
340 | class Conv3DSoftMax(ConvSoftMax):
341 | 
342 | 	def setupOutput(self):
343 | 		inputShape = self.input.get_shape()
344 | 		convResult = conv3d(self.input,self.W) + self.b
345 | 
346 | 		convResult = tf.reshape(convResult,[-1,self.units])	# flatten reduced image into a vector
347 | 		softMaxed = tf.nn.softmax(convResult)
348 | 		self.output = tf.reshape(softMaxed,[-1] + inputShape[1:4].as_list() + [self.units])
349 | 
350 | 
351 | 
352 | class MaxPool2x2(UtilityLayer):
353 | 
354 | 	def setupOutput(self):
355 | 		with tf.variable_scope(self.name):
356 | 			self.output = max_pool_2x2(self.input)
357 | 
358 | 
359 | 
360 | class MaxPool(UtilityLayer):
361 | 
362 | 	def __init__(self,input,shape,name):
363 | 		self.shape = shape
364 | 		super(MaxPool,self).__init__(input,name)
365 | 
366 | 	def setupOutput(self):
367 | 		with tf.variable_scope(self.name):
368 | 			self.output = max_pool(self.input,shape=self.shape)
369 | 
370 | 
371 | class MaxPool3D(MaxPool):
372 | 
373 | 	def setupOutput(self):
374 | 		with tf.variable_scope(self.name):
375 | 			self.output = max_pool3d(self.input,shape=self.shape)
376 | 
377 | 
378 | class L2Norm(UtilityLayer):
379 | 
380 | 	def __init__(self,input,name):
381 | 		super(L2Norm,self).__init__(input,name)
382 | 
383 | 	def setupOutput(self):
384 | 		with tf.variable_scope(self.name):
385 | 			self.output = tf.nn.l2_normalize(self.input,-1)
386 | 
387 | 
388 | class Resample(UtilityLayer):
389 | 
390 | 	def __init__(self,input,outputShape,name,method=tf.image.ResizeMethod.BICUBIC,alignCorners=True):
391 | 		self.outputShape = outputShape
392 | 		self.method = method
393 | 		self.alignCorners = alignCorners
394 | 		super(Resample,self).__init__(input,name)
395 | 
396 | 	def setupOutput(self):
397 | 		with tf.variable_scope(self.name):
398 | 			self.output = tf.image.resize_images(self.input,self.outputShape[0],self.outputShape[1],method=self.method)#,align_corners=self.alignCorners)
399 | 
400 | 
401 | class Dropout(UtilityLayer):
402 | 
403 | 	def __init__(self,input,name):
404 | 		self.input = input
405 | 		self.name = name
406 | 		super(Dropout,self).__init__(input,name)
407 | 
408 | 	def initialize(self):
409 | 		with tf.variable_scope(self.name):
410 | 			self.keepProb = tf.placeholder('float')			# Variable to hold the dropout probability
411 | 
412 | 	def setupOutput(self):
413 | 		with tf.variable_scope(self.name):
414 | 			self.output = tf.nn.dropout(self.input,self.keepProb)
415 | 			self.output.get_shape = self.input.get_shape		# DEBUG: remove this whenever TensorFlow fixes this bug
416 | 
417 | 
418 | 
419 | #*** Main Part ***
420 | if __name__ == '__main__':
421 | 	import input_data
422 | 	mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
423 | 	session = tf.InteractiveSession()
424 | 
425 | 	x = tf.placeholder('float',shape=[None,784],name='input')		# Input tensor
426 | 	y_ = tf.placeholder('float', shape=[None,10],name='correctLabels') 		# Correct labels
427 | 	trainingIterations = 5000
428 | 
429 | #	L1 = ReLu(x,512,'relu1')
430 | #	L2 = ReLu(L1.output,128,'relu2')
431 | #	L3 = ReLu(L2.output,64,'relu3')
432 | #	L4 = SoftMax(x,10,'softmax')
433 | #	y = L4.output
434 | #	trainDict = {}; testDict = trainDict
435 | #	logName = 'logs/softmax'
436 | 
437 | 
438 | 	xImage = tf.reshape(x,[-1,28,28,1])		# Reshape samples to 28x28x1 images
439 | 	L1 = Conv2D(xImage,[5,5,1,32],'Conv1')
440 | 	L2 = MaxPool2x2(L1.output,'MaxPool1')
441 | 	L3 = Conv2D(L2.output,[5,5,32,64],'Conv2')
442 | 	L4 = MaxPool2x2(L3.output,'MaxPool2')
443 | 	L5 = ReLu(L4.output,128,'relu1')
444 | 	L6 = Dropout(L5.output,'dropout')
445 | 	L7 = SoftMax(L5.output,10,'softmax')
446 | 	y = L7.output
447 | 	kp = 0.5; trainDict = {L6.keepProb:kp}
448 | 	kp = 1.0; testDict = {L6.keepProb:kp}
449 | 	logName = 'logs/Conv'
450 | 
451 | 	# Training and evaluation
452 | 	crossEntropy = -tf.reduce_sum(y_*tf.log(y))		# cost function
453 | 	trainStep = tf.train.GradientDescentOptimizer(0.01).minimize(crossEntropy)
454 | 	trainStep = tf.train.AdamOptimizer(1e-4).minimize(crossEntropy)
455 | 	correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
456 | 	accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
457 | 	train(session=session,trainingData=mnist.train,testingData=mnist.test,truth=y_,input=x,cost=crossEntropy,trainingStep=trainStep,accuracy=accuracy,iterations=trainingIterations,miniBatch=100,trainDict=trainDict,testDict=testDict,logName=logName)
458 | 
459 | 	#plotFields(L1,[28,28],figOffset=1)
460 | 


--------------------------------------------------------------------------------
/5_Autoencoders/autoencoderMNIST.py:
--------------------------------------------------------------------------------
  1 | from TensorFlowInterface import *
  2 | import input_data
  3 | from pylab import *
  4 | from numpy import *
  5 | import copy
  6 | 
  7 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
  8 | matplotlib.interactive(True)
  9 | session = tf.InteractiveSession()
 10 | 
 11 | x = tf.placeholder('float',shape=[None,784],name='input')		# Input tensor
 12 | y_ = tf.placeholder('float', shape=[None,10],name='correctLabels') 		# Correct labels
 13 | 
 14 | 
 15 | 
 16 | # ---------------------   Fully connected network with dropout  ---------------------
 17 | 
 18 | trainingIterations = -8
 19 | yp = x
 20 | inputDO = Dropout(x,'inputDropout')
 21 | 
 22 | layerPlan = [100,50]
 23 | layerPlan = [500,300,100];
 24 | 
 25 | layers = dict(encoder=[],decoder=[]); inLayer = inputDO
 26 | for l,nUnits in enumerate(layerPlan):
 27 | 	layer = ReLu(inLayer.output,nUnits,'encoder%d'%l)
 28 | 	dropout = Dropout(layer.output,'encoderDO%d'%l)
 29 | 	layers['encoder'].append([layer,dropout]); inLayer = dropout
 30 | 
 31 | for l,nUnits in enumerate(layerPlan[::-1]):
 32 | 	layer = ReLu(inLayer.output,nUnits,'decoder%d'%l)
 33 | 	dropout = Dropout(layer.output,'decoderDO%d'%l)
 34 | 	layers['decoder'].append([layer,dropout]); inLayer = dropout
 35 | 
 36 | layer = ReLu(inLayer.output,784,'decoder%d'%(l+1))
 37 | layers['decoder'].append([layer,None])
 38 | y = layer.output
 39 | 
 40 | inputKP = 0.7; hiddenKP = 0.3
 41 | #inputKP = 1.0; hiddenKP = 1.0
 42 | trainDict = dict([(layer[1].keepProb,inputKP) for layer in layers['encoder']])
 43 | trainDict.update(dict([(layer[1].keepProb,inputKP) for layer in layers['decoder'] if layer[1] is not None]))
 44 | trainDict[inputDO.keepProb] = hiddenKP
 45 | 
 46 | testDict = dict([(layer[1].keepProb,1.0) for layer in layers['encoder']])
 47 | testDict.update(dict([(layer[1].keepProb,1.0) for layer in layers['decoder'] if layer[1] is not None]))
 48 | testDict[inputDO.keepProb] = 1.0
 49 | 
 50 | logName = 'logs/autoencoderFullyConnected'
 51 | 
 52 | # ----------------------------------------------------------------------------------
 53 | 
 54 | # ---------------------   Convnet with dropout  ---------------------
 55 | if 0:
 56 | 	trainingIterations = 1000
 57 | 	xImage = tf.reshape(x,[-1,28,28,1])		# Reshape samples to 28x28x1 images
 58 | 	yp = xImage
 59 | 
 60 | 	inputDO = Dropout(xImage,'inputDropout')
 61 | 	layerPlan = [25,9]; layers = dict(encoder=[],decoder=[]); inLayer = inputDO; lastUnits = 1
 62 | 	for l,nUnits in enumerate(layerPlan):
 63 | 		layer = Conv2D(inLayer.output,[5,5,lastUnits,nUnits],'encoder%d'%l)
 64 | 		dropout = Dropout(layer.output,'encoderDO%d'%l)
 65 | 		layers['encoder'].append([layer,dropout]); inLayer = dropout; lastUnits = nUnits
 66 | 
 67 | 	for l,nUnits in enumerate(layerPlan[::-1]):
 68 | 		layer = Conv2D(inLayer.output,[5,5,lastUnits,nUnits],'decoder%d'%l)
 69 | 		dropout = Dropout(layer.output,'decoderDO%d'%l)
 70 | 		layers['decoder'].append([layer,dropout]); inLayer = dropout; lastUnits = nUnits
 71 | 
 72 | 	layer = Conv2D(inLayer.output,[5,5,lastUnits,nUnits],'decoder%d'%(l+1))
 73 | 	layers['decoder'].append([layer,None])
 74 | 	y = layer.output
 75 | 
 76 | 	inputKP = 0.7; hiddenKP = 0.3
 77 | 	#inputKP = 1.0; hiddenKP = 1.0
 78 | 	trainDict = dict([(layer[1].keepProb,inputKP) for layer in layers['encoder']])
 79 | 	trainDict.update(dict([(layer[1].keepProb,inputKP) for layer in layers['decoder'] if layer[1] is not None]))
 80 | 	trainDict[inputDO.keepProb] = hiddenKP
 81 | 
 82 | 	testDict = dict([(layer[1].keepProb,1.0) for layer in layers['encoder']])
 83 | 	testDict.update(dict([(layer[1].keepProb,1.0) for layer in layers['decoder'] if layer[1] is not None]))
 84 | 	testDict[inputDO.keepProb] = 1.0
 85 | 
 86 | 	logName = 'logs/autoencoderFullyConnected'
 87 | 
 88 | 
 89 | # Standard conv net
 90 | # L1 = Conv2D(xImage,[5,5,1,32],'Conv1')
 91 | # L2 = MaxPool2x2(L1.output,'MaxPool1')
 92 | # L0do = Dropout(L2.output,'dropout')
 93 | # L3 = Conv2D(L2.output,[5,5,32,64],'Conv2')
 94 | # L4 = MaxPool2x2(L3.output,'MaxPool2')
 95 | # L5 = ReLu(L4.output,128,'relu1')
 96 | # L6 = SoftMax(L5.output,10,'softmax')
 97 | # y = L6.output
 98 | # kp = 0.5; trainDict = {L0do.keepProb:kp}
 99 | # kp = 1.0; testDict = {L0do.keepProb:kp}
100 | # logName = None #logName = 'logs/Conv'
101 | 
102 | 
103 | 
104 | # Training and evaluation
105 | 
106 | # Mean squared error between the input and reconstruction
107 | cost = tf.sqrt(tf.reduce_mean(tf.square(yp-y)))
108 | 
109 | trainStep = tf.train.AdamOptimizer(1e-4).minimize(cost)
110 | train(session=session,trainingData=mnist.train,testingData=mnist.test,truth=y_,input=x,cost=cost,trainingStep=trainStep,iterations=trainingIterations,miniBatch=100,trainDict=trainDict,testDict=testDict,logName=logName)
111 | 
112 | 
113 | # Mini viewer
114 | plotFields(layers['encoder'][0][0],fieldShape=[28,28],maxFields=100,cmap=cm.coolwarm)
115 | 
116 | 
117 | 
118 | example = mnist.test.next_batch(1); image = reshape(example[0][0],(28,28))
119 | testDict.update({x:example[0],y_:example[1]})
120 | 
121 | for l in range(len(layers['encoder'])):
122 | 	figure('Layer Output'); clf()
123 | 	layer = layers['encoder'][l][0]
124 | 	fieldShape = [ceil(sqrt(layer.units)),ceil(layer.units/ceil(sqrt(layer.units)))]
125 | 	plotOutput(layer,feed_dict=testDict,fieldShape=fieldShape,cmap=cm.inferno,figOffset=None);
126 | 	savefig('test/layer%dOutput.pdf' % l,transparent=True)
127 | 
128 | figure('digitOriginal'); clf();
129 | imshow(example[0].reshape((28,28)),cmap=cm.gray)
130 | savefig('test/digitOriginal.jpg')
131 | recon = session.run(y,feed_dict=testDict)
132 | figure('digitReconstructed'); clf();
133 | imshow(recon.reshape((28,28)),cmap=cm.gray)
134 | savefig('test/digitOriginalRecon.jpg')
135 | 
136 | snr = 5.1
137 | image = copy.deepcopy(example[0]);
138 | image += random.normal(0,1./snr,shape(image))
139 | figure('noisy image'); clf();
140 | imshow(image.reshape((28,28)),cmap=cm.gray)
141 | savefig('test/digitSNR%0.1f.jpg'%snr)
142 | 
143 | testDict.update({x:image})
144 | recon = session.run(y,feed_dict=testDict)
145 | figure('reconstruction'); clf();
146 | imshow(recon.reshape((28,28)),cmap=cm.gray)
147 | savefig('test/digitSNR%dRecon.jpg'%snr)
148 | 


--------------------------------------------------------------------------------
/5_Autoencoders/digitOriginal.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/robb-brown/IntroToDeepLearning/ac6398427d0ac829e119122dafba1981bd68ff98/5_Autoencoders/digitOriginal.jpg


--------------------------------------------------------------------------------
/5_Autoencoders/digitOriginalRecon.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/robb-brown/IntroToDeepLearning/ac6398427d0ac829e119122dafba1981bd68ff98/5_Autoencoders/digitOriginalRecon.jpg


--------------------------------------------------------------------------------
/5_Autoencoders/digitSNR10.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/robb-brown/IntroToDeepLearning/ac6398427d0ac829e119122dafba1981bd68ff98/5_Autoencoders/digitSNR10.jpg


--------------------------------------------------------------------------------
/5_Autoencoders/digitSNR10Recon.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/robb-brown/IntroToDeepLearning/ac6398427d0ac829e119122dafba1981bd68ff98/5_Autoencoders/digitSNR10Recon.jpg


--------------------------------------------------------------------------------
/5_Autoencoders/input_data.py:
--------------------------------------------------------------------------------
  1 | """Functions for downloading and reading MNIST data."""
  2 | from __future__ import absolute_import
  3 | from __future__ import division
  4 | from __future__ import print_function
  5 | 
  6 | import gzip
  7 | import os
  8 | 
  9 | import numpy
 10 | from six.moves import urllib
 11 | from six.moves import xrange  # pylint: disable=redefined-builtin
 12 | 
 13 | import traceback
 14 | 
 15 | SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'
 16 | 
 17 | 
 18 | def maybe_download(filename, work_directory):
 19 |   """Download the data from Yann's website, unless it's already here."""
 20 |   if not os.path.exists(work_directory):
 21 |     os.mkdir(work_directory)
 22 |   filepath = os.path.join(work_directory, filename)
 23 |   if not os.path.exists(filepath):
 24 |     filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath)
 25 |     statinfo = os.stat(filepath)
 26 |     print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')
 27 |   return filepath
 28 | 
 29 | 
 30 | def _read32(bytestream):
 31 |   dt = numpy.dtype(numpy.uint32).newbyteorder('>')
 32 |   return numpy.frombuffer(bytestream.read(4), dtype=dt)
 33 | 
 34 | 
 35 | def extract_images(filename):
 36 |   """Extract the images into a 4D uint8 numpy array [index, y, x, depth]."""
 37 |   print('Extracting', filename)
 38 |   with gzip.open(filename) as bytestream:
 39 |     magic = _read32(bytestream)
 40 |     if magic != 2051:
 41 |       raise ValueError(
 42 |           'Invalid magic number %d in MNIST image file: %s' %
 43 |           (magic, filename))
 44 |     num_images = _read32(bytestream)
 45 |     rows = _read32(bytestream)
 46 |     cols = _read32(bytestream)
 47 |     buf = bytestream.read(rows * cols * num_images)
 48 |     data = numpy.frombuffer(buf, dtype=numpy.uint8)
 49 |     data = data.reshape(num_images, rows, cols, 1)
 50 |     return data
 51 | 
 52 | 
 53 | def dense_to_one_hot(labels_dense, num_classes=10):
 54 |   """Convert class labels from scalars to one-hot vectors."""
 55 |   num_labels = labels_dense.shape[0]
 56 |   index_offset = numpy.arange(num_labels) * num_classes
 57 |   labels_one_hot = numpy.zeros((num_labels, num_classes))
 58 |   labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1
 59 |   return labels_one_hot
 60 | 
 61 | 
 62 | def extract_labels(filename, one_hot=False):
 63 |   """Extract the labels into a 1D uint8 numpy array [index]."""
 64 |   print('Extracting', filename)
 65 |   with gzip.open(filename) as bytestream:
 66 |     magic = _read32(bytestream)
 67 |     if magic != 2049:
 68 |       raise ValueError(
 69 |           'Invalid magic number %d in MNIST label file: %s' %
 70 |           (magic, filename))
 71 |     num_items = _read32(bytestream)
 72 |     buf = bytestream.read(num_items)
 73 |     labels = numpy.frombuffer(buf, dtype=numpy.uint8)
 74 |     if one_hot:
 75 |       return dense_to_one_hot(labels)
 76 |     return labels
 77 | 
 78 | 
 79 | class DataSet(object):
 80 | 
 81 |   def __init__(self, images, labels, fake_data=False):
 82 |     if fake_data:
 83 |       self._num_examples = 10000
 84 |     else:
 85 |       assert images.shape[0] == labels.shape[0], (
 86 |           "images.shape: %s labels.shape: %s" % (images.shape,
 87 |                                                  labels.shape))
 88 |       self._num_examples = images.shape[0]
 89 | 
 90 |       # Convert shape from [num examples, rows, columns, depth]
 91 |       # to [num examples, rows*columns] (assuming depth == 1)
 92 |       self.imageShape = images.shape[1:]
 93 |       self.imageChannels = self.imageShape[2]
 94 | 
 95 |       images = images.reshape(images.shape[0],
 96 |                               images.shape[1] * images.shape[2] * images.shape[3])
 97 |       # Convert from [0, 255] -> [0.0, 1.0].
 98 |       images = images.astype(numpy.float32)
 99 |       images = numpy.multiply(images, 1.0 / 255.0)
100 |     self._images = images
101 |     self._labels = labels
102 |     try:
103 |       if len(numpy.shape(self._labels)) == 1:
104 |         self._labels = dense_to_one_hot(self._labels,len(numpy.unique(self._labels)))
105 |     except:
106 |       traceback.print_exc()
107 |     self._epochs_completed = 0
108 |     self._index_in_epoch = 0
109 | 
110 |   @property
111 |   def images(self):
112 |     return self._images
113 | 
114 |   @property
115 |   def labels(self):
116 |     return self._labels
117 | 
118 |   @property
119 |   def num_examples(self):
120 |     return self._num_examples
121 | 
122 |   @property
123 |   def epochs_completed(self):
124 |     return self._epochs_completed
125 | 
126 |   def next_batch(self, batch_size, fake_data=False):
127 |     """Return the next `batch_size` examples from this data set."""
128 |     if fake_data:
129 |       fake_image = [1.0 for _ in xrange(784)]
130 |       fake_label = 0
131 |       return [fake_image for _ in xrange(batch_size)], [
132 |           fake_label for _ in xrange(batch_size)]
133 |     start = self._index_in_epoch
134 |     self._index_in_epoch += batch_size
135 |     if self._index_in_epoch > self._num_examples:
136 |       # Finished epoch
137 |       self._epochs_completed += 1
138 |       # Shuffle the data
139 |       perm = numpy.arange(self._num_examples)
140 |       numpy.random.shuffle(perm)
141 |       self._images = self._images[perm]
142 |       self._labels = self._labels[perm]
143 |       # Start next epoch
144 |       start = 0
145 |       self._index_in_epoch = batch_size
146 |       assert batch_size <= self._num_examples
147 |     end = self._index_in_epoch
148 |     return self._images[start:end], self._labels[start:end]
149 | 
150 | 
151 | def read_data_sets(train_dir, fake_data=False, one_hot=False):
152 |   class DataSets(object):
153 |     pass
154 |   data_sets = DataSets()
155 | 
156 |   if fake_data:
157 |     data_sets.train = DataSet([], [], fake_data=True)
158 |     data_sets.validation = DataSet([], [], fake_data=True)
159 |     data_sets.test = DataSet([], [], fake_data=True)
160 |     return data_sets
161 | 
162 |   TRAIN_IMAGES = 'train-images-idx3-ubyte.gz'
163 |   TRAIN_LABELS = 'train-labels-idx1-ubyte.gz'
164 |   TEST_IMAGES = 't10k-images-idx3-ubyte.gz'
165 |   TEST_LABELS = 't10k-labels-idx1-ubyte.gz'
166 |   VALIDATION_SIZE = 5000
167 | 
168 |   local_file = maybe_download(TRAIN_IMAGES, train_dir)
169 |   train_images = extract_images(local_file)
170 | 
171 |   local_file = maybe_download(TRAIN_LABELS, train_dir)
172 |   train_labels = extract_labels(local_file, one_hot=one_hot)
173 | 
174 |   local_file = maybe_download(TEST_IMAGES, train_dir)
175 |   test_images = extract_images(local_file)
176 | 
177 |   local_file = maybe_download(TEST_LABELS, train_dir)
178 |   test_labels = extract_labels(local_file, one_hot=one_hot)
179 | 
180 |   validation_images = train_images[:VALIDATION_SIZE]
181 |   validation_labels = train_labels[:VALIDATION_SIZE]
182 |   train_images = train_images[VALIDATION_SIZE:]
183 |   train_labels = train_labels[VALIDATION_SIZE:]
184 | 
185 |   data_sets.train = DataSet(train_images, train_labels)
186 |   data_sets.validation = DataSet(validation_images, validation_labels)
187 |   data_sets.test = DataSet(test_images, test_labels)
188 | 
189 |   return data_sets
190 | 


--------------------------------------------------------------------------------
/6_MRISegmentation/ccseg.py:
--------------------------------------------------------------------------------
  1 | from tfs import *
  2 | from pylab import *
  3 | from numpy import *
  4 | import glob, os
  5 | import nibabel as nib
  6 | import numpy
  7 | 
  8 | matplotlib.interactive(True)
  9 | session = tf.InteractiveSession()
 10 | 
 11 | dataPath = './corpusCallosum/'
 12 | 
 13 | 
 14 | 
 15 | # Class to serve up segmented images
 16 | def computePad(dims,depth):
 17 | 	y1=y2=x1=x2=0; 
 18 | 	y,x = [numpy.ceil(dims[i]/float(2**depth)) * (2**depth) for i in range(-2,0)]
 19 | 	x = float(x); y = float(y);
 20 | 	y1 = int(numpy.floor((y - dims[-2])/2)); y2 = int(numpy.ceil((y - dims[-2])/2))
 21 | 	x1 = int(numpy.floor((x - dims[-1])/2)); x2 = int(numpy.ceil((x - dims[-1])/2))
 22 | 	return y1,y2,x1,x2
 23 | 
 24 | 
 25 | def padImage(img,depth):
 26 | 	"""Pads (or crops) an image so it is evenly divisible by 2**depth."""
 27 | 	y1,y2,x1,x2 = computePad(img.shape,depth)
 28 | 	dims = [(0,0) for i in img.shape]
 29 | 	dims[-2] = (y1,y2); dims[-1] = (x1,x2)
 30 | 	return numpy.pad(img,dims,'constant')
 31 | 
 32 | 
 33 | # Class to serve up segmented images
 34 | class CCData(object):
 35 | 	
 36 | 	def __init__(self,paths,padding=None):
 37 | 		self.paths = paths
 38 | 		self.padding = padding
 39 | 	
 40 | 	def getSlices(self,paths):
 41 | 		image,truth = paths
 42 | 		image = nib.load(image).get_data(); truth = nib.load(truth).get_data()
 43 | 		slicesWithValues = [unique(s) for s in where(truth>0)]
 44 | 		sliceAxis = argmin([len(s) for s in slicesWithValues])
 45 | 		slicesWithValues = slicesWithValues[sliceAxis]
 46 | 		slc = repeat(-1,3); slc[sliceAxis] = slicesWithValues[0]
 47 | 		if not self.padding is None:
 48 | 			image, truth = [padImage(im,self.padding) for im in (image[slc][0],truth[slc][0])]
 49 | 		else:
 50 | 			image, truth = (image[slc][0],truth[slc][0])
 51 | 		return (image,truth)
 52 | 	
 53 | 	def next_batch(self,miniBatch=None):
 54 | 		if miniBatch is None or miniBatch==len(self.paths):
 55 | 			batch = arange(0,len(self.paths))
 56 | 		else:
 57 | 			batch = random.choice(arange(0,len(self.paths)),miniBatch)
 58 | 		images = [self.getSlices(self.paths[i]) for i in batch]
 59 | 		return list(zip(*images))
 60 | 		
 61 | 
 62 | class Container(object):
 63 | 
 64 | 	def __init__(self,dataPath,reserve=2,**args):
 65 | 		self.dataPath = dataPath
 66 | 		images = glob.glob(os.path.join(dataPath,'?????.nii.gz'))
 67 | 		images = [(i,i.replace('.nii.gz','_cc.nii.gz')) for i in images]
 68 | 		self.train = CCData(images[0:-reserve],**args)
 69 | 		self.test = CCData(images[reserve:],**args)
 70 | 
 71 | 
 72 | 
 73 | data = Container(dataPath,reserve=2,padding=2)
 74 | batch = data.train.next_batch(2)
 75 | 
 76 | trainingIterations = 1000
 77 | 
 78 | x = tf.placeholder('float',shape=[None,None,None],name='input')
 79 | y_ = tf.placeholder('float', shape=[None,None,None],name='truth')
 80 | y_OneHot = tf.one_hot(indices=tf.cast(y_,tf.int32),depth=2,name='truthOneHot')
 81 | xInput = tf.expand_dims(x,axis=3,name='xInput')
 82 | 
 83 | 
 84 | 
 85 | #Standard conv net from Session 3 using new TensorFlow layers
 86 | net = LD1 = tf.layers.conv2d(
 87 | 					inputs=xInput,
 88 | 					filters=4,
 89 | 					kernel_size=[5,5],
 90 | 					strides = 1,
 91 | 					padding = 'same',
 92 | 					activation=tf.nn.relu,
 93 | 					name='convD1'
 94 | 				)
 95 | 
 96 | net = LO = tf.layers.conv2d(
 97 | 					inputs=net,
 98 | 					filters=2,
 99 | 					kernel_size=[5,5],
100 | 					strides = 1,
101 | 					padding = 'same',
102 | 					activation=tf.nn.relu,
103 | 					name='convLO'
104 | 				)
105 | 
106 | 
107 | logits = LO
108 | y = tf.nn.softmax(logits,-1)
109 | 
110 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y_OneHot, logits=logits)
111 | 
112 | 
113 | trainDict = {}
114 | testDict = {}
115 | logName = None #logName = 'logs/Conv'
116 | 
117 | 
118 | # Training and evaluation
119 | trainStep = tf.train.AdamOptimizer(1e-3).minimize(loss)
120 | 
121 | # Accuracy
122 | correctPrediction = tf.equal(tf.argmax(y,axis=-1), tf.argmax(y_OneHot,axis=-1))
123 | accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
124 | 
125 | # Jaccard
126 | output = tf.cast(tf.argmax(y,axis=-1), dtype=tf.float32)
127 | truth = tf.cast(tf.argmax(y_OneHot,axis=-1), dtype=tf.float32)
128 | intersection = tf.reduce_sum(tf.reduce_sum(tf.multiply(output, truth), axis=-1),axis=-1)
129 | union = tf.reduce_sum(tf.reduce_sum(tf.cast(tf.add(output, truth)>= 1, dtype=tf.float32), axis=-1),axis=-1)
130 | jaccard = tf.reduce_mean(intersection / union)
131 | 
132 | train(session=session,trainingData=data.train,testingData=data.test,truth=y_,input=x,cost=loss,trainingStep=trainStep,accuracy=jaccard,iterations=trainingIterations,miniBatch=2,trainDict=trainDict,testDict=testDict,logName=logName)
133 | # Get a couple of test examples
134 | batch = data.test.next_batch(2)
135 | d = {x:batch[0][0:1],y_:batch[1][0:1]}
136 | print("Test set Jaccard Index: {}").format(jaccard.eval(d))
137 | 
138 | 
139 | # Make a figure
140 | ex = array(batch[0])
141 | segmentation = y.eval({x:ex})
142 | 
143 | # Display each example
144 | figure('Example 1'); clf()
145 | imshow(batch[0][0].transpose(),cmap=cm.gray,origin='lower left');
146 | #contour(batch[1][0].transpose(),alpha=0.5,color='g'); 
147 | contour(segmentation[0,:,:,1].transpose(),alpha=0.5,color='b')
148 | figure('Example 2'); clf()
149 | imshow(batch[0][1].transpose(),cmap=cm.gray,origin='lower left');
150 | #contour(batch[1][1].transpose(),alpha=0.5,color='g'); 
151 | contour(segmentation[1,:,:,1].transpose(),alpha=0.5,color='b')
152 | plotOutput(LD1,{x:ex[0:1]},figOffset='Layer 1 Output')
153 | 


--------------------------------------------------------------------------------
/6_MRISegmentation/corpusCallosum.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/robb-brown/IntroToDeepLearning/ac6398427d0ac829e119122dafba1981bd68ff98/6_MRISegmentation/corpusCallosum.tar.gz


--------------------------------------------------------------------------------
/6_MRISegmentation/support/ccseg.py:
--------------------------------------------------------------------------------
  1 | from tfs import *
  2 | from pylab import *
  3 | import numpy
  4 | from numpy import *
  5 | import tensorflow as tf
  6 | import glob, os
  7 | import nibabel as nib
  8 | 
  9 | matplotlib.interactive(True)
 10 | session = tf.InteractiveSession()
 11 | 
 12 | dataPath = '../corpusCallosum/'
 13 | 
 14 | 
 15 | def computePad(dims,depth):
 16 | 	y1=y2=x1=x2=0; 
 17 | 	y,x = [numpy.ceil(dims[i]/float(2**depth)) * (2**depth) for i in range(-2,0)]
 18 | 	x = float(x); y = float(y);
 19 | 	y1 = int(numpy.floor((y - dims[-2])/2)); y2 = int(numpy.ceil((y - dims[-2])/2))
 20 | 	x1 = int(numpy.floor((x - dims[-1])/2)); x2 = int(numpy.ceil((x - dims[-1])/2))
 21 | 	return y1,y2,x1,x2
 22 | 
 23 | 
 24 | def padImage(img,depth):
 25 | 	"""Pads (or crops) an image so it is evenly divisible by 2**depth."""
 26 | 	y1,y2,x1,x2 = computePad(img.shape,depth)
 27 | 	dims = [(0,0) for i in img.shape]
 28 | 	dims[-2] = (y1,y2); dims[-1] = (x1,x2)
 29 | 	return numpy.pad(img,dims,'constant')
 30 | 
 31 | 
 32 | # Class to serve up segmented images
 33 | class CCData(object):
 34 | 	
 35 | 	def __init__(self,paths,padding=None):
 36 | 		self.paths = paths
 37 | 		self.padding = padding
 38 | 	
 39 | 	def getSlices(self,paths):
 40 | 		image,truth = paths
 41 | 		image = nib.load(image).get_data(); truth = nib.load(truth).get_data()
 42 | 		slicesWithValues = [unique(s) for s in where(truth>0)]
 43 | 		sliceAxis = argmin([len(s) for s in slicesWithValues])
 44 | 		slicesWithValues = slicesWithValues[sliceAxis]
 45 | 		slc = repeat(-1,3); slc[sliceAxis] = slicesWithValues[0]
 46 | 		if not self.padding is None:
 47 | 			image, truth = [padImage(im,self.padding) for im in (image[slc][0],truth[slc][0])]
 48 | 		else:
 49 | 			image, truth = (image[slc][0],truth[slc][0])
 50 | 		return (image,truth)
 51 | 	
 52 | 	def next_batch(self,miniBatch=None):
 53 | 		if miniBatch is None or miniBatch==len(self.paths):
 54 | 			batch = arange(0,len(self.paths))
 55 | 		else:
 56 | 			batch = random.choice(arange(0,len(self.paths)),miniBatch)
 57 | 		images = [self.getSlices(self.paths[i]) for i in batch]
 58 | 		return list(zip(*images))
 59 | 		
 60 | 
 61 | class Container(object):
 62 | 
 63 | 	def __init__(self,dataPath,reserve=2,**args):
 64 | 		self.dataPath = dataPath
 65 | 		images = glob.glob(os.path.join(dataPath,'?????.nii.gz'))
 66 | 		images = [(i,i.replace('.nii.gz','_cc.nii.gz')) for i in images]
 67 | 		self.train = CCData(images[0:-reserve],**args)
 68 | 		self.test = CCData(images[reserve:],**args)
 69 | 
 70 | 
 71 | data = Container(dataPath,reserve=2,padding=3)
 72 | 
 73 | trainingIterations = 5000
 74 | 
 75 | x = tf.placeholder('float',shape=[None,None,None],name='input')
 76 | y_ = tf.placeholder('float', shape=[None,None,None],name='truth')
 77 | y_OneHot = tf.one_hot(indices=tf.cast(y_,tf.int32),depth=2,name='truthOneHot')
 78 | xInput = tf.expand_dims(x,axis=3,name='xInput')
 79 | 
 80 | # ------------------------   Standard conv net from Session 3 ---------------------------------
 81 | # net = LD1 = Conv2D(xInput,[5,5,1,10],strides=1,name='Conv1')
 82 | # net = LD2 = Conv2D(net.output,[5,5,10,20],strides=1,name='Conv2')
 83 | # net = LSM = ConvSoftMax(net.output,[1,1,20,2],name='softmax-conv')
 84 | # y = net.output
 85 | 
 86 | # -------------------------  Standard conv net from Session 3 using new TensorFlow layers ---------------------------
 87 | # net = LD1 = tf.layers.conv2d(
 88 | # 					inputs=xInput,
 89 | # 					filters=10,
 90 | # 					kernel_size=[5,5],
 91 | # 					strides = 1,
 92 | # 					padding = 'same',
 93 | # 					activation=tf.nn.relu,
 94 | # 					name='convD1'
 95 | # 				)
 96 | # net = LD2 = tf.layers.conv2d(
 97 | # 					inputs=net,
 98 | # 					filters=20,
 99 | # 					kernel_size=[5,5],
100 | # 					strides = 1,
101 | # 					padding = 'same',
102 | # 					activation=tf.nn.relu,
103 | # 					name='convD2'
104 | # 				)
105 | # net = LTop = tf.layers.conv2d(
106 | # 					inputs=net,
107 | # 					filters=2,
108 | # 					kernel_size=[5,5],
109 | # 					strides = 1,
110 | # 					padding = 'same',
111 | # 					activation=tf.nn.relu,
112 | # 					name='topRelu'
113 | # 				)
114 | 
115 | 
116 | # ----------------------------------------  U Net ---------------------------------------
117 | 
118 | net = LD1 = tf.layers.conv2d(
119 | 					inputs=xInput,
120 | 					filters=10,
121 | 					kernel_size=[5,5],
122 | 					strides = 2,
123 | 					padding = 'same',
124 | 					activation=tf.nn.relu,
125 | 					name='convD1'
126 | 				)
127 | net = LD2 = tf.layers.conv2d(
128 | 					inputs=net,
129 | 					filters=20,
130 | 					kernel_size=[5,5],
131 | 					strides = 2,
132 | 					padding = 'same',
133 | 					activation=tf.nn.relu,
134 | 					name='convD2'
135 | 				)
136 | net = LB = tf.layers.conv2d(
137 | 					inputs=net,
138 | 					filters=30,
139 | 					kernel_size=[5,5],
140 | 					strides = 1,
141 | 					padding = 'same',
142 | 					activation=tf.nn.relu,
143 | 					name='convBottom'
144 | 				)
145 | net = LU2 = tf.layers.conv2d_transpose(
146 | 					inputs=net,
147 | 					filters=20,
148 | 					kernel_size=[5,5],
149 | 					strides = 2,
150 | 					padding = 'same',
151 | 					activation=tf.nn.relu,
152 | 					name='convU2'
153 | 				)
154 | net = LU1 = tf.layers.conv2d_transpose(
155 | 					inputs=net,
156 | 					filters=10,
157 | 					kernel_size=[5,5],
158 | 					strides = 2,
159 | 					padding = 'same',
160 | 					activation=tf.nn.relu,
161 | 					name='convU1'
162 | 				)
163 | net = LTop = tf.layers.conv2d(
164 | 					inputs=net,
165 | 					filters=2,
166 | 					kernel_size=[5,5],
167 | 					strides = 1,
168 | 					padding = 'same',
169 | 					activation=None,
170 | 					name='logits'
171 | 				)
172 | 				
173 | logits = LTop
174 | y = tf.nn.softmax(logits,-1)
175 | 
176 | kp = 0.5; trainDict = {}#{L0do.keepProb:kp}
177 | kp = 1.0; testDict = {}#{L0do.keepProb:kp}
178 | logName = None #logName = 'logs/Conv'
179 | 
180 | 
181 | # Training and evaluation
182 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y_OneHot, logits=logits)
183 | #loss = -tf.reduce_sum(y_OneHot*tf.log(y+1e-6))	
184 | trainStep = tf.train.AdamOptimizer(1e-3).minimize(loss)
185 | 
186 | # Accuracy
187 | correctPrediction = tf.equal(tf.argmax(y,axis=-1), tf.argmax(y_OneHot,axis=-1))
188 | accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
189 | 
190 | # Jaccard
191 | output = tf.cast(tf.argmax(y,axis=-1), dtype=tf.float32)
192 | truth = tf.cast(tf.argmax(y_OneHot,axis=-1), dtype=tf.float32)
193 | intersection = tf.reduce_sum(tf.reduce_sum(tf.multiply(output, truth), axis=-1),axis=-1)
194 | union = tf.reduce_sum(tf.reduce_sum(tf.cast(tf.add(output, truth)>= 1, dtype=tf.float32), axis=-1),axis=-1)
195 | jaccard = tf.reduce_mean(intersection / union)
196 | 
197 | 
198 | train(session=session,trainingData=data.train,testingData=data.test,truth=y_,input=x,cost=loss,trainingStep=trainStep,accuracy=jaccard,iterations=trainingIterations,miniBatch=2,trainDict=trainDict,testDict=testDict,logName=logName)
199 | 
200 | 
201 | # Make a figure
202 | # Get a couple of examples
203 | batch = data.test.next_batch(2)
204 | d = {x:batch[0][0:1],y_:batch[1][0:1]}
205 | ex = array(batch[0])
206 | segmentation = y.eval({x:ex})
207 | 
208 | # Display each example
209 | # Display each example
210 | figure('Example 1'); clf()
211 | imshow(batch[0][0].transpose(),cmap=cm.gray,origin='lower left');
212 | #contour(batch[1][0].transpose(),alpha=0.5,color='g'); 
213 | contour(segmentation[0,:,:,1].transpose(),alpha=0.5,color='b')
214 | figure('Example 2'); clf()
215 | imshow(batch[0][1].transpose(),cmap=cm.gray,origin='lower left');
216 | #contour(batch[1][1].transpose(),alpha=0.5,color='g'); 
217 | contour(segmentation[1,:,:,1].transpose(),alpha=0.5,color='b')
218 | plotOutput(LD1,{x:ex[0:1]},figOffset='Layer 1 Output')
219 | plotOutput(LD2,{x:ex[0:1]},figOffset='Layer 2 Output')
220 | plotOutput(LU1,{x:ex[0:1]},figOffset='Layer 2 Output')
221 | 
222 | 
223 | 


--------------------------------------------------------------------------------
/6_MRISegmentation/support/prepareData.py:
--------------------------------------------------------------------------------
 1 | import nibabel as nib
 2 | import glob,os
 3 | import numpy
 4 | 
 5 | dataPath = '.'
 6 | 
 7 | images = glob.glob(os.path.join(dataPath,'?????.nii.gz'))
 8 | images = [(i,i.replace('.nii.gz','_cc.nii.gz')) for i in images]
 9 | 
10 | for image in images:
11 | 	im = nib.load(image[0]); mask = nib.load(image[1])
12 | 	slices = numpy.unique(numpy.where(mask.get_data()>0)[0])
13 | 	im2 = im.get_data()[slices.min()-1:slices.max()+2]
14 | 	mask2 = mask.get_data()[slices.min()-1:slices.max()+2]
15 | 	im2 = nib.Nifti1Image(im2.astype(numpy.int16),im.affine)
16 | 	nib.save(im2,os.path.join(dataPath,'smaller',os.path.basename(image[0])))
17 | 	mask2 = nib.Nifti1Image(mask2.astype(numpy.uint8),mask.affine)
18 | 	nib.save(mask2,os.path.join(dataPath,'smaller',os.path.basename(image[1])))
19 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2016 Robert Brown
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Notes for the Practical Deep Learning Series
 2 | This repo contains workshop notes and code from a deep learning seminar series, first presented as the Practical Deep Learning Series with the Toronto Data Literacy Group.
 3 | 
 4 | A presentation of this workshop was also livestreamed from the Montreal Neurological Institute Brain Imaging Centre. The video (8 hours, uncut) is available <a href="https://www.mcgill.ca/bic/channels/news/bic-deep-learning-educational-workshop-265598">here</a> (free registration required).
 5 | 
 6 | Session 1 discuses background theory for artificial neural networks (ANNs), deep learning, and computation graph based libraries, particularly Google's TensorFlow.  
 7 | 
 8 | Session 2 introduces the ANNs covered in session 1 in the form of TensorFlow code.
 9 | 
10 | Session 3 covers convolutional networks
11 | 
12 | Session 4 and 5 are under development, but cover fully convolutional networks and autoencoders, respectively.
13 | 
14 | INSTALLATION
15 | 
16 | Prerequisites:
17 | 
18 | If you are using Linux, make sure you have a compiler toolchain installed (e.g. gcc).  Generally Linux distributions come with this installed by default.
19 | 
20 | If on OS X, you will need the developer tools from Apple installed.  If you're on a recent version of OS X (El Capitan +) you can install just the command line tools by following the instructions at <a href="http://osxdaily.com/2014/02/12/install-command-line-tools-mac-os-x/">http://osxdaily.com/2014/02/12/install-command-line-tools-mac-os-x/</a>.
21 | 
22 | If you're using Windows, you will also need to install gcc.  The easiest way to do this is probably to get the Anaconda Python distribution from <a href="https://anaconda.org/anaconda/python">https://anaconda.org/anaconda/python</a>. If you're using Anaconda, replace "pip install" with "conda install" in the following instructions.  If you have difficulty installing TensorFlow with conda, try pip install instead.
23 | 
24 | Installation instructions:
25 | 
26 | 1) If you do not already have a copy of python 3.6+, install it from <a href="www.python.org">python.org</a> or your Linux distribution's package manager.
27 | 
28 | 2) Download or clone this repository
29 | 
30 | 3) From the main directory (the one with 1_Introduction, 2_Perceptron, etc.) type:
31 | 
32 | pip install -e tfs
33 | 
34 | This command should install the code for this workshop plus the dependencies numpy and matplotlib.  The -e is optional, but installs the package with links so that if you update your github clone it will automatically update the installed module as well. Tensorflow is NOT installed as a dependency in this step because it is not available through pip for all systems.
35 | 
36 | 4) Install tensorflow by following the platform specific instructions for your system at https://www.tensorflow.org/install/.  Note that if you are using a Mac, pip install tensorflow will not work... you will have to use the alternate method.
37 | 
38 | 5) if you change into the directory 2_Perceptron and type:
39 | 
40 | python perceptronMNIST.py
41 | 
42 | you should see a series of lines printed similar to "Accuracy at step 100: train: 0.77999997139".
43 | 
44 | 
45 | 
46 | The CC-BY-ND license applies to the workshop notes only, not the code.  The code is licensed under the MIT three-clause license (included here at in each source file) where applicable.  The code in the file "input_data.py" found in Session1 is redistributed verbatim from <a href="www.tensorflow.com">Google's TensorFlow tutorial</a> where it appears to be unlicensed.  This code appears to originate with <a href="http://yann.lecun.com/exdb/mnist/">Yann LeCun</a>.
47 | 
48 | <a rel="license" href="http://creativecommons.org/licenses/by-nd/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by-nd/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nd/4.0/">Creative Commons Attribution-NoDerivatives 4.0 International License</a>.
49 | 
50 | 
51 | The MIT License (MIT)
52 | 
53 | Copyright (c) 2015 <a href="www.robbtech.com">Robert A. Brown</a> (<a href="https://github.com/robb-brown">Github</a>)
54 | 
55 | Permission is hereby granted, free of charge, to any person obtaining a copy
56 | of this software and associated documentation files (the "Software"), to deal
57 | in the Software without restriction, including without limitation the rights
58 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
59 | copies of the Software, and to permit persons to whom the Software is
60 | furnished to do so, subject to the following conditions:
61 | 
62 | The above copyright notice and this permission notice shall be included in all
63 | copies or substantial portions of the Software.
64 | 
65 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
66 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
67 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
68 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
69 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
70 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
71 | SOFTWARE.
72 | 


--------------------------------------------------------------------------------
/tfs/setup.py:
--------------------------------------------------------------------------------
 1 | from setuptools import setup
 2 | 
 3 | setup(name='tfs',
 4 | 	version='0.1',
 5 | 	description='Support Package for the Tensor Flow Deep Learning Seminar',
 6 | 	url='http://github.com/robb-brown/DeepLearning',
 7 | 	author='Robert A. Brown',
 8 | 	author_email='robert.brown@mcgill.ca',
 9 | 	license='MIT',
10 | 	packages=['tfs'],
11 | 	install_requires=[
12 | 		'numpy',
13 | 		'matplotlib',
14 | 		'nibabel',
15 | 	],
16 | 	zip_safe=False)
17 | 
18 | 


--------------------------------------------------------------------------------
/tfs/tfs/TensorFlowInterface.py:
--------------------------------------------------------------------------------
  1 | """The MIT License (MIT)
  2 | 
  3 | Copyright (c) 2016 Robert A. Brown (www.robbtech.com)
  4 | 
  5 | Permission is hereby granted, free of charge, to any person obtaining a copy
  6 | of this software and associated documentation files (the "Software"), to deal
  7 | in the Software without restriction, including without limitation the rights
  8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9 | copies of the Software, and to permit persons to whom the Software is
 10 | furnished to do so, subject to the following conditions:
 11 | 
 12 | The above copyright notice and this permission notice shall be included in all
 13 | copies or substantial portions of the Software.
 14 | 
 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 21 | SOFTWARE.
 22 | """
 23 | 
 24 | 
 25 | import tensorflow as tf
 26 | import math
 27 | import pylab
 28 | import numpy
 29 | import time
 30 | 
 31 | 
 32 | class Base(object):
 33 | 	
 34 | 	def __init__(self,name):
 35 | 		self.name = name
 36 | 		self.setup()
 37 | 	
 38 | 	def setup(self):
 39 | 		pass
 40 | 	
 41 | 	def output(self):
 42 | 		return None
 43 | 
 44 | 
 45 | class Layer(Base):
 46 | 
 47 | 	def __init__(self,inputShape,outputShape,activation,name,initialization=None):
 48 | 		self.inputShape = inputShape
 49 | 		self.outputShape = outputShape
 50 | 		self.activation = activation
 51 | 		self.name = name
 52 | 		if initialization is None:
 53 | 			initialization = 'random'
 54 | 		self.initialization = initialization
 55 | 		
 56 | 		self.setup()
 57 | 
 58 | 	def setup(self):
 59 | 		if self.initialization == 'random':
 60 | 			self.W = tf.Variable(tf.random.truncated_normal([self.inputShape,self.outputShape],stddev= 1.0 / math.sqrt(self.outputShape)),name=self.name+'W')
 61 | 		elif self.initialization == 'zeros':
 62 | 			self.W = tf.Variable(tf.zeros([self.inputShape,self.outputShape]),name=self.name+'W')
 63 | 		self.b = tf.Variable(tf.zeros([self.outputShape]),name=self.name+'b')
 64 | 
 65 | 	def variables(self):
 66 | 		return [self.W,self.b]
 67 | 
 68 | 	def output(self,x):
 69 | 		return self.activation(tf.matmul(x,self.W) + self.b)
 70 | 		
 71 | 
 72 | class Model(Base):
 73 | 
 74 | 	def __init__(self,layers,name):
 75 | 		self.layers = layers
 76 | 		self.name = name
 77 | 		self.setup()
 78 | 	
 79 | 	def variables(self):
 80 | 		variables = []
 81 | 		for layer in self.layers:
 82 | 			variables = variables + layer.variables()
 83 | 		return variables
 84 | 
 85 | 	def output(self, x):
 86 | 		for layer in self.layers:
 87 | 			x = layer.output(x)
 88 | 		return x		
 89 | 
 90 | 
 91 | class CrossEntropyCost(Base):
 92 | 
 93 | 	def output(self,y,y_):
 94 | 		return -tf.reduce_sum(y_*tf.math.log(y))
 95 | 
 96 | 
 97 | class AccuracyMetric(Base):
 98 | 
 99 | 	def output(self,y,y_):
100 | 		correctPrediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
101 | 		accuracy = tf.reduce_mean(tf.cast(correctPrediction,'float'))
102 | 		return accuracy
103 | 
104 | 
105 | class Trainer(Base):
106 | 	
107 | 	def __init__(self,model,cost,metrics,name):
108 | 		self.model = model
109 | 		self.cost = cost
110 | 		self.metrics = metrics
111 | 		self.name = name
112 | 		self.setup()
113 | 		
114 | 	def gradient(self,x,y_):
115 | 		variables = self.model.variables()
116 | 		with tf.GradientTape() as tape:
117 | 			y = self.model.output(x=x)
118 | 			cost = self.cost.output(y=y,y_=y_)
119 | 		gradient = tape.gradient(target=cost,sources=variables)
120 | 		return gradient
121 | 		
122 | 	def train(self,data,iterations=3000,learning_rate=1e-3,batchSize=100):
123 | 		optimizer = tf.optimizers.Adam(learning_rate=learning_rate)
124 | 		variables = self.model.variables()
125 | 		for iteration in range(iterations):
126 | 			x,y_ = data.train.next_batch(batchSize)
127 | 			optimizer.apply_gradients(zip(self.gradient(x,y_),variables))
128 | 			if iteration % 100 == 0:
129 | 				y = self.model.output(x=x)
130 | 				print(f'Training metrics at step {iteration}:')
131 | 				print(f'  {self.cost.name} = {self.cost.output(y=y,y_=y_)}')
132 | 				for metric in self.metrics:
133 | 					print(f'  {metric.name} = {metric.output(y=y,y_=y_)}')
134 | 
135 | 
136 | 
137 | 
138 | 
139 | 
140 | def weightVariable(shape,std=1.0,name=None):
141 | 	# Create a set of weights initialized with truncated normal random values
142 | 	name = 'weights' if name is None else name
143 | 	return tf.get_variable(name,shape,initializer=tf.truncated_normal_initializer(stddev=std/math.sqrt(shape[0])))
144 | 
145 | def biasVariable(shape,bias=0.1,name=None):
146 | 	# create a set of bias nodes initialized with a constant 0.1
147 | 	name = 'biases' if name is None else name
148 | 	return tf.get_variable(name,shape,initializer=tf.constant_initializer(bias))
149 | 
150 | def conv2d(x,W,strides=[1,1,1,1],name=None):
151 | 	# return an op that convolves x with W
152 | 	strides = numpy.array(strides)
153 | 	if strides.size == 1:
154 | 		strides = numpy.array([1,strides,strides,1])
155 | 	elif strides.size == 2:
156 | 		strides = numpy.array([1,strides[0],strides[1],1])
157 | 	if numpy.any(strides < 1):
158 | 		strides = numpy.around(1./strides).astype(numpy.uint8)
159 | 		return tf.nn.conv2d_transpose(x,W,strides=strides.tolist(),padding='SAME',name=name)
160 | 	else:
161 | 		return tf.nn.conv2d(x,W,strides=strides.tolist(),padding='SAME',name=name)
162 | 	
163 | 
164 | def conv3d(x,W,strides=1,name=None):
165 | 	# return an op that convolves x with W
166 | 	strides = numpy.array(strides)
167 | 	if strides.size == 1:
168 | 		strides = numpy.array([1,strides,strides,strides[0],1])
169 | 	elif strides.size == 3:
170 | 		strides = numpy.array([1,strides[0],strides[1],strides[2],1])
171 | 	if numpy.any(strides < 1):
172 | 		strides = numpy.around(1./strides).astype(numpy.uint8)
173 | 		return tf.nn.conv3d_transpose(x,W,strides=strides.tolist(),padding='SAME',name=name)
174 | 	else:
175 | 		return tf.nn.conv3d(x,W,strides=strides.tolist(),padding='SAME',name=name)
176 | 
177 | def max_pool_2x2(x,name=None):
178 | 	# return an op that performs max pooling across a 2D image
179 | 	return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME',name=name)
180 | 
181 | def max_pool(x,shape,name=None):
182 | 	# return an op that performs max pooling across a 2D image
183 | 	return tf.nn.max_pool(x,ksize=[1]+shape+[1],strides=[1]+shape+[1],padding='SAME',name=name)
184 | 
185 | def max_pool3d(x,shape,name=None):
186 | 	# return an op that performs max pooling across a 2D image
187 | 	return tf.nn.max_pool3d(x,ksize=[1]+shape+[1],strides=[1]+shape+[1],padding='SAME',name=name)
188 | 	
189 | 	
190 | def plotFields(layer,fieldShape=None,channel=None,figOffset=1,cmap=None,padding=0.01):
191 | 	# Receptive Fields Summary
192 | 	try:
193 | 		W = layer.W
194 | 	except:
195 | 		W = layer
196 | 	wp = W.eval().transpose();
197 | 	if len(numpy.shape(wp)) < 4:		# Fully connected layer, has no shape
198 | 		fields = numpy.reshape(wp,list(wp.shape[0:-1])+fieldShape)	
199 | 	else:			# Convolutional layer already has shape
200 | 		features, channels, iy, ix = numpy.shape(wp)
201 | 		if channel is not None:
202 | 			fields = wp[:,channel,:,:]
203 | 		else:
204 | 			fields = numpy.reshape(wp,[features*channels,iy,ix])
205 | 
206 | 	perRow = int(math.floor(math.sqrt(fields.shape[0])))
207 | 	perColumn = int(math.ceil(fields.shape[0]/float(perRow)))
208 | 
209 | 	fig = pylab.figure(figOffset); pylab.clf()
210 | 	
211 | 	# Using image grid
212 | 	from mpl_toolkits.axes_grid1 import ImageGrid
213 | 	grid = ImageGrid(fig,111,nrows_ncols=(perRow,perColumn),axes_pad=padding,cbar_mode='single')
214 | 	for i in range(0,numpy.shape(fields)[0]):
215 | 		im = grid[i].imshow(fields[i],cmap=cmap); 
216 | 
217 | 	grid.cbar_axes[0].colorbar(im)
218 | 	pylab.title('%s Receptive Fields' % layer.name)
219 | 	
220 | 	# old way
221 | 	# fields2 = numpy.vstack([fields,numpy.zeros([perRow*perColumn-fields.shape[0]] + list(fields.shape[1:]))])
222 | 	# tiled = []
223 | 	# for i in range(0,perColumn*perRow,perColumn):
224 | 	# 	tiled.append(numpy.hstack(fields2[i:i+perColumn]))
225 | 	# 
226 | 	# tiled = numpy.vstack(tiled)
227 | 	# pylab.figure(figOffset); pylab.clf(); pylab.imshow(tiled,cmap=cmap); pylab.title('%s Receptive Fields' % layer.name); pylab.colorbar();
228 | 	pylab.figure(figOffset+1); pylab.clf(); pylab.imshow(numpy.sum(numpy.abs(fields),0),cmap=cmap); pylab.title('%s Total Absolute Input Dependency' % layer.name); pylab.colorbar()
229 | 
230 | 
231 | 
232 | def plotOutput(layer,feed_dict,fieldShape=None,channel=None,figOffset=1,cmap=None):
233 | 	# Output summary
234 | 	try:
235 | 		W = layer.output
236 | 	except:
237 | 		W = layer
238 | 	wp = W.eval(feed_dict=feed_dict);
239 | 	if len(numpy.shape(wp)) < 4:		# Fully connected layer, has no shape
240 | 		temp = numpy.zeros(numpy.product(fieldShape)); temp[0:numpy.shape(wp.ravel())[0]] = wp.ravel()
241 | 		fields = numpy.reshape(temp,[1]+fieldShape)
242 | 	else:			# Convolutional layer already has shape
243 | 		wp = numpy.rollaxis(wp,3,0)
244 | 		features, channels, iy,ix = numpy.shape(wp)
245 | 		if channel is not None:
246 | 			fields = wp[:,channel,:,:]
247 | 		else:
248 | 			fields = numpy.reshape(wp,[features*channels,iy,ix])
249 | 
250 | 	perRow = int(math.floor(math.sqrt(fields.shape[0])))
251 | 	perColumn = int(math.ceil(fields.shape[0]/float(perRow)))
252 | 	fields2 = numpy.vstack([fields,numpy.zeros([perRow*perColumn-fields.shape[0]] + list(fields.shape[1:]))])
253 | 	tiled = []
254 | 	for i in range(0,perColumn*perRow,perColumn):
255 | 		tiled.append(numpy.hstack(fields2[i:i+perColumn]))
256 | 
257 | 	tiled = numpy.vstack(tiled)
258 | 	if figOffset is not None:
259 | 		pylab.figure(figOffset); pylab.clf(); 
260 | 
261 | 	pylab.imshow(tiled,cmap=cmap); pylab.title('%s Output' % layer.name); pylab.colorbar();
262 | 
263 | 
264 | 
265 | def train(session,trainingData,testingData,input,truth,cost,trainingStep,accuracy,iterations=5000,miniBatch=100,trainDict={},testDict=None,logName=None,initialize=True,addSummaryOps=True):
266 | 	testDict = trainDict if testDict is None else testDict
267 | 	
268 | 	if addSummaryOps:
269 | 		costSummary = tf.summary.scalar("Cost Function", cost)
270 | 		if accuracy is None:
271 | 			accuracy = cost
272 | 		accuracySummary = tf.summary.scalar("accuracy", accuracy)
273 | 		mergedSummary = tf.summary.merge_all()
274 | 		if logName is not None:
275 | 			writer = tf.train.SummaryWriter(logName, session.graph_def)
276 | 
277 | 	if initialize:
278 | 		tf.global_variables_initializer().run()		# Take initial values and actually put them in variables
279 | 
280 | 	lastTime = 0; lastIterations = 0
281 | 	print("Doing {} iterations".format(iterations))
282 | 	for i in range(iterations):						# Do some training
283 | 		batch = trainingData.next_batch(miniBatch)
284 | 		if (i%100 == 0) or (time.time()-lastTime > 5):
285 | 			
286 | 			testDict.update({input:batch[0],truth:batch[1]})
287 | #			trainAccuracy = accuracy.eval(feed_dict=testDict)
288 | 
289 | 			# Test accuracy for TensorBoard
290 | #			testDict.update({input:testingData.images,truth:testingData.labels})
291 | 			if addSummaryOps:
292 | 				summary,testAccuracy,testCost = session.run([mergedSummary,accuracy,cost],feed_dict=testDict)
293 | 				if logName is not None:
294 | 					writer.add_summary(summary,i)
295 | 			else:
296 | 				testAccuracy,testCost = session.run([accuracy,cost],feed_dict=testDict)[0]
297 | 
298 | 			print('At batch {}: accuracy: {} cost: {} ({} samples/s)'.format(i,testAccuracy,testCost,(i-lastIterations)/(time.time()-lastTime)*miniBatch))
299 | 			lastTime = time.time(); lastIterations = i
300 | 
301 | 		trainDict.update({input:batch[0],truth:batch[1]})
302 | 		trainingStep.run(feed_dict=trainDict)
303 | 
304 | 	try:
305 | 		# Only works with mnist-type data object
306 | 		testDict.update({input:testingData.images, truth:testingData.labels})
307 | 		print('Test accuracy: {}'.format(accuracy.eval(feed_dict=testDict)))
308 | 	except:
309 | 		pass
310 | 	
311 | 
312 | 
313 | 


--------------------------------------------------------------------------------
/tfs/tfs/__init__.py:
--------------------------------------------------------------------------------
1 | import sys
2 | if sys.version_info[0] >= 3:
3 | 	from .TensorFlowInterface import *
4 | 	from . import input_data
5 | else:
6 | 	from TensorFlowInterface import *
7 | 	import input_data
8 | 	


--------------------------------------------------------------------------------
/tfs/tfs/input_data.py:
--------------------------------------------------------------------------------
  1 | """Functions for downloading and reading MNIST data."""
  2 | from __future__ import absolute_import
  3 | from __future__ import division
  4 | from __future__ import print_function
  5 | 
  6 | import gzip
  7 | import os
  8 | 
  9 | import numpy
 10 | from six.moves import urllib
 11 | from six.moves import xrange  # pylint: disable=redefined-builtin
 12 | 
 13 | import traceback
 14 | 
 15 | SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'
 16 | 
 17 | 
 18 | def maybe_download(filename, work_directory):
 19 |   """Download the data from Yann's website, unless it's already here."""
 20 |   if not os.path.exists(work_directory):
 21 |     os.mkdir(work_directory)
 22 |   filepath = os.path.join(work_directory, filename)
 23 |   if not os.path.exists(filepath):
 24 |     filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath)
 25 |     statinfo = os.stat(filepath)
 26 |     print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')
 27 |   return filepath
 28 | 
 29 | 
 30 | def _read32(bytestream):
 31 |   dt = numpy.dtype(numpy.uint32).newbyteorder('>')
 32 |   return numpy.frombuffer(bytestream.read(4), dtype=dt)[0]
 33 | 
 34 | 
 35 | def extract_images(filename):
 36 |   """Extract the images into a 4D uint8 numpy array [index, y, x, depth]."""
 37 |   print('Extracting', filename)
 38 |   with gzip.open(filename) as bytestream:
 39 |     magic = _read32(bytestream)
 40 |     if magic != 2051:
 41 |       raise ValueError(
 42 |           'Invalid magic number %d in MNIST image file: %s' %
 43 |           (magic, filename))
 44 |     num_images = _read32(bytestream)
 45 |     rows = _read32(bytestream)
 46 |     cols = _read32(bytestream)
 47 |     buf = bytestream.read(rows * cols * num_images)
 48 |     data = numpy.frombuffer(buf, dtype=numpy.uint8)
 49 |     data = data.reshape(num_images, rows, cols, 1)
 50 |     return data
 51 | 
 52 | 
 53 | def dense_to_one_hot(labels_dense, num_classes=10):
 54 |   """Convert class labels from scalars to one-hot vectors."""
 55 |   num_labels = labels_dense.shape[0]
 56 |   index_offset = numpy.arange(num_labels) * num_classes
 57 |   labels_one_hot = numpy.zeros((num_labels, num_classes))
 58 |   labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1
 59 |   return labels_one_hot
 60 | 
 61 | 
 62 | def extract_labels(filename, one_hot=False):
 63 |   """Extract the labels into a 1D uint8 numpy array [index]."""
 64 |   print('Extracting', filename)
 65 |   with gzip.open(filename) as bytestream:
 66 |     magic = _read32(bytestream)
 67 |     if magic != 2049:
 68 |       raise ValueError(
 69 |           'Invalid magic number %d in MNIST label file: %s' %
 70 |           (magic, filename))
 71 |     num_items = _read32(bytestream)
 72 |     buf = bytestream.read(num_items)
 73 |     labels = numpy.frombuffer(buf, dtype=numpy.uint8)
 74 |     if one_hot:
 75 |       return dense_to_one_hot(labels)
 76 |     return labels
 77 | 
 78 | 
 79 | class DataSet(object):
 80 | 
 81 |   def __init__(self, images, labels, fake_data=False):
 82 |     if fake_data:
 83 |       self._num_examples = 10000
 84 |     else:
 85 |       assert images.shape[0] == labels.shape[0], (
 86 |           "images.shape: %s labels.shape: %s" % (images.shape,
 87 |                                                  labels.shape))
 88 |       self._num_examples = images.shape[0]
 89 | 
 90 |       # Convert shape from [num examples, rows, columns, depth]
 91 |       # to [num examples, rows*columns] (assuming depth == 1)
 92 |       self.imageShape = images.shape[1:]
 93 |       self.imageChannels = self.imageShape[2]
 94 | 
 95 |       images = images.reshape(images.shape[0],
 96 |                               images.shape[1] * images.shape[2] * images.shape[3])
 97 |       # Convert from [0, 255] -> [0.0, 1.0].
 98 |       images = images.astype(numpy.float32)
 99 |       images = numpy.multiply(images, 1.0 / 255.0)
100 |     self._images = images
101 |     self._labels = labels
102 |     try:
103 |       if len(numpy.shape(self._labels)) == 1:
104 |         self._labels = dense_to_one_hot(self._labels,len(numpy.unique(self._labels)))
105 |     except:
106 |       traceback.print_exc()
107 |     self._epochs_completed = 0
108 |     self._index_in_epoch = 0
109 | 
110 |   @property
111 |   def images(self):
112 |     return self._images
113 | 
114 |   @property
115 |   def labels(self):
116 |     return self._labels
117 | 
118 |   @property
119 |   def num_examples(self):
120 |     return self._num_examples
121 | 
122 |   @property
123 |   def epochs_completed(self):
124 |     return self._epochs_completed
125 | 
126 |   def next_batch(self, batch_size, fake_data=False):
127 |     """Return the next `batch_size` examples from this data set."""
128 |     if fake_data:
129 |       fake_image = [1.0 for _ in xrange(784)]
130 |       fake_label = 0
131 |       return [fake_image for _ in xrange(batch_size)], [
132 |           fake_label for _ in xrange(batch_size)]
133 |     start = self._index_in_epoch
134 |     self._index_in_epoch += batch_size
135 |     if self._index_in_epoch > self._num_examples:
136 |       # Finished epoch
137 |       self._epochs_completed += 1
138 |       # Shuffle the data
139 |       perm = numpy.arange(self._num_examples)
140 |       numpy.random.shuffle(perm)
141 |       self._images = self._images[perm]
142 |       self._labels = self._labels[perm]
143 |       # Start next epoch
144 |       start = 0
145 |       self._index_in_epoch = batch_size
146 |       assert batch_size <= self._num_examples
147 |     end = self._index_in_epoch
148 |     return self._images[start:end], self._labels[start:end]
149 | 
150 | 
151 | def read_data_sets(train_dir, fake_data=False, one_hot=False):
152 |   class DataSets(object):
153 |     pass
154 |   data_sets = DataSets()
155 | 
156 |   if fake_data:
157 |     data_sets.train = DataSet([], [], fake_data=True)
158 |     data_sets.validation = DataSet([], [], fake_data=True)
159 |     data_sets.test = DataSet([], [], fake_data=True)
160 |     return data_sets
161 | 
162 |   TRAIN_IMAGES = 'train-images-idx3-ubyte.gz'
163 |   TRAIN_LABELS = 'train-labels-idx1-ubyte.gz'
164 |   TEST_IMAGES = 't10k-images-idx3-ubyte.gz'
165 |   TEST_LABELS = 't10k-labels-idx1-ubyte.gz'
166 |   VALIDATION_SIZE = 5000
167 | 
168 |   local_file = maybe_download(TRAIN_IMAGES, train_dir)
169 |   train_images = extract_images(local_file)
170 | 
171 |   local_file = maybe_download(TRAIN_LABELS, train_dir)
172 |   train_labels = extract_labels(local_file, one_hot=one_hot)
173 | 
174 |   local_file = maybe_download(TEST_IMAGES, train_dir)
175 |   test_images = extract_images(local_file)
176 | 
177 |   local_file = maybe_download(TEST_LABELS, train_dir)
178 |   test_labels = extract_labels(local_file, one_hot=one_hot)
179 | 
180 |   validation_images = train_images[:VALIDATION_SIZE]
181 |   validation_labels = train_labels[:VALIDATION_SIZE]
182 |   train_images = train_images[VALIDATION_SIZE:]
183 |   train_labels = train_labels[VALIDATION_SIZE:]
184 | 
185 |   data_sets.train = DataSet(train_images, train_labels)
186 |   data_sets.validation = DataSet(validation_images, validation_labels)
187 |   data_sets.test = DataSet(test_images, test_labels)
188 | 
189 |   return data_sets
190 | 


--------------------------------------------------------------------------------