├── 1-custom_models_layers_losses ├── praticed_py_files │ ├── 1_tf_functional.py │ ├── 2_multi_output.py │ ├── 3_intro_siamese.py │ ├── 4_custom_losses.py │ ├── 5_custom_class_loss.py │ ├── 6_lambda_layers.py │ ├── 7_custom_layer.py │ ├── 8_resnets_demo.py │ └── 9_resnet_demo2.py ├── week-1 │ ├── C1W1_Assignment.ipynb │ ├── C1_W1_Lab_1_functional-practice.ipynb │ ├── C1_W1_Lab_2_multi-output.ipynb │ ├── C1_W1_Lab_3_siamese-network.ipynb │ ├── base-model.png │ ├── model.png │ ├── outer-model.png │ └── utf-8''utils.py ├── week-2 │ ├── C1W2_Assignment.ipynb │ ├── C1_W2_Lab_1_huber-loss.ipynb │ ├── C1_W2_Lab_2_huber-object-loss.ipynb │ └── utils.py ├── week-3 │ ├── C1W3_Assignment.ipynb │ ├── C1_W3_Lab_1_lambda-layer.ipynb │ ├── C1_W3_Lab_2_custom-dense-layer.ipynb │ ├── C1_W3_Lab_3_custom-layer-activation.ipynb │ └── utils.py └── week-4 │ ├── C1_W4_Lab_1_basic-model.ipynb │ ├── C1_W4_Lab_2_resnet-example.ipynb │ └── data │ └── mnist │ └── 3.0.1 │ ├── dataset_info.json │ ├── image.image.json │ └── mnist-test.tfrecord-00000-of-00001 ├── 2-custom_and_distributed_training ├── pratice_py │ ├── 1_tf_var_layer.py │ ├── 2_gradient_tape.py │ ├── 3_gradient_calcuation.py │ ├── 4_into_customloop.py │ └── 5_custom_1.py ├── week-1 │ ├── C2W1_Assignment.ipynb │ ├── C2_W1_Lab_1_basic-tensors.ipynb │ └── C2_W1_Lab_2_gradient-tape-basics.ipynb ├── week-2 │ ├── C2W2_Assignment.ipynb │ ├── C2_W2_Lab_1_training-basics.ipynb │ ├── C2_W2_Lab_2_training-categorical.ipynb │ └── data │ │ └── fashion_mnist │ │ └── 3.0.1 │ │ ├── dataset_info.json │ │ └── fashion_mnist-test.tfrecord-00000-of-00001 ├── week-3 │ ├── C2W3_Assignment.ipynb │ ├── C2_W3_Lab_1_autograph-basics.ipynb │ ├── C2_W3_Lab_2-graphs-for-complex-code.ipynb │ └── data │ │ └── horses_or_humans │ │ └── 3.0.0 │ │ └── dataset_info.json └── week-4 │ ├── C2W4_Assignment.ipynb │ ├── C2_W4_Lab_1_basic-mirrored-strategy.ipynb │ ├── C2_W4_Lab_2_multi-GPU-mirrored-strategy.ipynb │ ├── C2_W4_Lab_3_using-TPU-strategy.ipynb │ ├── C2_W4_Lab_4_one-device-strategy.ipynb │ └── data │ └── mnist │ └── 3.0.1 │ ├── dataset_info.json │ ├── image.image.json │ └── mnist-test.tfrecord-00000-of-00001 ├── 3-Advance_computer_vision ├── week -1 │ ├── C3W1_Assignment.ipynb │ ├── C3_W1_Lab_1_transfer_learning_cats_dogs.ipynb │ ├── C3_W1_Lab_2_Transfer_Learning_CIFAR_10.ipynb │ └── C3_W1_Lab_3_Object_Localization.ipynb ├── week-2 │ ├── C3_W2_Lab_1_Simple_Object_Detection.ipynb │ ├── C3_W2_Lab_2_Object_Detection (1).ipynb │ ├── Copy_of_C3W2_Assignment.ipynb │ ├── interactive_eager_few_shot_od_training_colab.ipynb │ └── results.data ├── week-3 │ ├── C3W3_Assignment.ipynb │ ├── C3_W3_Lab_1_VGG16_FCN8_CamVid.ipynb │ ├── C3_W3_Lab_2_OxfordPets_UNet.ipynb │ └── C3_W3_Lab_3_Mask_RCNN_ImageSegmentation.ipynb └── week-4 │ ├── C3W4_Assignment.ipynb │ ├── C3_W4_Lab_1_FashionMNIST_CAM.ipynb │ ├── C3_W4_Lab_2_CatsDogs_CAM.ipynb │ ├── C3_W4_Lab_3_Saliency.ipynb │ └── C3_W4_Lab_4_GradCam.ipynb ├── 4-Generative-deeplearning-with-tensorflow ├── week-1 │ ├── C4W1_Assignment.ipynb │ ├── C4_W1_Lab_1_Neural_Style_Transfer.ipynb │ ├── C4_W1_Lab_2_Fast_NST.ipynb │ └── puppy.png ├── week-2 │ ├── C4W2_Assignment.ipynb │ ├── C4_W2_Lab_1_FirstAutoEncoder.ipynb │ ├── C4_W2_Lab_2_MNIST_Autoencoder.ipynb │ ├── C4_W2_Lab_3_MNIST_DeepAutoencoder.ipynb │ ├── C4_W2_Lab_4_FashionMNIST_CNNAutoEncoder.ipynb │ ├── C4_W2_Lab_5_FashionMNIST_NoisyCNNAutoEncoder.ipynb │ └── mymodel.h5 ├── week-3 │ ├── C4W3_Assignment.ipynb │ ├── C4_W3_Lab_1_VAE_MNIST.ipynb │ └── cvae-tutorial.ipynb └── week-4 │ ├── C4W4_Assignment.ipynb │ ├── C4_W4_Lab_1_First_GAN.ipynb │ ├── C4_W4_Lab_2_First_DCGAN.ipynb │ ├── C4_W4_Lab_3_CelebA_GAN_Experiments.ipynb │ └── my-signs.zip └── README.md /1-custom_models_layers_losses/praticed_py_files/1_tf_functional.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.models import Model 3 | from tensorflow.keras.layers import Input, Flatten, Dense, Conv2D, MaxPooling2D 4 | 5 | 6 | mnist = tf.keras.datasets.mnist 7 | 8 | (train_images, train_labels), (test_images, test_labels) = mnist.load_data() 9 | 10 | train_images = train_images / 255.0 11 | test_images = test_images / 255.0 12 | 13 | 14 | 15 | def model() : 16 | 17 | input_layer = Input(shape = (28, 28)) 18 | 19 | 20 | flaten = Flatten()(input_layer) 21 | dense_1 = Dense(128, activation='relu')(flaten) 22 | dense_2 = Dense(54, activation='relu')(dense_1) 23 | output_layer = Dense(10, activation='softmax')(dense_2) 24 | 25 | model = Model(inputs = input_layer, outputs = output_layer) 26 | return model 27 | 28 | 29 | 30 | model = model() 31 | 32 | model.compile(optimizer=tf.optimizers.Adam(), 33 | loss='sparse_categorical_crossentropy', 34 | metrics=['accuracy']) 35 | 36 | 37 | model.fit(train_images, train_labels, epochs = 5) 38 | 39 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/praticed_py_files/2_multi_output.py: -------------------------------------------------------------------------------- 1 | # THIS IS SAMPLE CODE 2 | # DON'T RUN IT 3 | 4 | import tensorflow as tf 5 | from tensorflow.keras.models import Model 6 | from tensorflow.keras.layers import Flatten, Dense , Input 7 | 8 | input_layer = Input( shape = (None, None)) 9 | 10 | layer_1 = Dense(64, activation= 'relu', name ='deses_1')(input_layer) 11 | layer_2 = Dense(32, activation= tf.nn.tanh(), name = 'dense_2')(layer_1) 12 | 13 | 14 | y_1_output = Dense(1, activation=tf.nn.sigmoid())(layer_2) 15 | 16 | 17 | layer_3 = Dense( 128, activation= tf.nn.relu())(layer_2) 18 | 19 | y_2_output = Dense(1, activation=tf.nn.sigmoid())(layer_3) 20 | 21 | model = Model(inputs = input_layer, outputs = [y_1_output, y_2_output]) 22 | 23 | # compile the model 24 | 25 | model.compile(optimizer = tf.keras.optimizers.Adam(), 26 | loss = { 'y_1_output' : 'mse', 27 | 'y_2_output': "mae" 28 | }, 29 | metircs = { 'y_1_output' : tf.keras.metrics.RootMeanSquaredError() , 30 | 'y_2_output' : tf.keras.metircs.RootMeanSquaredError() 31 | } 32 | ) 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/praticed_py_files/3_intro_siamese.py: -------------------------------------------------------------------------------- 1 | # don't run code 2 | 3 | import tensorflow as tf 4 | from tensorflow.keras.layers import Dense, Flatten, Lambda , Input 5 | from tensorflow.keras.models import Model 6 | from tensorflow.keras import backend as K 7 | from tensorflow.keras.utils import plot_model 8 | 9 | def base_model() : 10 | 11 | inputs = Input(shape = (28, 28), name = 'input_layer') 12 | x = Flatten(name = 'Flatten_layer')(inputs) 13 | x = Dense(64, activation= 'relu')(x) 14 | x = Dense(32, activation= 'relu')(x) 15 | 16 | return Model(inputs = inputs, outputs = x) 17 | 18 | 19 | def euclidean_distance(vects): 20 | x, y = vects 21 | sum_square = K.sum(K.square(x - y), axis=1, keepdims=True) 22 | return K.sqrt(K.maximum(sum_square, K.epsilon())) 23 | 24 | 25 | def eucl_dist_output_shape(shapes): 26 | shape1, shape2 = shapes 27 | return (shape1[0], 1) 28 | 29 | 30 | # initialize the base network 31 | 32 | base_network = base_model() 33 | 34 | # create the left input and point to the base network 35 | 36 | input_a = Input(shape=(28,28,), name="left_input") 37 | vect_output_a = base_network(input_a) 38 | 39 | # create the right input and point to the base network 40 | 41 | input_b = Input(shape=(28,28,), name="right_input") 42 | vect_output_b = base_network(input_b) 43 | 44 | # measure the similarity of the two vector outputs 45 | # lambda layer gives flexibility for custom layers 46 | 47 | output = Lambda(euclidean_distance, name="output_layer", 48 | output_shape=eucl_dist_output_shape)([vect_output_a, vect_output_b]) 49 | 50 | # specify the inputs and output of the model 51 | model = Model([input_a, input_b], output) 52 | 53 | plot_model(model, to_file = 'siamese.png', show_shapes= True, show_layer_names=True) 54 | 55 | 56 | #loss for siamese network 57 | def contrastive_loss_with_margin(margin): 58 | def contrastive_loss(y_true, y_pred): 59 | '''Contrastive loss from Hadsell-et-al.'06 60 | http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf 61 | ''' 62 | square_pred = K.square(y_pred) 63 | margin_square = K.square(K.maximum(margin - y_pred, 0)) 64 | return K.mean(y_true * square_pred + (1 - y_true) * margin_square) 65 | return contrastive_loss -------------------------------------------------------------------------------- /1-custom_models_layers_losses/praticed_py_files/4_custom_losses.py: -------------------------------------------------------------------------------- 1 | # DON'T RUN THIS CODE 2 | 3 | import tensorflow as tf 4 | from tensorflow import keras 5 | 6 | # code for custom huber loss 7 | 8 | def my_huber_loss(y_true, y_pred): 9 | 10 | threshold = 1 11 | error = y_true - y_pred 12 | is_small_error = tf.abs(error) <= threshold 13 | small_error_loss = tf.square(error) / 2 14 | big_error_loss = threshold * (tf.abs(error) - (0.5 * threshold)) 15 | return tf.where(is_small_error, small_error_loss, big_error_loss) 16 | 17 | 18 | model = tf.keras.models.Sequential([ tf.keras.layers.Dense(20, activation='relu')]) 19 | 20 | model.compile(optimizer = 'sgd', loss = my_huber_loss) # using custom loss 21 | 22 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/praticed_py_files/5_custom_class_loss.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow import keras 3 | from tensorflow.keras.losses import Loss 4 | 5 | # custom loss with hyperparameter 6 | 7 | def my_huber_loss_with_threshold(threshold): 8 | 9 | def my_huber_loss(y_true, y_pred) : 10 | error = y_true - y_pred 11 | is_small_error = tf.abs(error) <= threshold 12 | small_error_loss = tf.square(error) / 2 13 | big_error_loss = threshold * (tf.abs(error) - (0.5 * threshold)) 14 | 15 | return tf.where(is_small_error, small_error_loss, big_error_loss) 16 | 17 | # return the inner function tuned by the hyperparameter 18 | return my_huber_loss 19 | 20 | model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])]) 21 | model.compile(optimizer='sgd', loss=my_huber_loss_with_threshold(threshold=1.2)) #calling loss 22 | 23 | 24 | 25 | # CUSTOM LOSS AS A CLASS 26 | 27 | 28 | class MyHuberLoss(Loss) : 29 | 30 | threshold = 1 31 | 32 | def __init__(self, threshold): 33 | super().__init__() 34 | self.threshold = threshold 35 | 36 | def call(self, y_true, y_pred) : 37 | error = y_true - y_pred 38 | is_small_error = tf.abs(error) <= self.threshold 39 | small_error_loss = tf.square(error) / 2 40 | big_error_loss = self.threshold * (tf.abs(error) - (0.5 * self.threshold)) 41 | return tf.where(is_small_error, small_error_loss, big_error_loss) 42 | 43 | 44 | 45 | model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])]) 46 | model.compile(optimizer='sgd', loss=MyHuberLoss(threshold=1.02)) #calling loss 47 | 48 | 49 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/praticed_py_files/6_lambda_layers.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras import backend as K 3 | 4 | 5 | 6 | model = tf.keras.models.Sequential([ 7 | 8 | tf.keras.layers.Flatten() , 9 | tf.keras.layers.Dense(128) , 10 | tf.keras.layers.Lambda(lambda x : tf.abs(x)) , 11 | # it takes values from previous layer and take absolute values of x 12 | 13 | 14 | tf.keras.layers.Dense(10, activation= 'softmax') 15 | ]) 16 | 17 | 18 | 19 | #custom relu function 20 | def my_relu(x): 21 | return K.maximum(-0.1, x) 22 | 23 | 24 | model = tf.keras.models.Sequential([ 25 | tf.keras.layers.Flatten(), 26 | tf.keras.layers.Dense(128) , 27 | tf.keras.layers.Lambda(my_relu) , 28 | tf.keras.layers.Dense(10, activation='softmax') 29 | 30 | ]) 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/praticed_py_files/7_custom_layer.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Layer 3 | 4 | 5 | class MyDense(Layer) : 6 | 7 | def __init__(self, units = 32, activation = None) : 8 | super().__init__() 9 | self.units = units 10 | self.activation = tf.keras.activations.get(activation) 11 | 12 | def build(self, input_shape) : # create the state of the layer weights 13 | 14 | w_init = tf.random_normal_initializer() 15 | 16 | self.w = tf.Variable(name = 'kernal', 17 | initial_value= w_init(shape = (input_shape[-1], self.units), 18 | dtype= 'float32') , trainable= True) 19 | 20 | b_init = tf.zeros_initializer() 21 | 22 | self.b = tf.Variable(name = 'bias', 23 | initial_value= b_init(shape = (self.units,), 24 | dtype= 'float32'), trainable= True) 25 | 26 | super().build(input_shape) 27 | 28 | def call(self, inputs) : # defines the computation from inputs to outputs 29 | return self.activation(tf.matmul(inputs, self.w) + self.b) 30 | 31 | 32 | 33 | 34 | 35 | my_dense = MyDense(units= 1) 36 | 37 | x = tf.ones((1,1)) 38 | y = my_dense(x) 39 | 40 | print(my_dense.variables) -------------------------------------------------------------------------------- /1-custom_models_layers_losses/praticed_py_files/8_resnets_demo.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, Conv2D, Input, Layer 3 | from tensorflow.keras.models import Model 4 | 5 | 6 | class CNNResidual(Model) : 7 | 8 | def __init__(self, layers, filters, **kwargs) : 9 | super(CNNResidual, self).__init__(**kwargs) 10 | 11 | self.HiddenCNN = [Conv2D(filters, (3,3), activation='relu') for _ in range(layers)] 12 | 13 | 14 | 15 | def call(self, inputs) : 16 | x = inputs 17 | for layer in self.HiddenCNN: 18 | x = layer(x) 19 | return inputs + x 20 | 21 | 22 | class DNNResidual(Model) : 23 | 24 | def __init__(self, layers, neurons, **kwargs): 25 | super(DNNResidual, self).__init__(**kwargs) 26 | 27 | self.HiddenDNN = [Dense(neurons, activation='relu') for _ in range(layers)] 28 | 29 | 30 | def call(self, inputs) : 31 | x = inputs 32 | for layer in self.HiddenDNN : 33 | x = layer(x) 34 | return inputs + x 35 | 36 | 37 | class MyResidual(Model) : 38 | 39 | def __init__(self, **kwargs) : 40 | super(MyResidual, self).__init__(self) 41 | 42 | self.hidden1 = Dense(60, activation= 'relu') 43 | self.block1 = CNNResidual(3, 64) 44 | self.block2 = DNNResidual(3, 64) 45 | self.out = Dense(1) 46 | 47 | 48 | def call(self, inputs) : 49 | 50 | x = self.hidden1(inputs) 51 | x = self.block1(x) 52 | for _ in range(1, 4): 53 | x = self.block2(x) 54 | 55 | return self.out(x) 56 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/praticed_py_files/9_resnet_demo2.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, Conv2D, Input, Layer ,Add ,BatchNormalization, Activation 3 | 4 | from tensorflow.keras.models import Model 5 | 6 | class IdentityBlock(Model) : 7 | def __init__(self, filters, kernal_size, **kwargs) : 8 | super(IdentityBlock, self).__init__(**kwargs) 9 | 10 | self.conv1 = Conv2D(filters, kernal_size, padding='same') 11 | self.bn1 = BatchNormalization() 12 | 13 | self.act = Activation('relu') 14 | 15 | self.add = Add() 16 | 17 | 18 | def call(self, inputs) : 19 | 20 | x = self.conv1(inputs) 21 | x = self.bn1(x) 22 | x = self.act(x) 23 | 24 | x = self.conv1(x) 25 | x = self.bn1(x) 26 | x = self.act(x) 27 | 28 | 29 | x = self.add([x, inputs]) 30 | x = self.act(x) 31 | 32 | return x 33 | 34 | 35 | 36 | class ResNet(Model) : 37 | 38 | def __init__(self, num_classes, **kwargs) : 39 | super(ResNet, self).__init__(**kwargs) 40 | 41 | self.conv = Conv2D(64, 7, padding= 'same') 42 | self.bn = BatchNormalization() 43 | self.act = Activation('relu') 44 | 45 | self.max_pool = tf.keras.layers.MaxPool2D((3,3)) 46 | 47 | self.idblock1 = IdentityBlock(64, 3) 48 | self.idblock2 = IdentityBlock(64, 3) 49 | 50 | self.glob_pool = tf.keras.layers.GlobalAveragePooling2D() 51 | 52 | self.classifier = Dense(num_classes, activation= 'softmax') 53 | 54 | 55 | def call(self, inputs) : 56 | 57 | x = self.conv(inputs) 58 | x = self.bn(x) 59 | x = self.act(x) 60 | 61 | x = self.max_pool(x) 62 | 63 | x = self.idblock1(x) 64 | x = self.idblock2(x) 65 | 66 | x = self.glob_pool(x) 67 | 68 | x = self.classifier(x) 69 | 70 | return x 71 | 72 | 73 | resnet = ResNet(10) 74 | 75 | 76 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-1/base-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/1-custom_models_layers_losses/week-1/base-model.png -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-1/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/1-custom_models_layers_losses/week-1/model.png -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-1/outer-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/1-custom_models_layers_losses/week-1/outer-model.png -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-1/utf-8''utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from tensorflow.keras.models import Model 4 | from tensorflow.keras.layers import Dense, Input 5 | from sklearn.model_selection import train_test_split 6 | 7 | def test_loop(test_cases): 8 | 9 | success = 0 10 | fails = 0 11 | 12 | for test_case in test_cases: 13 | try: 14 | assert test_case["result"] == test_case["expected"] 15 | success += 1 16 | 17 | except: 18 | fails += 1 19 | print(f'{test_case["name"]}: {test_case["error_message"]}\nExpected: {test_case["expected"]}\nResult: {test_case["result"]}\nPlease open utils.py if you want to see the unit test here.\n') 20 | 21 | if fails == 0: 22 | print("\033[92m All public tests passed") 23 | 24 | else: 25 | print('\033[92m', success," Tests passed") 26 | print('\033[91m', fails, " Tests failed") 27 | raise Exception(test_case["error_message"]) 28 | 29 | def test_white_df(white_df): 30 | 31 | test_cases = [ 32 | { 33 | "name": "type_check", 34 | "result": type(white_df.is_red[0]), 35 | "expected": np.int64, 36 | "error_message": f'white_df.is_red has an incorrect type.' 37 | }, 38 | { 39 | "name": "output_check", 40 | "result": white_df.is_red[0], 41 | "expected": 0, 42 | "error_message": "white_df.is_red is not set correctly" 43 | }, 44 | { 45 | "name": "len_check", 46 | "result": len(white_df), 47 | "expected": 3961, 48 | "error_message": "Number of rows is incorrect. Please drop duplicates." 49 | } 50 | ] 51 | 52 | test_loop(test_cases) 53 | 54 | def test_red_df(red_df): 55 | 56 | test_cases = [ 57 | { 58 | "name": "type_check", 59 | "result": type(red_df.is_red[0]), 60 | "expected": np.int64, 61 | "error_message": f'red_df.is_red has an incorrect type.' 62 | }, 63 | { 64 | "name": "output_check", 65 | "result": red_df.is_red[0], 66 | "expected": 1, 67 | "error_message": "red_df.is_red is not set correctly" 68 | }, 69 | { 70 | "name": "len_check", 71 | "result": len(red_df), 72 | "expected": 1359, 73 | "error_message": "Number of rows is incorrect. Please drop duplicates." 74 | } 75 | ] 76 | 77 | test_loop(test_cases) 78 | 79 | def test_df_drop(df): 80 | 81 | test_cases = [ 82 | { 83 | "name": "df.alcohol[0]_check", 84 | "result": df.alcohol[0], 85 | "expected": 9.4, 86 | "error_message": f'Value is not as expected. Please check quality interval.' 87 | }, 88 | { 89 | "name": "df.alcohol[100]_check", 90 | "result": df.alcohol[100], 91 | "expected": 10.9, 92 | "error_message": f'Value is not as expected. Please check quality interval.' 93 | } 94 | ] 95 | 96 | test_loop(test_cases) 97 | 98 | def test_data_sizes(train_size, test_size, val_size): 99 | 100 | test_cases = [ 101 | { 102 | "name": "train_test_size_check", 103 | "result": train_size > test_size, 104 | "expected": True, 105 | "error_message": f'train.size is too small. Please check implementation.' 106 | }, 107 | { 108 | "name": "train_val_size_check", 109 | "result": train_size > val_size, 110 | "expected": True, 111 | "error_message": f'train.size is too small. Please check implementation.' 112 | }, 113 | { 114 | "name": "test_val_size_check", 115 | "result": test_size > val_size, 116 | "expected": True, 117 | "error_message": f'test.size is too small. Please check implementation.' 118 | } 119 | ] 120 | 121 | test_loop(test_cases) 122 | 123 | def test_format_output(df, train_Y, val_Y, test_Y): 124 | 125 | train, test = train_test_split(df, test_size=0.2, random_state=1) 126 | train, val = train_test_split(train, test_size=0.2, random_state=1) 127 | 128 | test_cases = [ 129 | { 130 | "name": "train_Y[0]_check", 131 | "result": np.all(train_Y[0] == np.array(train.quality)), 132 | "expected": True, 133 | "error_message": f'train_Y[0] is not equal to train.quality. Please check implementation.' 134 | }, 135 | { 136 | "name": "train_Y[1]_check", 137 | "result": np.all(train_Y[1] == np.array(train.is_red)), 138 | "expected": True, 139 | "error_message": f'train_Y[1] is not equal to train.is_red. Please check implementation.' 140 | }, 141 | { 142 | "name": "val_Y[0]_check", 143 | "result": np.all(val_Y[0] == np.array(val.quality)), 144 | "expected": True, 145 | "error_message": f'train_Y[0] is not equal to val.quality. Please check implementation.' 146 | }, 147 | { 148 | "name": "val_Y[1]_check", 149 | "result": np.all(val_Y[1] == np.array(val.is_red)), 150 | "expected": True, 151 | "error_message": f'train_Y[1] is not equal to val.is_red. Please check implementation.' 152 | }, 153 | { 154 | "name": "test_Y[0]_check", 155 | "result": np.all(test_Y[0] == np.array(test.quality)), 156 | "expected": True, 157 | "error_message": f'test_Y[0] is not equal to test.quality. Please check implementation.' 158 | }, 159 | { 160 | "name": "test_Y[1]_check", 161 | "result": np.all(test_Y[1] == np.array(test.is_red)), 162 | "expected": True, 163 | "error_message": f'test_Y[1] is not equal to test.is_red. Please check implementation.' 164 | } 165 | ] 166 | 167 | test_loop(test_cases) 168 | 169 | def test_norm(norm_train_X, norm_val_X, norm_test_X, train, val, test): 170 | 171 | from pandas.core.frame import DataFrame 172 | 173 | test_cases = [ 174 | { 175 | "name": "norm_train_X_type_check", 176 | "result": type(norm_train_X), 177 | "expected": DataFrame, 178 | "error_message": f'norm_train_X has an incorrect type.' 179 | }, 180 | { 181 | "name": "norm_val_X_type_check", 182 | "result": type(norm_val_X), 183 | "expected": DataFrame, 184 | "error_message": f'norm_val_X has an incorrect type.' 185 | }, 186 | { 187 | "name": "norm_test_X_type_check", 188 | "result": type(norm_test_X), 189 | "expected": DataFrame, 190 | "error_message": f'norm_test_X has an incorrect type.' 191 | }, 192 | { 193 | "name": "norm_train_X_length_check", 194 | "result": len(norm_train_X), 195 | "expected": len(train), 196 | "error_message": f'norm_train_X has an incorrect length.' 197 | }, 198 | { 199 | "name": "norm_val_X_length_check", 200 | "result": len(norm_val_X), 201 | "expected": len(val), 202 | "error_message": f'norm_val_X has an incorrect length.' 203 | }, 204 | { 205 | "name": "norm_test_X_length_check", 206 | "result": len(norm_test_X), 207 | "expected": len(test), 208 | "error_message": f'norm_test_X has an incorrect length.' 209 | }, 210 | ] 211 | 212 | test_loop(test_cases) 213 | 214 | def test_base_model(base_model): 215 | 216 | test_inputs = tf.keras.layers.Input(shape=(11,)) 217 | test_output = base_model(test_inputs) 218 | test_model = Model(inputs=test_inputs, outputs=test_output) 219 | 220 | test_cases = [ 221 | { 222 | "name": "return_type_check", 223 | "result": type(test_output), 224 | "expected": tf.Tensor, 225 | "error_message": 'Return type is incorrect. Please check implementation.' 226 | }, 227 | { 228 | "name": "return_shape_check", 229 | "result": str(test_output.shape), 230 | "expected": '(None, 128)', 231 | "error_message": 'Return shape is incorrect. Please check implementation.' 232 | }, 233 | { 234 | "name": "tensor_dtype_check", 235 | "result": str(test_output.dtype), 236 | "expected": "", 237 | "error_message": 'model dtype is incorrect. Please check implementation.' 238 | }, 239 | { 240 | "name": "base_model_num_layers_check", 241 | "result": len(test_model.layers), 242 | "expected": 3, 243 | "error_message": 'There are too many layers. Please check implementation.' 244 | }, 245 | { 246 | "name": "base_model_layer1_check", 247 | "result": type(test_model.layers[-2]), 248 | "expected": Dense, 249 | "error_message": 'First layer type is incorrect. Please check implementation.' 250 | }, 251 | { 252 | "name": "base_model_layer2_check", 253 | "result": type(test_model.layers[-1]), 254 | "expected": Dense, 255 | "error_message": 'Output layer type is incorrect. Please check implementation.' 256 | }, 257 | ] 258 | 259 | test_loop(test_cases) 260 | 261 | def test_final_model(final_model): 262 | 263 | test_inputs = tf.keras.layers.Input(shape=(11,)) 264 | test_output = final_model(test_inputs) 265 | 266 | test_cases = [ 267 | { 268 | "name": "return_type_check", 269 | "result": type(test_output), 270 | "expected": tf.keras.Model, 271 | "error_message": 'Return type is incorrect. Please check implementation.' 272 | }, 273 | { 274 | "name": "layer_3_activation_check", 275 | "result": test_output.layers[4].activation, 276 | "expected": tf.keras.activations.sigmoid, 277 | "error_message": 'wine_quality layer has an incorrect activation. Please check implementation.' 278 | }, 279 | ] 280 | 281 | test_loop(test_cases) 282 | 283 | def test_model_compile(model): 284 | 285 | from tensorflow.python.keras.metrics import MeanMetricWrapper 286 | 287 | test_cases = [ 288 | { 289 | "name": "metrics_0_check", 290 | "result": type(model.metrics[0]), 291 | "expected": tf.keras.metrics.RootMeanSquaredError, 292 | "error_message": 'wine quality metrics is incorrect. Please check implementation.' 293 | }, 294 | { 295 | "name": "metrics_1_check", 296 | "result": (model.metrics[1].name == 'wine_type_accuracy') or 297 | (model.metrics[1].name == 'wine_type_binary_accuracy'), 298 | "expected": True, 299 | "error_message": f'wine type metrics: {model.metrics[1].name} is incorrect. Please check implementation.' 300 | }, 301 | { 302 | "name": "wine_type_loss_check", 303 | "result": (model.loss['wine_type'] == 'binary_crossentropy') or 304 | (model.loss['wine_type'].name == 'binary_crossentropy') or 305 | (str(model.loss['wine_type']).split()[1] == 'binary_crossentropy'), 306 | "expected": True, 307 | "error_message": f'wine type loss: {model.loss["wine_type"]} is incorrect. Please check implementation.' 308 | }, 309 | { 310 | "name": "wine_quality_loss_check", 311 | "result": (model.loss['wine_quality'] in ['mse', 'mean_squared_error']) or 312 | (str(model.loss['wine_quality']).split()[1] == 'mean_squared_error') or 313 | (model.loss['wine_quality'].name == 'mean_squared_error'), 314 | "expected": True, 315 | "error_message": f'wine quality loss: {model.loss["wine_type"]} is incorrect. Please check implementation.' 316 | }, 317 | ] 318 | 319 | test_loop(test_cases) 320 | 321 | def test_history(history): 322 | 323 | vars_history = vars(history) 324 | 325 | test_cases = [ 326 | { 327 | "name": "type_check", 328 | "result": type(history), 329 | "expected": tf.keras.callbacks.History, 330 | "error_message": 'history type is incorrect. Please check model.fit().' 331 | }, 332 | { 333 | "name": "params_samples_check", 334 | "result": vars_history['params']['samples'], 335 | "expected": 3155, 336 | "error_message": 'Training samples is incorrect. Please check arguments to model.fit().' 337 | }, 338 | { 339 | "name": "params_do_validation_check", 340 | "result": vars_history['params']['do_validation'], 341 | "expected": True, 342 | "error_message": 'No validation data is present. Please check arguments to model.fit().' 343 | }, 344 | ] 345 | 346 | test_loop(test_cases) 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-2/C1W2_Assignment.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# W2 Assignment: Creating a Custom Loss Function" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "This short exercise will require you to write a simple linear regression neural network that is trained on two arrays: $xs$ (inputs) and $ys$ (labels), where the relationship between each corresponding element is $y=2x-1$.\n", 15 | "\n", 16 | "\n", 17 | "$xs = [-1.0, 0.0, 1.0, 2.0, 3.0, 4.0]$\n", 18 | "\n", 19 | "$ys = [-3.0, -1.0, 1.0, 3.0, 5.0, 7.0]$\n", 20 | "\n", 21 | "\n", 22 | "You will need to implement a custom loss function that returns the root mean square error (RMSE) of $y_{true} - y_{pred}$. Let's begin!" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 1, 28 | "metadata": { 29 | "colab": {}, 30 | "colab_type": "code", 31 | "id": "0pajvrhrInPa" 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "import tensorflow as tf\n", 36 | "import numpy as np\n", 37 | "from tensorflow import keras\n", 38 | "from tensorflow.keras import backend as K\n", 39 | "\n", 40 | "import utils" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "# inputs\n", 50 | "xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float)\n", 51 | "\n", 52 | "# labels. relationship with the inputs above is y=2x-1.\n", 53 | "ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "### Define the custom loss function (TODO)\n", 61 | "Define the custom loss function below called `my_rmse()` that returns the RMSE between the target (`y_true`) and prediction (`y_pred`). \n", 62 | "\n", 63 | "You will return $\\sqrt{error}$, where $error$ = $mean((y_{true} - y_{pred})^2)$\n", 64 | "- error: the difference between the true label and predicted label.\n", 65 | "- sqr_error: the square of the error.\n", 66 | "- mean_sqr_error: the mean of the square of the error\n", 67 | "- sqrt_mean_sqr_error: the square root of hte mean of the square of the error (the root mean squared error).\n", 68 | "- Please use `K.mean`, `K.square`, and `K.sqrt`\n", 69 | "- The steps are broken down into separate lines of code for clarity. Feel free to combine them, and just remember to return the root mean squared error." 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 3, 75 | "metadata": { 76 | "colab": {}, 77 | "colab_type": "code", 78 | "deletable": false, 79 | "id": "bXNGIkq2Azmf", 80 | "nbgrader": { 81 | "cell_type": "code", 82 | "checksum": "8301324615aba1e02e1f756b4bf1b092", 83 | "grade": false, 84 | "grade_id": "cell-31648b482908e493", 85 | "locked": false, 86 | "schema_version": 3, 87 | "solution": true, 88 | "task": false 89 | } 90 | }, 91 | "outputs": [], 92 | "source": [ 93 | "# Please uncomment all lines in this cell and replace those marked with `# YOUR CODE HERE`.\n", 94 | "# You can select all lines in this code cell with Ctrl+A (Windows/Linux) or Cmd+A (Mac), then press Ctrl+/ (Windows/Linux) or Cmd+/ (Mac) to uncomment.\n", 95 | "\n", 96 | "\n", 97 | "\n", 98 | "def my_rmse(y_true, y_pred):\n", 99 | " error = y_true - y_pred \n", 100 | " sqr_error = K.square(error)\n", 101 | " mean_sqr_error = K.mean(sqr_error) \n", 102 | " sqrt_mean_sqr_error = K.sqrt(mean_sqr_error) \n", 103 | " return sqrt_mean_sqr_error" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 4, 109 | "metadata": { 110 | "deletable": false, 111 | "editable": false, 112 | "nbgrader": { 113 | "cell_type": "code", 114 | "checksum": "afa4ace3428496820b8b6fb542ca5117", 115 | "grade": true, 116 | "grade_id": "cell-578f76b36f8ee858", 117 | "locked": true, 118 | "points": 1, 119 | "schema_version": 3, 120 | "solution": false, 121 | "task": false 122 | } 123 | }, 124 | "outputs": [ 125 | { 126 | "name": "stdout", 127 | "output_type": "stream", 128 | "text": [ 129 | "\u001b[92m All public tests passed\n" 130 | ] 131 | } 132 | ], 133 | "source": [ 134 | "utils.test_my_rmse(my_rmse)\n" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "### Define a model using the custom loss function (TODO)\n", 142 | "Similar to the ungraded labs, you will define a simple model and pass the function you just coded as the loss.\n", 143 | "- When compiling the model, you'll choose the `sgd` optimizer and set the `loss` parameter to the custom loss function that you just defined.\n", 144 | "- For grading purposes, please leave the other parameter values as is." 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 5, 150 | "metadata": { 151 | "colab": { 152 | "base_uri": "https://localhost:8080/", 153 | "height": 34 154 | }, 155 | "colab_type": "code", 156 | "deletable": false, 157 | "id": "2eY7fw0EHwda", 158 | "nbgrader": { 159 | "cell_type": "code", 160 | "checksum": "8af71f8408d04ff7abaf41eb3414c8f6", 161 | "grade": false, 162 | "grade_id": "cell-5a29bb71c93124fc", 163 | "locked": false, 164 | "schema_version": 3, 165 | "solution": true, 166 | "task": false 167 | }, 168 | "outputId": "a3ea92e4-050e-463d-82c9-9b149554ae41" 169 | }, 170 | "outputs": [ 171 | { 172 | "name": "stdout", 173 | "output_type": "stream", 174 | "text": [ 175 | "[[19.097672]]\n" 176 | ] 177 | } 178 | ], 179 | "source": [ 180 | "# Please uncomment all lines in this cell and replace those marked with `# YOUR CODE HERE`.\n", 181 | "# You can select all lines in this code cell with Ctrl+A (Windows/Linux) or Cmd+A (Mac), then press Ctrl+/ (Windows/Linux) or Cmd+/ (Mac) to uncomment.\n", 182 | "\n", 183 | "\n", 184 | "\n", 185 | "# define the model architecture\n", 186 | "model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])\n", 187 | "\n", 188 | "# use the function you just coded as the loss\n", 189 | "model.compile(optimizer='sgd', loss=my_rmse)\n", 190 | " \n", 191 | "# train the model \n", 192 | "model.fit(xs, ys, epochs=500,verbose=0)\n", 193 | " \n", 194 | "# test with a sample input\n", 195 | "print(model.predict([10.0]))" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 6, 201 | "metadata": { 202 | "deletable": false, 203 | "editable": false, 204 | "nbgrader": { 205 | "cell_type": "code", 206 | "checksum": "b8da4dc42fa87a1722251adddae9516c", 207 | "grade": true, 208 | "grade_id": "cell-e46bc4e00375b387", 209 | "locked": true, 210 | "points": 1, 211 | "schema_version": 3, 212 | "solution": false, 213 | "task": false 214 | } 215 | }, 216 | "outputs": [ 217 | { 218 | "name": "stdout", 219 | "output_type": "stream", 220 | "text": [ 221 | "\u001b[92m All public tests passed\n" 222 | ] 223 | } 224 | ], 225 | "source": [ 226 | "utils.test_model_loss(model.loss)\n" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": null, 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [] 235 | } 236 | ], 237 | "metadata": { 238 | "colab": { 239 | "include_colab_link": true, 240 | "name": "exercise-answer.ipynb", 241 | "provenance": [] 242 | }, 243 | "kernelspec": { 244 | "display_name": "Python 3", 245 | "language": "python", 246 | "name": "python3" 247 | }, 248 | "language_info": { 249 | "codemirror_mode": { 250 | "name": "ipython", 251 | "version": 3 252 | }, 253 | "file_extension": ".py", 254 | "mimetype": "text/x-python", 255 | "name": "python", 256 | "nbconvert_exporter": "python", 257 | "pygments_lexer": "ipython3", 258 | "version": "3.7.6" 259 | } 260 | }, 261 | "nbformat": 4, 262 | "nbformat_minor": 4 263 | } 264 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-2/C1_W2_Lab_1_huber-loss.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ungraded Lab: Huber Loss\n", 8 | "\n", 9 | "In this lab, we'll walk through how to create custom loss functions. In particular, we'll code the [Huber Loss](https://en.wikipedia.org/wiki/Huber_loss) and use that in training the model." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Imports" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": { 23 | "colab": {}, 24 | "colab_type": "code", 25 | "id": "0p84I7yFHRT2" 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "try:\n", 30 | " # %tensorflow_version only exists in Colab.\n", 31 | " %tensorflow_version 2.x\n", 32 | "except Exception:\n", 33 | " pass\n", 34 | "\n", 35 | "import tensorflow as tf\n", 36 | "import numpy as np\n", 37 | "from tensorflow import keras" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Prepare the Data\n", 45 | "\n", 46 | "Our dummy dataset is just a pair of arrays `xs` and `ys` defined by the relationship $y = 2x - 1$. `xs` are the inputs while `ys` are the labels." 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 2, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "# inputs\n", 56 | "xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float)\n", 57 | "\n", 58 | "# labels\n", 59 | "ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float)" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "## Training the model\n", 67 | "\n", 68 | "Let's build a simple model and train using a built-in loss function like the `mean_squared_error`." 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": { 75 | "colab": {}, 76 | "colab_type": "code", 77 | "id": "2eY7fw0EHwda" 78 | }, 79 | "outputs": [ 80 | { 81 | "name": "stdout", 82 | "output_type": "stream", 83 | "text": [ 84 | "[[18.984268]]\n" 85 | ] 86 | } 87 | ], 88 | "source": [ 89 | "model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])\n", 90 | "model.compile(optimizer='sgd', loss='mean_squared_error')\n", 91 | "model.fit(xs, ys, epochs=500,verbose=0)\n", 92 | "\n", 93 | "print(model.predict([10.0]))" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "### Custom Loss\n", 101 | "\n", 102 | "Now let's see how we can use a custom loss. We first define a function that accepts the ground truth labels (`y_true`) and model predictions (`y_pred`) as parameters. We then compute and return the loss value in the function definition." 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 4, 108 | "metadata": { 109 | "colab": {}, 110 | "colab_type": "code", 111 | "id": "fHtDxE0lI_Hg" 112 | }, 113 | "outputs": [], 114 | "source": [ 115 | "def my_huber_loss(y_true, y_pred):\n", 116 | " threshold = 1\n", 117 | " error = y_true - y_pred\n", 118 | " is_small_error = tf.abs(error) <= threshold\n", 119 | " small_error_loss = tf.square(error) / 2\n", 120 | " big_error_loss = threshold * (tf.abs(error) - (0.5 * threshold))\n", 121 | " return tf.where(is_small_error, small_error_loss, big_error_loss)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "Using the loss function is as simple as specifying the loss function in the `loss` argument of `model.compile()`." 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 5, 134 | "metadata": { 135 | "colab": {}, 136 | "colab_type": "code", 137 | "id": "K7rgmb5qH5QX" 138 | }, 139 | "outputs": [ 140 | { 141 | "name": "stdout", 142 | "output_type": "stream", 143 | "text": [ 144 | "[[18.40429]]\n" 145 | ] 146 | } 147 | ], 148 | "source": [ 149 | "model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])\n", 150 | "model.compile(optimizer='sgd', loss=my_huber_loss)\n", 151 | "model.fit(xs, ys, epochs=500,verbose=0)\n", 152 | "print(model.predict([10.0]))" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [] 161 | } 162 | ], 163 | "metadata": { 164 | "colab": { 165 | "include_colab_link": true, 166 | "name": "first-loss.ipynb", 167 | "provenance": [] 168 | }, 169 | "kernelspec": { 170 | "display_name": "Python 3", 171 | "language": "python", 172 | "name": "python3" 173 | }, 174 | "language_info": { 175 | "codemirror_mode": { 176 | "name": "ipython", 177 | "version": 3 178 | }, 179 | "file_extension": ".py", 180 | "mimetype": "text/x-python", 181 | "name": "python", 182 | "nbconvert_exporter": "python", 183 | "pygments_lexer": "ipython3", 184 | "version": "3.7.6" 185 | } 186 | }, 187 | "nbformat": 4, 188 | "nbformat_minor": 4 189 | } 190 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-2/C1_W2_Lab_2_huber-object-loss.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ungraded Lab: Huber Loss hyperparameter and class\n", 8 | "\n", 9 | "In this lab, we'll extend our previous Huber loss function and show how you can include hyperparameters in defining loss functions. We'll also look at how to implement a custom loss as an object by inheriting the [Loss](https://www.tensorflow.org/api_docs/python/tf/keras/losses/Loss) class." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Imports" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 9, 22 | "metadata": { 23 | "colab": {}, 24 | "colab_type": "code", 25 | "id": "0p84I7yFHRT2" 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "try:\n", 30 | " # %tensorflow_version only exists in Colab.\n", 31 | " %tensorflow_version 2.x\n", 32 | "except Exception:\n", 33 | " pass\n", 34 | "\n", 35 | "import tensorflow as tf\n", 36 | "import numpy as np\n", 37 | "from tensorflow import keras" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Dataset\n", 45 | "\n", 46 | "As before, this model will be trained on the `xs` and `ys` below where the relationship is $y = 2x-1$. Thus, later, when we test for `x=10`, whichever version of the model gets the closest answer to `19` will be deemed more accurate." 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 10, 52 | "metadata": { 53 | "colab": {}, 54 | "colab_type": "code", 55 | "id": "0pajvrhrInPa" 56 | }, 57 | "outputs": [], 58 | "source": [ 59 | "# inputs\n", 60 | "xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float)\n", 61 | "\n", 62 | "# labels\n", 63 | "ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "## Custom loss with hyperparameter\n", 71 | "\n", 72 | "The `loss` argument in `model.compile()` only accepts functions that accepts two parameters: the ground truth (`y_true`) and the model predictions (`y_pred`). If we want to include a hyperparameter that we can tune, then we can define a wrapper function that accepts this hyperparameter." 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 11, 78 | "metadata": { 79 | "colab": {}, 80 | "colab_type": "code", 81 | "id": "FvRZpxk2tGn6" 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "# wrapper function that accepts the hyperparameter\n", 86 | "def my_huber_loss_with_threshold(threshold):\n", 87 | " \n", 88 | " # function that accepts the ground truth and predictions\n", 89 | " def my_huber_loss(y_true, y_pred):\n", 90 | " error = y_true - y_pred\n", 91 | " is_small_error = tf.abs(error) <= threshold\n", 92 | " small_error_loss = tf.square(error) / 2\n", 93 | " big_error_loss = threshold * (tf.abs(error) - (0.5 * threshold))\n", 94 | " \n", 95 | " return tf.where(is_small_error, small_error_loss, big_error_loss) \n", 96 | "\n", 97 | " # return the inner function tuned by the hyperparameter\n", 98 | " return my_huber_loss" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "We can now specify the `loss` as the wrapper function above. Notice that we can now set the `threshold` value. Try varying this value and see the results you get." 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 12, 111 | "metadata": { 112 | "colab": {}, 113 | "colab_type": "code", 114 | "id": "K7rgmb5qH5QX" 115 | }, 116 | "outputs": [ 117 | { 118 | "name": "stdout", 119 | "output_type": "stream", 120 | "text": [ 121 | "[[18.788555]]\n" 122 | ] 123 | } 124 | ], 125 | "source": [ 126 | "model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])\n", 127 | "model.compile(optimizer='sgd', loss=my_huber_loss_with_threshold(threshold=1.2))\n", 128 | "model.fit(xs, ys, epochs=500,verbose=0)\n", 129 | "print(model.predict([10.0]))" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "## Implement Custom Loss as a Class\n", 137 | "\n", 138 | "We can also implement our custom loss as a class. It inherits from the Keras Loss class and the syntax and required methods are shown below." 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 16, 144 | "metadata": { 145 | "colab": {}, 146 | "colab_type": "code", 147 | "id": "iderl2m0saTa" 148 | }, 149 | "outputs": [], 150 | "source": [ 151 | "from tensorflow.keras.losses import Loss\n", 152 | "\n", 153 | "class MyHuberLoss(Loss):\n", 154 | " \n", 155 | " # class attribute\n", 156 | " threshold = 1\n", 157 | " \n", 158 | " # initialize instance attributes\n", 159 | " def __init__(self, threshold):\n", 160 | " super().__init__()\n", 161 | " self.threshold = threshold\n", 162 | "\n", 163 | " # compute loss\n", 164 | " def call(self, y_true, y_pred):\n", 165 | " error = y_true - y_pred\n", 166 | " is_small_error = tf.abs(error) <= self.threshold\n", 167 | " small_error_loss = tf.square(error) / 2\n", 168 | " big_error_loss = self.threshold * (tf.abs(error) - (0.5 * self.threshold))\n", 169 | " return tf.where(is_small_error, small_error_loss, big_error_loss)" 170 | ] 171 | }, 172 | { 173 | "cell_type": "markdown", 174 | "metadata": {}, 175 | "source": [ 176 | "You can specify the loss by instantiating an object from your custom loss class." 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 17, 182 | "metadata": { 183 | "colab": {}, 184 | "colab_type": "code", 185 | "id": "lwKWfSmDIayP" 186 | }, 187 | "outputs": [ 188 | { 189 | "name": "stdout", 190 | "output_type": "stream", 191 | "text": [ 192 | "[[18.328104]]\n" 193 | ] 194 | } 195 | ], 196 | "source": [ 197 | "model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])\n", 198 | "model.compile(optimizer='sgd', loss=MyHuberLoss(threshold=1.02))\n", 199 | "model.fit(xs, ys, epochs=500,verbose=0)\n", 200 | "print(model.predict([10.0]))" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [] 209 | } 210 | ], 211 | "metadata": { 212 | "colab": { 213 | "include_colab_link": true, 214 | "name": "huber-object-loss.ipynb", 215 | "provenance": [] 216 | }, 217 | "kernelspec": { 218 | "display_name": "Python 3", 219 | "language": "python", 220 | "name": "python3" 221 | }, 222 | "language_info": { 223 | "codemirror_mode": { 224 | "name": "ipython", 225 | "version": 3 226 | }, 227 | "file_extension": ".py", 228 | "mimetype": "text/x-python", 229 | "name": "python", 230 | "nbconvert_exporter": "python", 231 | "pygments_lexer": "ipython3", 232 | "version": "3.7.6" 233 | } 234 | }, 235 | "nbformat": 4, 236 | "nbformat_minor": 4 237 | } 238 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-2/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.model_selection import train_test_split 3 | from tensorflow.python.framework.ops import EagerTensor 4 | from numpy import int64 5 | 6 | def test_loop(test_cases): 7 | 8 | success = 0 9 | fails = 0 10 | 11 | for test_case in test_cases: 12 | try: 13 | assert test_case["result"] == test_case["expected"] 14 | success += 1 15 | 16 | except: 17 | fails += 1 18 | print(f'{test_case["name"]}: {test_case["error_message"]}\nExpected: {test_case["expected"]}\nResult: {test_case["result"]}\n') 19 | 20 | if fails == 0: 21 | print("\033[92m All public tests passed") 22 | 23 | else: 24 | print('\033[92m', success," Tests passed") 25 | print('\033[91m', fails, " Tests failed") 26 | raise Exception(test_case["error_message"]) 27 | 28 | 29 | def test_my_rmse(my_rmse): 30 | 31 | test_y_true = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float) 32 | test_y_pred = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float) 33 | 34 | expected = 1.7795130420052185 35 | 36 | result = my_rmse(test_y_true, test_y_pred) 37 | 38 | test_cases = [ 39 | { 40 | "name": "type_check", 41 | "result": type(result), 42 | "expected": EagerTensor, 43 | "error_message": f'output has an incorrect type.' 44 | }, 45 | { 46 | "name": "output_check", 47 | "result": result, 48 | "expected": expected, 49 | "error_message": "Output is incorrect. Please check the equation." 50 | } 51 | ] 52 | 53 | test_loop(test_cases) 54 | 55 | def test_model_loss(model_loss): 56 | 57 | test_y_true = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float) 58 | test_y_pred = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float) 59 | 60 | expected = 1.7795130420052185 61 | 62 | result = model_loss(test_y_true, test_y_pred) 63 | 64 | test_cases = [ 65 | { 66 | "name": "type_check", 67 | "result": type(result), 68 | "expected": EagerTensor, 69 | "error_message": f'output has an incorrect type.' 70 | }, 71 | { 72 | "name": "output_check", 73 | "result": result, 74 | "expected": expected, 75 | "error_message": "Output is incorrect. Please check the equation." 76 | } 77 | ] 78 | 79 | test_loop(test_cases) -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-3/C1W3_Assignment.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Week 3 Assignment: Implement a Quadratic Layer" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this week's programming exercise, you will build a custom quadratic layer which computes $y = ax^2 + bx + c$. Similar to the ungraded lab, this layer will be plugged into a model that will be trained on the MNIST dataset. Let's get started!" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "### Imports" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 4, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import tensorflow as tf\n", 31 | "from tensorflow.keras.layers import Layer\n", 32 | "\n", 33 | "import utils" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "### Define the quadratic layer (TODO)\n", 41 | "Implement a simple quadratic layer. It has 3 state variables: $a$, $b$ and $c$. The computation returned is $ax^2 + bx + c$. Make sure it can also accept an activation function.\n", 42 | "\n", 43 | "#### `__init__`\n", 44 | "- call `super(my_fun, self)` to access the base class of `my_fun`, and call the `__init__()` function to initialize that base class. In this case, `my_fun` is `SimpleQuadratic` and its base class is `Layer`.\n", 45 | "- self.units: set this using one of the function parameters.\n", 46 | "- self.activation: The function parameter `activation` will be passed in as a string. To get the tensorflow object associated with the string, please use `tf.keras.activations.get()` \n", 47 | "\n", 48 | "\n", 49 | "#### `build`\n", 50 | "The following are suggested steps for writing your code. If you prefer to use fewer lines to implement it, feel free to do so. Either way, you'll want to set `self.a`, `self.b` and `self.c`.\n", 51 | "\n", 52 | "- a_init: set this to tensorflow's `random_normal_initializer()`\n", 53 | "- a_init_val: Use the `random_normal_initializer()` that you just created and invoke it, setting the `shape` and `dtype`.\n", 54 | " - The `shape` of `a` should have its row dimension equal to the last dimension of `input_shape`, and its column dimension equal to the number of units in the layer. \n", 55 | " - This is because you'll be matrix multiplying x^2 * a, so the dimensions should be compatible.\n", 56 | " - set the dtype to 'float32'\n", 57 | "- self.a: create a tensor using tf.Variable, setting the initial_value and set trainable to True.\n", 58 | "\n", 59 | "- b_init, b_init_val, and self.b: these will be set in the same way that you implemented a_init, a_init_val and self.a\n", 60 | "- c_init: set this to `tf.zeros_initializer`.\n", 61 | "- c_init_val: Set this by calling the tf.zeros_initializer that you just instantiated, and set the `shape` and `dtype`\n", 62 | " - shape: This will be a vector equal to the number of units. This expects a tuple, and remember that a tuple `(9,)` includes a comma.\n", 63 | " - dtype: set to 'float32'.\n", 64 | "- self.c: create a tensor using tf.Variable, and set the parameters `initial_value` and `trainable`.\n", 65 | "\n", 66 | "#### `call`\n", 67 | "The following section performs the multiplication x^2*a + x*b + c. The steps are broken down for clarity, but you can also perform this calculation in fewer lines if you prefer.\n", 68 | "- x_squared: use tf.math.square()\n", 69 | "- x_squared_times_a: use tf.matmul(). \n", 70 | " - If you see an error saying `InvalidArgumentError: Matrix size-incompatible`, please check the order of the matrix multiplication to make sure that the matrix dimensions line up.\n", 71 | "- x_times_b: use tf.matmul().\n", 72 | "- x2a_plus_xb_plus_c: add the three terms together.\n", 73 | "- activated_x2a_plus_xb_plus_c: apply the class's `activation` to the sum of the three terms.\n" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 15, 79 | "metadata": { 80 | "colab": {}, 81 | "colab_type": "code", 82 | "deletable": false, 83 | "id": "Ga20PttZFXm4", 84 | "nbgrader": { 85 | "cell_type": "code", 86 | "checksum": "0df055c519bde80c488c22be89fdb8ef", 87 | "grade": false, 88 | "grade_id": "cell-c302ddc177c098f8", 89 | "locked": false, 90 | "schema_version": 3, 91 | "solution": true, 92 | "task": false 93 | } 94 | }, 95 | "outputs": [], 96 | "source": [ 97 | "# Please uncomment all lines in this cell and replace those marked with `# YOUR CODE HERE`.\n", 98 | "# You can select all lines in this code cell with Ctrl+A (Windows/Linux) or Cmd+A (Mac), then press Ctrl+/ (Windows/Linux) or Cmd+/ (Mac) to uncomment.\n", 99 | "\n", 100 | "\n", 101 | "\n", 102 | "class SimpleQuadratic(Layer):\n", 103 | "\n", 104 | " def __init__(self, units=32, activation=None):\n", 105 | " '''Initializes the class and sets up the internal variables'''\n", 106 | " # YOUR CODE HERE \n", 107 | " super(SimpleQuadratic, self).__init__() \n", 108 | " self.units = units \n", 109 | " self.activation = tf.keras.activations.get(activation) \n", 110 | " \n", 111 | " def build(self, input_shape):\n", 112 | " '''Create the state of the layer (weights)'''\n", 113 | " # a and b should be initialized with random normal, c (or the bias) with zeros.\n", 114 | " # remember to set these as trainable.\n", 115 | " # YOUR CODE HERE\n", 116 | " \n", 117 | " a_init = tf.random_normal_initializer() \n", 118 | " self.a = tf.Variable(name = 'a', initial_value = a_init(shape = (input_shape[-1], self.units), dtype = 'float32'), trainable = True) \n", 119 | " \n", 120 | " b_init = tf.random_normal_initializer() \n", 121 | " self.b = tf.Variable(name = 'b', initial_value = b_init(shape = (input_shape[-1], self.units), dtype = 'float32'), trainable = True) \n", 122 | " \n", 123 | " c_init = tf.zeros_initializer() \n", 124 | " self.c = tf.Variable(name = 'bias', initial_value = c_init(shape = (self.units,), dtype = 'float32'),trainable = True)\n", 125 | " \n", 126 | " #super().build(input_shape) \n", 127 | " \n", 128 | " def call(self, inputs):\n", 129 | " '''Defines the computation from inputs to outputs'''\n", 130 | " # YOUR CODE HERE \n", 131 | " return self.activation(tf.matmul(tf.math.square(inputs), self.a) + tf.matmul(inputs, self.b) + self.c) \n", 132 | " " 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "metadata": {}, 138 | "source": [ 139 | "Test your implementation" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 16, 145 | "metadata": { 146 | "deletable": false, 147 | "editable": false, 148 | "nbgrader": { 149 | "cell_type": "code", 150 | "checksum": "0965bec4878a263cf06b286cd0fe3b2a", 151 | "grade": true, 152 | "grade_id": "cell-c3ebc4cccbb7f454", 153 | "locked": true, 154 | "points": 1, 155 | "schema_version": 3, 156 | "solution": false, 157 | "task": false 158 | } 159 | }, 160 | "outputs": [ 161 | { 162 | "name": "stdout", 163 | "output_type": "stream", 164 | "text": [ 165 | "\u001b[92m All public tests passed\n" 166 | ] 167 | } 168 | ], 169 | "source": [ 170 | "utils.test_simple_quadratic(SimpleQuadratic)\n" 171 | ] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": {}, 176 | "source": [ 177 | "Train your model with the `SimpleQuadratic` loss function that you just implemented." 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 17, 183 | "metadata": { 184 | "colab": {}, 185 | "colab_type": "code", 186 | "id": "14tl1CluExjJ" 187 | }, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n", 194 | "11493376/11490434 [==============================] - 0s 0us/step\n", 195 | "Train on 60000 samples\n", 196 | "Epoch 1/5\n", 197 | "60000/60000 [==============================] - 13s 220us/sample - loss: 0.2697 - accuracy: 0.9204\n", 198 | "Epoch 2/5\n", 199 | "60000/60000 [==============================] - 13s 213us/sample - loss: 0.1335 - accuracy: 0.9594- loss: 0.133\n", 200 | "Epoch 3/5\n", 201 | "60000/60000 [==============================] - 13s 212us/sample - loss: 0.0999 - accuracy: 0.9697\n", 202 | "Epoch 4/5\n", 203 | "60000/60000 [==============================] - 13s 210us/sample - loss: 0.0831 - accuracy: 0.9742\n", 204 | "Epoch 5/5\n", 205 | "60000/60000 [==============================] - 13s 212us/sample - loss: 0.0734 - accuracy: 0.9776\n", 206 | "10000/10000 [==============================] - 1s 76us/sample - loss: 0.0813 - accuracy: 0.9763\n" 207 | ] 208 | }, 209 | { 210 | "data": { 211 | "text/plain": [ 212 | "[0.08129077708083204, 0.9763]" 213 | ] 214 | }, 215 | "execution_count": 17, 216 | "metadata": {}, 217 | "output_type": "execute_result" 218 | } 219 | ], 220 | "source": [ 221 | "# THIS CODE SHOULD RUN WITHOUT MODIFICATION\n", 222 | "# AND SHOULD RETURN TRAINING/TESTING ACCURACY at 97%+\n", 223 | "\n", 224 | "mnist = tf.keras.datasets.mnist\n", 225 | "\n", 226 | "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n", 227 | "x_train, x_test = x_train / 255.0, x_test / 255.0\n", 228 | "\n", 229 | "model = tf.keras.models.Sequential([\n", 230 | " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", 231 | " SimpleQuadratic(128, activation='relu'),\n", 232 | " tf.keras.layers.Dropout(0.2),\n", 233 | " tf.keras.layers.Dense(10, activation='softmax')\n", 234 | "])\n", 235 | "\n", 236 | "model.compile(optimizer='adam',\n", 237 | " loss='sparse_categorical_crossentropy',\n", 238 | " metrics=['accuracy'])\n", 239 | "\n", 240 | "model.fit(x_train, y_train, epochs=5)\n", 241 | "model.evaluate(x_test, y_test)" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [] 250 | } 251 | ], 252 | "metadata": { 253 | "colab": { 254 | "authorship_tag": "ABX9TyMTFXTWT0EUVuqg6u/LBbJK", 255 | "collapsed_sections": [], 256 | "include_colab_link": true, 257 | "name": "QuadraticLayer_Answer.ipynb", 258 | "provenance": [] 259 | }, 260 | "kernelspec": { 261 | "display_name": "Python 3", 262 | "language": "python", 263 | "name": "python3" 264 | }, 265 | "language_info": { 266 | "codemirror_mode": { 267 | "name": "ipython", 268 | "version": 3 269 | }, 270 | "file_extension": ".py", 271 | "mimetype": "text/x-python", 272 | "name": "python", 273 | "nbconvert_exporter": "python", 274 | "pygments_lexer": "ipython3", 275 | "version": "3.7.6" 276 | } 277 | }, 278 | "nbformat": 4, 279 | "nbformat_minor": 4 280 | } 281 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-3/C1_W3_Lab_1_lambda-layer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Ungraded Lab: Lambda Layer\n", 8 | "\n", 9 | "This lab will show how you can define custom layers with the [Lambda](https://keras.io/api/layers/core_layers/lambda/) layer. You can either use [lambda functions](https://www.w3schools.com/python/python_lambda.asp) within the Lambda layer or define a custom function that the Lambda layer will call. Let's get started!" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Imports" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": { 23 | "colab": {}, 24 | "colab_type": "code", 25 | "id": "CmI9MQA6Z72_" 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "try:\n", 30 | " # %tensorflow_version only exists in Colab.\n", 31 | " %tensorflow_version 2.x\n", 32 | "except Exception:\n", 33 | " pass\n", 34 | "\n", 35 | "import tensorflow as tf\n", 36 | "from tensorflow.keras import backend as K" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "## Prepare the Dataset" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 2, 49 | "metadata": { 50 | "colab": {}, 51 | "colab_type": "code", 52 | "id": "14tl1CluExjJ" 53 | }, 54 | "outputs": [ 55 | { 56 | "name": "stdout", 57 | "output_type": "stream", 58 | "text": [ 59 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n", 60 | "11493376/11490434 [==============================] - 0s 0us/step\n" 61 | ] 62 | } 63 | ], 64 | "source": [ 65 | "mnist = tf.keras.datasets.mnist\n", 66 | "\n", 67 | "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n", 68 | "x_train, x_test = x_train / 255.0, x_test / 255.0" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "## Build the Model\n", 76 | "\n", 77 | "Here, we'll use a Lambda layer to define a custom layer in our network. We're using a lambda function to get the absolute value of the layer input." 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 3, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "model = tf.keras.models.Sequential([\n", 87 | " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", 88 | " tf.keras.layers.Dense(128),\n", 89 | " tf.keras.layers.Lambda(lambda x: tf.abs(x)), \n", 90 | " tf.keras.layers.Dense(10, activation='softmax')\n", 91 | "])" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 4, 97 | "metadata": {}, 98 | "outputs": [ 99 | { 100 | "name": "stdout", 101 | "output_type": "stream", 102 | "text": [ 103 | "Train on 60000 samples\n", 104 | "Epoch 1/5\n", 105 | "60000/60000 [==============================] - 5s 75us/sample - loss: 0.2161 - accuracy: 0.9404\n", 106 | "Epoch 2/5\n", 107 | "60000/60000 [==============================] - 4s 70us/sample - loss: 0.0899 - accuracy: 0.9733\n", 108 | "Epoch 3/5\n", 109 | "60000/60000 [==============================] - 4s 70us/sample - loss: 0.0623 - accuracy: 0.9812\n", 110 | "Epoch 4/5\n", 111 | "60000/60000 [==============================] - 4s 71us/sample - loss: 0.0463 - accuracy: 0.9855\n", 112 | "Epoch 5/5\n", 113 | "60000/60000 [==============================] - 4s 71us/sample - loss: 0.0374 - accuracy: 0.9880\n", 114 | "10000/10000 [==============================] - 0s 35us/sample - loss: 0.0872 - accuracy: 0.9772\n" 115 | ] 116 | }, 117 | { 118 | "data": { 119 | "text/plain": [ 120 | "[0.08715012353332131, 0.9772]" 121 | ] 122 | }, 123 | "execution_count": 4, 124 | "metadata": {}, 125 | "output_type": "execute_result" 126 | } 127 | ], 128 | "source": [ 129 | "model.compile(optimizer='adam',\n", 130 | " loss='sparse_categorical_crossentropy',\n", 131 | " metrics=['accuracy'])\n", 132 | "\n", 133 | "model.fit(x_train, y_train, epochs=5)\n", 134 | "model.evaluate(x_test, y_test)" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "Another way to use the Lambda layer is to pass in a function defined outside the model. The code below shows how a custom ReLU function is used as a custom layer in the model." 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 5, 147 | "metadata": { 148 | "colab": { 149 | "base_uri": "https://localhost:8080/", 150 | "height": 221 151 | }, 152 | "colab_type": "code", 153 | "id": "jG53KYeq-3_X", 154 | "outputId": "5b4fd048-e604-4109-f4f2-8fcd859a54e9" 155 | }, 156 | "outputs": [ 157 | { 158 | "name": "stdout", 159 | "output_type": "stream", 160 | "text": [ 161 | "Train on 60000 samples\n", 162 | "Epoch 1/5\n", 163 | "60000/60000 [==============================] - 4s 75us/sample - loss: 0.2588 - accuracy: 0.9259\n", 164 | "Epoch 2/5\n", 165 | "60000/60000 [==============================] - 4s 73us/sample - loss: 0.1160 - accuracy: 0.9659\n", 166 | "Epoch 3/5\n", 167 | "60000/60000 [==============================] - 4s 72us/sample - loss: 0.0792 - accuracy: 0.9765\n", 168 | "Epoch 4/5\n", 169 | "60000/60000 [==============================] - 4s 73us/sample - loss: 0.0593 - accuracy: 0.9817\n", 170 | "Epoch 5/5\n", 171 | "60000/60000 [==============================] - 4s 72us/sample - loss: 0.0468 - accuracy: 0.9857\n", 172 | "10000/10000 [==============================] - 0s 30us/sample - loss: 0.0815 - accuracy: 0.9758\n" 173 | ] 174 | }, 175 | { 176 | "data": { 177 | "text/plain": [ 178 | "[0.08153247374091298, 0.9758]" 179 | ] 180 | }, 181 | "execution_count": 5, 182 | "metadata": {}, 183 | "output_type": "execute_result" 184 | } 185 | ], 186 | "source": [ 187 | "def my_relu(x):\n", 188 | " return K.maximum(-0.1, x)\n", 189 | "\n", 190 | "model = tf.keras.models.Sequential([\n", 191 | " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", 192 | " tf.keras.layers.Dense(128),\n", 193 | " tf.keras.layers.Lambda(my_relu), \n", 194 | " tf.keras.layers.Dense(10, activation='softmax')\n", 195 | "])\n", 196 | "\n", 197 | "model.compile(optimizer='adam',\n", 198 | " loss='sparse_categorical_crossentropy',\n", 199 | " metrics=['accuracy'])\n", 200 | "\n", 201 | "model.fit(x_train, y_train, epochs=5)\n", 202 | "model.evaluate(x_test, y_test)" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": null, 208 | "metadata": {}, 209 | "outputs": [], 210 | "source": [] 211 | } 212 | ], 213 | "metadata": { 214 | "colab": { 215 | "collapsed_sections": [], 216 | "include_colab_link": true, 217 | "name": "LambdaLayer.ipynb", 218 | "provenance": [] 219 | }, 220 | "kernelspec": { 221 | "display_name": "Python 3", 222 | "language": "python", 223 | "name": "python3" 224 | }, 225 | "language_info": { 226 | "codemirror_mode": { 227 | "name": "ipython", 228 | "version": 3 229 | }, 230 | "file_extension": ".py", 231 | "mimetype": "text/x-python", 232 | "name": "python", 233 | "nbconvert_exporter": "python", 234 | "pygments_lexer": "ipython3", 235 | "version": "3.7.6" 236 | } 237 | }, 238 | "nbformat": 4, 239 | "nbformat_minor": 4 240 | } 241 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-3/C1_W3_Lab_2_custom-dense-layer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ungraded Lab: Building a Custom Dense Layer\n", 8 | "\n", 9 | "In this lab, we'll walk through how to create a custom layer that inherits the [Layer](https://keras.io/api/layers/base_layer/#layer-class) class. Unlike simple Lambda layers you did previously, the custom layer here will contain weights that can be updated during training." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Imports" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": { 23 | "colab": {}, 24 | "colab_type": "code", 25 | "id": "DpioxwFXD9Is" 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "try:\n", 30 | " # %tensorflow_version only exists in Colab.\n", 31 | " %tensorflow_version 2.x\n", 32 | "except Exception:\n", 33 | " pass\n", 34 | "\n", 35 | "import tensorflow as tf\n", 36 | "import numpy as np" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "## Custom Layer with weights\n", 44 | "\n", 45 | "To make custom layer that is trainable, we need to define a class that inherits the [Layer](https://keras.io/api/layers/base_layer/#layer-class) base class from Keras. The Python syntax is shown below in the class declaration. This class requires three functions: `__init__()`, `build()` and `call()`. These ensure that our custom layer has a *state* and *computation* that can be accessed during training or inference." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": { 52 | "colab": {}, 53 | "colab_type": "code", 54 | "id": "jnVrzRT6BPWl" 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# inherit from this base class\n", 59 | "from tensorflow.keras.layers import Layer\n", 60 | "\n", 61 | "class SimpleDense(Layer):\n", 62 | "\n", 63 | " def __init__(self, units=32):\n", 64 | " '''Initializes the instance attributes'''\n", 65 | " super(SimpleDense, self).__init__()\n", 66 | " self.units = units\n", 67 | "\n", 68 | " def build(self, input_shape):\n", 69 | " '''Create the state of the layer (weights)'''\n", 70 | " # initialize the weights\n", 71 | " w_init = tf.random_normal_initializer()\n", 72 | " self.w = tf.Variable(name=\"kernel\",\n", 73 | " initial_value=w_init(shape=(input_shape[-1], self.units),\n", 74 | " dtype='float32'),\n", 75 | " trainable=True)\n", 76 | "\n", 77 | " # initialize the biases\n", 78 | " b_init = tf.zeros_initializer()\n", 79 | " self.b = tf.Variable(name=\"bias\",\n", 80 | " initial_value=b_init(shape=(self.units,), dtype='float32'),\n", 81 | " trainable=True)\n", 82 | "\n", 83 | " def call(self, inputs):\n", 84 | " '''Defines the computation from inputs to outputs'''\n", 85 | " return tf.matmul(inputs, self.w) + self.b" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "Now we can use our custom layer like below:" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "# declare an instance of the class\n", 102 | "my_dense = SimpleDense(units=1)\n", 103 | "\n", 104 | "# define an input and feed into the layer\n", 105 | "x = tf.ones((1, 1))\n", 106 | "y = my_dense(x)\n", 107 | "\n", 108 | "# parameters of the base Layer class like `variables` can be used\n", 109 | "print(my_dense.variables)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "Let's then try using it in simple network:" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": { 123 | "colab": {}, 124 | "colab_type": "code", 125 | "id": "WwTPJT4DkTyW" 126 | }, 127 | "outputs": [], 128 | "source": [ 129 | "# define the dataset\n", 130 | "xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float)\n", 131 | "ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float)\n", 132 | "\n", 133 | "\n", 134 | "# use the Sequential API to build a model with our custom layer\n", 135 | "my_layer = SimpleDense(units=1)\n", 136 | "model = tf.keras.Sequential([my_layer])\n", 137 | "\n", 138 | "# configure and train the model\n", 139 | "model.compile(optimizer='sgd', loss='mean_squared_error')\n", 140 | "model.fit(xs, ys, epochs=500,verbose=0)\n", 141 | "\n", 142 | "# perform inference\n", 143 | "print(model.predict([10.0]))\n", 144 | "\n", 145 | "# see the updated state of the variables\n", 146 | "print(my_layer.variables)" 147 | ] 148 | } 149 | ], 150 | "metadata": { 151 | "colab": { 152 | "authorship_tag": "ABX9TyMm2GqP/jiEhAWgS1uK97vE", 153 | "collapsed_sections": [], 154 | "include_colab_link": true, 155 | "name": "CustomDenseLayer.ipynb", 156 | "provenance": [] 157 | }, 158 | "kernelspec": { 159 | "display_name": "Python 3", 160 | "language": "python", 161 | "name": "python3" 162 | }, 163 | "language_info": { 164 | "codemirror_mode": { 165 | "name": "ipython", 166 | "version": 3 167 | }, 168 | "file_extension": ".py", 169 | "mimetype": "text/x-python", 170 | "name": "python", 171 | "nbconvert_exporter": "python", 172 | "pygments_lexer": "ipython3", 173 | "version": "3.7.4" 174 | } 175 | }, 176 | "nbformat": 4, 177 | "nbformat_minor": 4 178 | } 179 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-3/C1_W3_Lab_3_custom-layer-activation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ungraded Lab: Activation in Custom Layers\n", 8 | "\n", 9 | "In this lab, we extend our knowledge of building custom layers by adding an activation parameter. The implementation is pretty straightforward as you'll see below." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Imports" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": { 23 | "colab": {}, 24 | "colab_type": "code", 25 | "id": "DpioxwFXD9Is" 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "try:\n", 30 | " # %tensorflow_version only exists in Colab.\n", 31 | " %tensorflow_version 2.x\n", 32 | "except Exception:\n", 33 | " pass\n", 34 | "\n", 35 | "import tensorflow as tf\n", 36 | "from tensorflow.keras.layers import Layer" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "## Adding an activation layer\n", 44 | "\n", 45 | "To use the built-in activations in Keras, we can specify an `activation` parameter in the `__init__()` method of our custom layer class. From there, we can initialize it by using the `tf.keras.activations.get()` method. This takes in a string identifier that corresponds to one of the [available activations](https://keras.io/api/layers/activations/#available-activations) in Keras. Next, you can now pass in the forward computation to this activation in the `call()` method." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 2, 51 | "metadata": { 52 | "colab": {}, 53 | "colab_type": "code", 54 | "id": "jnVrzRT6BPWl" 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "class SimpleDense(Layer):\n", 59 | "\n", 60 | " # add an activation parameter\n", 61 | " def __init__(self, units=32, activation=None):\n", 62 | " super(SimpleDense, self).__init__()\n", 63 | " self.units = units\n", 64 | " \n", 65 | " # define the activation to get from the built-in activation layers in Keras\n", 66 | " self.activation = tf.keras.activations.get(activation)\n", 67 | "\n", 68 | "\n", 69 | " def build(self, input_shape):\n", 70 | " w_init = tf.random_normal_initializer()\n", 71 | " self.w = tf.Variable(name=\"kernel\",\n", 72 | " initial_value=w_init(shape=(input_shape[-1], self.units),\n", 73 | " dtype='float32'),\n", 74 | " trainable=True)\n", 75 | " b_init = tf.zeros_initializer()\n", 76 | " self.b = tf.Variable(name=\"bias\",\n", 77 | " initial_value=b_init(shape=(self.units,), dtype='float32'),\n", 78 | " trainable=True)\n", 79 | " #super().build(input_shape)\n", 80 | "\n", 81 | "\n", 82 | " def call(self, inputs):\n", 83 | " \n", 84 | " # pass the computation to the activation layer\n", 85 | " return self.activation(tf.matmul(inputs, self.w) + self.b)" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "We can now pass in an activation parameter to our custom layer. The string identifier is mostly the same as the function name so 'relu' below will get `tf.keras.activations.relu`." 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 3, 98 | "metadata": { 99 | "colab": {}, 100 | "colab_type": "code", 101 | "id": "WwTPJT4DkTyW" 102 | }, 103 | "outputs": [ 104 | { 105 | "name": "stdout", 106 | "output_type": "stream", 107 | "text": [ 108 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n", 109 | "11493376/11490434 [==============================] - 0s 0us/step\n", 110 | "Train on 60000 samples\n", 111 | "Epoch 1/5\n", 112 | "60000/60000 [==============================] - 5s 76us/sample - loss: 0.2954 - accuracy: 0.9136\n", 113 | "Epoch 2/5\n", 114 | "60000/60000 [==============================] - 4s 73us/sample - loss: 0.1424 - accuracy: 0.9572\n", 115 | "Epoch 3/5\n", 116 | "60000/60000 [==============================] - 4s 72us/sample - loss: 0.1091 - accuracy: 0.9671\n", 117 | "Epoch 4/5\n", 118 | "60000/60000 [==============================] - 4s 72us/sample - loss: 0.0880 - accuracy: 0.9729\n", 119 | "Epoch 5/5\n", 120 | "60000/60000 [==============================] - 4s 72us/sample - loss: 0.0769 - accuracy: 0.9760\n", 121 | "10000/10000 [==============================] - 0s 27us/sample - loss: 0.0718 - accuracy: 0.9777\n" 122 | ] 123 | }, 124 | { 125 | "data": { 126 | "text/plain": [ 127 | "[0.07182098130043596, 0.9777]" 128 | ] 129 | }, 130 | "execution_count": 3, 131 | "metadata": {}, 132 | "output_type": "execute_result" 133 | } 134 | ], 135 | "source": [ 136 | "mnist = tf.keras.datasets.mnist\n", 137 | "\n", 138 | "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n", 139 | "x_train, x_test = x_train / 255.0, x_test / 255.0\n", 140 | "\n", 141 | "model = tf.keras.models.Sequential([\n", 142 | " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", 143 | " SimpleDense(128, activation='relu'),\n", 144 | " tf.keras.layers.Dropout(0.2),\n", 145 | " tf.keras.layers.Dense(10, activation='softmax')\n", 146 | "])\n", 147 | "\n", 148 | "model.compile(optimizer='adam',\n", 149 | " loss='sparse_categorical_crossentropy',\n", 150 | " metrics=['accuracy'])\n", 151 | "\n", 152 | "model.fit(x_train, y_train, epochs=5)\n", 153 | "model.evaluate(x_test, y_test)" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [] 162 | } 163 | ], 164 | "metadata": { 165 | "colab": { 166 | "authorship_tag": "ABX9TyNuZnKsU8CXVWlkaCl0Um5Q", 167 | "collapsed_sections": [], 168 | "include_colab_link": true, 169 | "name": "CustomWithActivation.ipynb", 170 | "provenance": [] 171 | }, 172 | "kernelspec": { 173 | "display_name": "Python 3", 174 | "language": "python", 175 | "name": "python3" 176 | }, 177 | "language_info": { 178 | "codemirror_mode": { 179 | "name": "ipython", 180 | "version": 3 181 | }, 182 | "file_extension": ".py", 183 | "mimetype": "text/x-python", 184 | "name": "python", 185 | "nbconvert_exporter": "python", 186 | "pygments_lexer": "ipython3", 187 | "version": "3.7.6" 188 | } 189 | }, 190 | "nbformat": 4, 191 | "nbformat_minor": 4 192 | } 193 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-3/utils.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.resource_variable_ops import ResourceVariable 3 | import numpy as np 4 | 5 | def test_loop(test_cases): 6 | 7 | success = 0 8 | fails = 0 9 | 10 | for test_case in test_cases: 11 | try: 12 | assert test_case["result"] == test_case["expected"] 13 | success += 1 14 | 15 | except: 16 | fails += 1 17 | print(f'{test_case["name"]}: {test_case["error_message"]}\nExpected: {test_case["expected"]}\nResult: {test_case["result"]}\n') 18 | 19 | if fails == 0: 20 | print("\033[92m All public tests passed") 21 | 22 | else: 23 | print('\033[92m', success," Tests passed") 24 | print('\033[91m', fails, " Tests failed") 25 | raise Exception(test_case["error_message"]) 26 | 27 | 28 | def test_simple_quadratic(SimpleQuadratic): 29 | 30 | expected_units = 128 31 | expected_activation_function = tf.keras.activations.relu 32 | expected_activation_string = 'relu' 33 | shape_0 = 8 34 | shape_1 = 2 35 | 36 | test_layer = SimpleQuadratic(units=expected_units, activation=expected_activation_string) 37 | 38 | test_layer.build((shape_0, shape_1)) 39 | 40 | test_inputs = tf.random.uniform((shape_0, shape_1)) 41 | 42 | test_call_value = test_layer.call(test_inputs) 43 | 44 | a_type = type(test_layer.a) 45 | b_type = type(test_layer.b) 46 | c_type = type(test_layer.c) 47 | 48 | test_layer_forced_weights = SimpleQuadratic(units=1, activation=None) 49 | test_layer_forced_weights.a = tf.constant([2.0], dtype='float32', shape=(1,1)) 50 | test_layer_forced_weights.b = tf.constant([2.0], dtype='float32', shape=(1,1)) 51 | test_layer_forced_weights.c = tf.constant([2.0], dtype='float32', shape=(1,1)) 52 | test_layer_forced_weights_inputs = tf.constant([4.0], dtype='float32', shape=(1,1)) 53 | test_layer_forced_weights_expected_output = 42.0 54 | 55 | test_cases = [ 56 | { 57 | "name": "units_check", 58 | "result": test_layer.units, 59 | "expected": expected_units, 60 | "error_message": f'Incorrect number of units.' 61 | }, 62 | { 63 | "name": "activations_check", 64 | "result": test_layer.activation, 65 | "expected": tf.keras.activations.relu, 66 | "error_message": "Got different activation function." 67 | }, 68 | { 69 | "name": "a_type_check", 70 | "result": a_type, 71 | "expected": ResourceVariable, 72 | "error_message": f'State variable a is of different type. Expected ResourceVariable but got {a_type}' 73 | }, 74 | { 75 | "name": "b_type_check", 76 | "result": b_type, 77 | "expected": ResourceVariable, 78 | "error_message": f'State variable b is of different type. Expected ResourceVariable but got {b_type}' 79 | }, 80 | { 81 | "name": "c_type_check", 82 | "result": c_type, 83 | "expected": ResourceVariable, 84 | "error_message": f'State variable c is of different type. Expected ResourceVariable but got {c_type}' 85 | }, 86 | { 87 | "name": "a_initializer_check", 88 | "result": test_layer.a.numpy().sum() != 0, 89 | "expected": True, 90 | "error_message": f'State variable a is not initialized randomly. Please check initializer used.' 91 | }, 92 | { 93 | "name": "b_initializer_check", 94 | "result": test_layer.b.numpy().sum() != 0, 95 | "expected": True, 96 | "error_message": f'State variable b is not initialized randomly. Please check initializer used.' 97 | }, 98 | { 99 | "name": "c_initializer_check", 100 | "result": test_layer.c.numpy().sum() == 0, 101 | "expected": True, 102 | "error_message": f'State variable c is not initialized to zeroes. Please check initializer used.' 103 | }, 104 | { 105 | "name": "output_check", 106 | "result": test_layer_forced_weights.call(test_layer_forced_weights_inputs).numpy()[0][0], 107 | "expected": test_layer_forced_weights_expected_output, 108 | "error_message": f'Expected output is incorrect. Please check operations in the call() method.' 109 | } 110 | ] 111 | 112 | test_loop(test_cases) 113 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-4/C1_W4_Lab_2_resnet-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "colab_type": "text", 7 | "id": "view-in-github" 8 | }, 9 | "source": [ 10 | "\"Open" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": {}, 16 | "source": [ 17 | "# Ungraded Lab: Implementing ResNet\n", 18 | "\n", 19 | "In this lab, you will continue exploring Model subclassing by building a more complex architecture. \n", 20 | "\n", 21 | "[Residual Networks](https://arxiv.org/abs/1512.03385) make use of skip connections to make deep models easier to train. \n", 22 | "- There are branches as well as many repeating blocks of layers in this type of network. \n", 23 | "- You can define a model class to help organize this more complex code, and to make it easier to re-use your code when building the model.\n", 24 | "- As before, you will inherit from the [Model class](https://keras.io/api/models/model/) so that you can make use of the other built-in methods that Keras provides." 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "## Imports" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": { 38 | "colab": {}, 39 | "colab_type": "code", 40 | "id": "CmI9MQA6Z72_" 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "try:\n", 45 | " # %tensorflow_version only exists in Colab.\n", 46 | " %tensorflow_version 2.x\n", 47 | "except Exception:\n", 48 | " pass\n", 49 | "\n", 50 | "import tensorflow as tf\n", 51 | "import tensorflow_datasets as tfds\n", 52 | "from tensorflow.keras.layers import Layer" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": {}, 58 | "source": [ 59 | "## Implement Model subclasses\n", 60 | "\n", 61 | "As shown in the lectures, you will first implement the Identity Block which contains the skip connections (i.e. the `add()` operation below. This will also inherit the Model class and implement the `__init__()` and `call()` methods." 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 3, 67 | "metadata": { 68 | "colab": {}, 69 | "colab_type": "code", 70 | "id": "-FIkYUttchv5" 71 | }, 72 | "outputs": [], 73 | "source": [ 74 | "class IdentityBlock(tf.keras.Model):\n", 75 | " def __init__(self, filters, kernel_size):\n", 76 | " super(IdentityBlock, self).__init__(name='')\n", 77 | "\n", 78 | " self.conv1 = tf.keras.layers.Conv2D(filters, kernel_size, padding='same')\n", 79 | " self.bn1 = tf.keras.layers.BatchNormalization()\n", 80 | "\n", 81 | " self.conv2 = tf.keras.layers.Conv2D(filters, kernel_size, padding='same')\n", 82 | " self.bn2 = tf.keras.layers.BatchNormalization()\n", 83 | "\n", 84 | " self.act = tf.keras.layers.Activation('relu')\n", 85 | " self.add = tf.keras.layers.Add()\n", 86 | " \n", 87 | " def call(self, input_tensor):\n", 88 | " x = self.conv1(input_tensor)\n", 89 | " x = self.bn1(x)\n", 90 | " x = self.act(x)\n", 91 | "\n", 92 | " x = self.conv2(x)\n", 93 | " x = self.bn2(x)\n", 94 | " x = self.act(x)\n", 95 | "\n", 96 | " x = self.add([x, input_tensor])\n", 97 | " x = self.act(x)\n", 98 | " return x" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "From there, you can build the rest of the ResNet model. \n", 106 | "- You will call your `IdentityBlock` class two times below and that takes care of inserting those blocks of layers into this network." 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 4, 112 | "metadata": { 113 | "colab": {}, 114 | "colab_type": "code", 115 | "id": "YnMkmeecxw28" 116 | }, 117 | "outputs": [], 118 | "source": [ 119 | "class ResNet(tf.keras.Model):\n", 120 | " def __init__(self, num_classes):\n", 121 | " super(ResNet, self).__init__()\n", 122 | " self.conv = tf.keras.layers.Conv2D(64, 7, padding='same')\n", 123 | " self.bn = tf.keras.layers.BatchNormalization()\n", 124 | " self.act = tf.keras.layers.Activation('relu')\n", 125 | " self.max_pool = tf.keras.layers.MaxPool2D((3, 3))\n", 126 | "\n", 127 | " # Use the Identity blocks that you just defined\n", 128 | " self.id1a = IdentityBlock(64, 3)\n", 129 | " self.id1b = IdentityBlock(64, 3)\n", 130 | "\n", 131 | " self.global_pool = tf.keras.layers.GlobalAveragePooling2D()\n", 132 | " self.classifier = tf.keras.layers.Dense(num_classes, activation='softmax')\n", 133 | "\n", 134 | " def call(self, inputs):\n", 135 | " x = self.conv(inputs)\n", 136 | " x = self.bn(x)\n", 137 | " x = self.act(x)\n", 138 | " x = self.max_pool(x)\n", 139 | "\n", 140 | " # insert the identity blocks in the middle of the network\n", 141 | " x = self.id1a(x)\n", 142 | " x = self.id1b(x)\n", 143 | "\n", 144 | " x = self.global_pool(x)\n", 145 | " return self.classifier(x)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": {}, 151 | "source": [ 152 | "## Training the Model\n", 153 | "\n", 154 | "As mentioned before, inheriting the Model class allows you to make use of the other APIs that Keras provides, such as:\n", 155 | "- training\n", 156 | "- serialization\n", 157 | "- evaluation\n", 158 | "\n", 159 | "You can instantiate a Resnet object and train it as usual like below:\n", 160 | "\n", 161 | "**Note**: If you have issues with training in the Coursera lab environment, you can also run this in Colab using the \"open in colab\" badge link." 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 5, 167 | "metadata": { 168 | "colab": {}, 169 | "colab_type": "code", 170 | "id": "6dMHKPz_dIc8" 171 | }, 172 | "outputs": [ 173 | { 174 | "name": "stdout", 175 | "output_type": "stream", 176 | "text": [ 177 | "1875/1875 [==============================] - 242s 129ms/step - loss: 0.1407 - accuracy: 0.9644\n" 178 | ] 179 | }, 180 | { 181 | "data": { 182 | "text/plain": [ 183 | "" 184 | ] 185 | }, 186 | "execution_count": 5, 187 | "metadata": {}, 188 | "output_type": "execute_result" 189 | } 190 | ], 191 | "source": [ 192 | "# utility function to normalize the images and return (image, label) pairs.\n", 193 | "def preprocess(features):\n", 194 | " return tf.cast(features['image'], tf.float32) / 255., features['label']\n", 195 | "\n", 196 | "# create a ResNet instance with 10 output units for MNIST\n", 197 | "resnet = ResNet(10)\n", 198 | "resnet.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n", 199 | "\n", 200 | "# load and preprocess the dataset\n", 201 | "dataset = tfds.load('mnist', split=tfds.Split.TRAIN, data_dir='./data')\n", 202 | "dataset = dataset.map(preprocess).batch(32)\n", 203 | "\n", 204 | "# train the model.\n", 205 | "resnet.fit(dataset, epochs=1)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": {}, 212 | "outputs": [], 213 | "source": [] 214 | } 215 | ], 216 | "metadata": { 217 | "colab": { 218 | "collapsed_sections": [], 219 | "include_colab_link": true, 220 | "name": "ResNetExample.ipynb", 221 | "provenance": [] 222 | }, 223 | "kernelspec": { 224 | "display_name": "Python 3", 225 | "language": "python", 226 | "name": "python3" 227 | }, 228 | "language_info": { 229 | "codemirror_mode": { 230 | "name": "ipython", 231 | "version": 3 232 | }, 233 | "file_extension": ".py", 234 | "mimetype": "text/x-python", 235 | "name": "python", 236 | "nbconvert_exporter": "python", 237 | "pygments_lexer": "ipython3", 238 | "version": "3.7.6" 239 | } 240 | }, 241 | "nbformat": 4, 242 | "nbformat_minor": 4 243 | } 244 | -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-4/data/mnist/3.0.1/dataset_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "citation": "@article{lecun2010mnist,\n title={MNIST handwritten digit database},\n author={LeCun, Yann and Cortes, Corinna and Burges, CJ},\n journal={ATT Labs [Online]. Available: http://yann. lecun. com/exdb/mnist},\n volume={2},\n year={2010}\n}\n", 3 | "description": "The MNIST database of handwritten digits.", 4 | "downloadSize": "11594722", 5 | "location": { 6 | "urls": [ 7 | "http://yann.lecun.com/exdb/mnist/" 8 | ] 9 | }, 10 | "name": "mnist", 11 | "schema": { 12 | "feature": [ 13 | { 14 | "name": "image", 15 | "shape": { 16 | "dim": [ 17 | { 18 | "size": "28" 19 | }, 20 | { 21 | "size": "28" 22 | }, 23 | { 24 | "size": "1" 25 | } 26 | ] 27 | }, 28 | "type": "INT" 29 | }, 30 | { 31 | "name": "label", 32 | "type": "INT" 33 | } 34 | ] 35 | }, 36 | "splits": [ 37 | { 38 | "name": "test", 39 | "numBytes": "3150056", 40 | "numShards": "1", 41 | "shardLengths": [ 42 | "10000" 43 | ], 44 | "statistics": { 45 | "features": [ 46 | { 47 | "name": "image", 48 | "numStats": { 49 | "commonStats": { 50 | "numNonMissing": "10000" 51 | }, 52 | "max": 255.0 53 | } 54 | }, 55 | { 56 | "name": "label", 57 | "numStats": { 58 | "commonStats": { 59 | "numNonMissing": "10000" 60 | }, 61 | "max": 9.0 62 | } 63 | } 64 | ], 65 | "numExamples": "10000" 66 | } 67 | }, 68 | { 69 | "name": "train", 70 | "numBytes": "18872042", 71 | "numShards": "1", 72 | "shardLengths": [ 73 | "60000" 74 | ], 75 | "statistics": { 76 | "features": [ 77 | { 78 | "name": "image", 79 | "numStats": { 80 | "commonStats": { 81 | "numNonMissing": "60000" 82 | }, 83 | "max": 255.0 84 | } 85 | }, 86 | { 87 | "name": "label", 88 | "numStats": { 89 | "commonStats": { 90 | "numNonMissing": "60000" 91 | }, 92 | "max": 9.0 93 | } 94 | } 95 | ], 96 | "numExamples": "60000" 97 | } 98 | } 99 | ], 100 | "supervisedKeys": { 101 | "input": "image", 102 | "output": "label" 103 | }, 104 | "version": "3.0.1" 105 | } -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-4/data/mnist/3.0.1/image.image.json: -------------------------------------------------------------------------------- 1 | {"encoding_format": "png", "shape": [28, 28, 1]} -------------------------------------------------------------------------------- /1-custom_models_layers_losses/week-4/data/mnist/3.0.1/mnist-test.tfrecord-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/1-custom_models_layers_losses/week-4/data/mnist/3.0.1/mnist-test.tfrecord-00000-of-00001 -------------------------------------------------------------------------------- /2-custom_and_distributed_training/pratice_py/1_tf_var_layer.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Layer 3 | 4 | class Variables(Layer) : 5 | 6 | def __init__(self): 7 | super().__init__() 8 | self.one = tf.Variable(100) 9 | self.two = [tf.Variable(x) for x in range(4)] 10 | 11 | 12 | var = Variables() 13 | 14 | print([varss.numpy() for varss in var.variables]) 15 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/pratice_py/2_gradient_tape.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0]) 5 | ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0]) 6 | 7 | #trainable variables 8 | w = tf.Variable(np.random.random(), trainable=True) 9 | b = tf.Variable(np.random.random(), trainable=True) 10 | 11 | 12 | # loss 13 | def simple_loss(real_y, pred_y) : 14 | return tf.abs(real_y - pred_y) 15 | 16 | learning_rate = 0.001 17 | 18 | def fit_data(x_real, y_real): 19 | 20 | with tf.GradientTape(persistent=True) as tape: 21 | pred_y = w * x_real + b 22 | reg_loss = simple_loss(y_real, pred_y) 23 | 24 | w_gradient = tape.gradient(reg_loss, w) 25 | b_gradient = tape.gradient(reg_loss, b) 26 | 27 | w.assign_sub(w_gradient * learning_rate) 28 | b.assign_sub(b_gradient * learning_rate) 29 | 30 | 31 | for _ in range(500) : 32 | fit_data(xs, ys) 33 | 34 | print(f"w = {w.numpy()} + b = {b.numpy()}") -------------------------------------------------------------------------------- /2-custom_and_distributed_training/pratice_py/3_gradient_calcuation.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | # calcuate (d/dw) w-square = 2w <-- formula 4 | 5 | x = tf.Variable([1.0]) 6 | 7 | with tf.GradientTape() as tape : 8 | grad = x * x 9 | 10 | 11 | tape.gradient(grad, x) 12 | 13 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/pratice_py/4_into_customloop.py: -------------------------------------------------------------------------------- 1 | # class loop() : 2 | 3 | # def __init__(self) : 4 | 5 | # self.w = 8 6 | # self.b = 2 7 | 8 | # def __call__(self) : 9 | 10 | # return self.w * self.b 11 | 12 | 13 | # l = loop() 14 | 15 | # print(l()) 16 | # print(vars(l).keys()) 17 | # print(vars(l).values()) 18 | 19 | # print(dir(loop)) 20 | 21 | class ren() : 22 | def __init__(self, a : int, b : float ) : 23 | self.a = a 24 | self.b = b 25 | print(self.a, self.b) 26 | 27 | 28 | 29 | 30 | r = ren(a = 44.3, b = 45) 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/pratice_py/5_custom_1.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.keras.layers import Dense, Flatten, Input 3 | from tensorflow.keras.models import Model 4 | 5 | 6 | def base_model() : 7 | 8 | inputs = Input(shape = (), name = 'clothing') 9 | x = Dense(64, activation= 'relu', name = 'dense_1')(inputs) 10 | x = Dense(64, activation= 'relu', name = 'dense_2')(x) 11 | output = Dense(10, activation= 'softmax', name = 'output') 12 | 13 | model = Model(inputs = inputs, outputs = output) 14 | return model 15 | 16 | 17 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-1/C2W1_Assignment.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Basic Tensor operations and GradientTape.\n", 8 | "\n", 9 | "In this graded assignment, you will perform different tensor operations as well as use [GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape). These are important building blocks for the next parts of this course so it's important to master the basics. Let's begin!" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": { 16 | "colab": {}, 17 | "colab_type": "code", 18 | "id": "jqev488WJ9-R" 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "import tensorflow as tf\n", 23 | "import numpy as np" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "## Exercise 1 - [tf.constant]((https://www.tensorflow.org/api_docs/python/tf/constant))\n", 31 | "\n", 32 | "Creates a constant tensor from a tensor-like object. " 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 2, 38 | "metadata": { 39 | "colab": {}, 40 | "colab_type": "code", 41 | "id": "MYdVyiSoLPgO" 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "# Convert NumPy array to Tensor using `tf.constant`\n", 46 | "def tf_constant(array):\n", 47 | " \"\"\"\n", 48 | " Args:\n", 49 | " array (numpy.ndarray): tensor-like array.\n", 50 | "\n", 51 | " Returns:\n", 52 | " tensorflow.python.framework.ops.EagerTensor: tensor.\n", 53 | " \"\"\"\n", 54 | " ### START CODE HERE ###\n", 55 | " tf_constant_array = tf.constant(array) \n", 56 | " ### END CODE HERE ###\n", 57 | " return tf_constant_array" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 3, 63 | "metadata": {}, 64 | "outputs": [ 65 | { 66 | "data": { 67 | "text/plain": [ 68 | "" 69 | ] 70 | }, 71 | "execution_count": 3, 72 | "metadata": {}, 73 | "output_type": "execute_result" 74 | } 75 | ], 76 | "source": [ 77 | "tmp_array = np.arange(1,10)\n", 78 | "x = tf_constant(tmp_array)\n", 79 | "x\n", 80 | "\n", 81 | "# Expected output:\n", 82 | "# " 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "Note that for future docstrings, the type `EagerTensor` will be used as a shortened version of `tensorflow.python.framework.ops.EagerTensor`." 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": {}, 95 | "source": [ 96 | "## Exercise 2 - [tf.square](https://www.tensorflow.org/api_docs/python/tf/math/square)\n", 97 | "\n", 98 | "Computes the square of a tensor element-wise." 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 4, 104 | "metadata": { 105 | "colab": {}, 106 | "colab_type": "code", 107 | "id": "W6BTwNJCLjV8" 108 | }, 109 | "outputs": [], 110 | "source": [ 111 | "# Square the input tensor\n", 112 | "def tf_square(array):\n", 113 | " \"\"\"\n", 114 | " Args:\n", 115 | " array (numpy.ndarray): tensor-like array.\n", 116 | "\n", 117 | " Returns:\n", 118 | " EagerTensor: tensor.\n", 119 | " \"\"\"\n", 120 | " # make sure it's a tensor\n", 121 | " array = tf.constant(array)\n", 122 | " \n", 123 | " ### START CODE HERE ###\n", 124 | " tf_squared_array = array ** 2 \n", 125 | " ### END CODE HERE ###\n", 126 | " return tf_squared_array" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 5, 132 | "metadata": {}, 133 | "outputs": [ 134 | { 135 | "data": { 136 | "text/plain": [ 137 | "" 138 | ] 139 | }, 140 | "execution_count": 5, 141 | "metadata": {}, 142 | "output_type": "execute_result" 143 | } 144 | ], 145 | "source": [ 146 | "tmp_array = tf.constant(np.arange(1, 10))\n", 147 | "x = tf_square(tmp_array)\n", 148 | "x\n", 149 | "\n", 150 | "# Expected output:\n", 151 | "# " 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "## Exercise 3 - [tf.reshape](https://www.tensorflow.org/api_docs/python/tf/reshape)\n", 159 | "\n", 160 | "Reshapes a tensor." 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 6, 166 | "metadata": { 167 | "colab": {}, 168 | "colab_type": "code", 169 | "id": "7nzBSX8-L0Xt" 170 | }, 171 | "outputs": [], 172 | "source": [ 173 | "# Reshape tensor into a 3 x 3 matrix\n", 174 | "def tf_reshape(array, shape):\n", 175 | " \"\"\"\n", 176 | " Args:\n", 177 | " array (EagerTensor): tensor to reshape.\n", 178 | " shape (tuple): desired shape.\n", 179 | "\n", 180 | " Returns:\n", 181 | " EagerTensor: reshaped tensor.\n", 182 | " \"\"\"\n", 183 | " # make sure it's a tensor\n", 184 | " array = tf.constant(array)\n", 185 | " ### START CODE HERE ###\n", 186 | " tf_reshaped_array = tf.reshape(array, shape = shape) \n", 187 | " ### END CODE HERE ###\n", 188 | " return tf_reshaped_array" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 7, 194 | "metadata": {}, 195 | "outputs": [ 196 | { 197 | "data": { 198 | "text/plain": [ 199 | "" 203 | ] 204 | }, 205 | "execution_count": 7, 206 | "metadata": {}, 207 | "output_type": "execute_result" 208 | } 209 | ], 210 | "source": [ 211 | "# Check your function\n", 212 | "tmp_array = np.array([1,2,3,4,5,6,7,8,9])\n", 213 | "# Check that your function reshapes a vector into a matrix\n", 214 | "x = tf_reshape(tmp_array, (3, 3))\n", 215 | "x\n", 216 | "\n", 217 | "# Expected output:\n", 218 | "# " 271 | ] 272 | }, 273 | "execution_count": 9, 274 | "metadata": {}, 275 | "output_type": "execute_result" 276 | } 277 | ], 278 | "source": [ 279 | "# Check your function\n", 280 | "tmp_array = [1,2,3,4]\n", 281 | "x = tf_cast(tmp_array, tf.float32)\n", 282 | "x\n", 283 | "\n", 284 | "# Expected output:\n", 285 | "# " 286 | ] 287 | }, 288 | { 289 | "cell_type": "markdown", 290 | "metadata": {}, 291 | "source": [ 292 | "## Exercise 5 - [tf.multiply](https://www.tensorflow.org/api_docs/python/tf/multiply)\n", 293 | "\n", 294 | "Returns an element-wise x * y." 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 10, 300 | "metadata": { 301 | "colab": {}, 302 | "colab_type": "code", 303 | "id": "ivepGtD5MKP5" 304 | }, 305 | "outputs": [], 306 | "source": [ 307 | "# Multiply tensor1 and tensor2\n", 308 | "def tf_multiply(tensor1, tensor2):\n", 309 | " \"\"\"\n", 310 | " Args:\n", 311 | " tensor1 (EagerTensor): a tensor.\n", 312 | " tensor2 (EagerTensor): another tensor.\n", 313 | "\n", 314 | " Returns:\n", 315 | " EagerTensor: resulting tensor.\n", 316 | " \"\"\"\n", 317 | " # make sure these are tensors\n", 318 | " tensor1 = tf.constant(tensor1)\n", 319 | " tensor2 = tf.constant(tensor2)\n", 320 | " \n", 321 | " ### START CODE HERE ###\n", 322 | " product = tf.multiply(tensor1, tensor2) \n", 323 | " ### END CODE HERE ###\n", 324 | " return product\n" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": 11, 330 | "metadata": {}, 331 | "outputs": [ 332 | { 333 | "data": { 334 | "text/plain": [ 335 | "" 338 | ] 339 | }, 340 | "execution_count": 11, 341 | "metadata": {}, 342 | "output_type": "execute_result" 343 | } 344 | ], 345 | "source": [ 346 | "# Check your function\n", 347 | "tmp_1 = tf.constant(np.array([[1,2],[3,4]]))\n", 348 | "tmp_2 = tf.constant(np.array(2))\n", 349 | "result = tf_multiply(tmp_1, tmp_2)\n", 350 | "result\n", 351 | "\n", 352 | "# Expected output:\n", 353 | "# " 356 | ] 357 | }, 358 | { 359 | "cell_type": "markdown", 360 | "metadata": {}, 361 | "source": [ 362 | "## Exercise 6 - [tf.add](https://www.tensorflow.org/api_docs/python/tf/add)\n", 363 | "\n", 364 | "Returns x + y element-wise." 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": 12, 370 | "metadata": { 371 | "colab": {}, 372 | "colab_type": "code", 373 | "id": "BVlntdYnMboh" 374 | }, 375 | "outputs": [], 376 | "source": [ 377 | "# Add tensor1 and tensor2\n", 378 | "def tf_add(tensor1, tensor2):\n", 379 | " \"\"\"\n", 380 | " Args:\n", 381 | " tensor1 (EagerTensor): a tensor.\n", 382 | " tensor2 (EagerTensor): another tensor.\n", 383 | "\n", 384 | " Returns:\n", 385 | " EagerTensor: resulting tensor.\n", 386 | " \"\"\"\n", 387 | " # make sure these are tensors\n", 388 | " tensor1 = tf.constant(tensor1)\n", 389 | " tensor2 = tf.constant(tensor2)\n", 390 | " \n", 391 | " ### START CODE HERE ###\n", 392 | " total = tf.add(tensor1, tensor2) \n", 393 | " ### END CODE HERE ###\n", 394 | " return total" 395 | ] 396 | }, 397 | { 398 | "cell_type": "code", 399 | "execution_count": 13, 400 | "metadata": {}, 401 | "outputs": [ 402 | { 403 | "data": { 404 | "text/plain": [ 405 | "" 406 | ] 407 | }, 408 | "execution_count": 13, 409 | "metadata": {}, 410 | "output_type": "execute_result" 411 | } 412 | ], 413 | "source": [ 414 | "# Check your function\n", 415 | "tmp_1 = tf.constant(np.array([1, 2, 3]))\n", 416 | "tmp_2 = tf.constant(np.array([4, 5, 6]))\n", 417 | "tf_add(tmp_1, tmp_2)\n", 418 | "\n", 419 | "# Expected output:\n", 420 | "# " 421 | ] 422 | }, 423 | { 424 | "cell_type": "markdown", 425 | "metadata": { 426 | "colab_type": "text", 427 | "id": "9EN0W15EWNjD" 428 | }, 429 | "source": [ 430 | "## Exercise 7 - Gradient Tape\n", 431 | "\n", 432 | "Implement the function `tf_gradient_tape` by replacing the instances of `None` in the code below. The instructions are given in the code comments.\n", 433 | "\n", 434 | "You can review the [docs](https://www.tensorflow.org/api_docs/python/tf/GradientTape) or revisit the lectures to complete this task." 435 | ] 436 | }, 437 | { 438 | "cell_type": "code", 439 | "execution_count": 14, 440 | "metadata": { 441 | "colab": {}, 442 | "colab_type": "code", 443 | "id": "p3K94BWZM6nW" 444 | }, 445 | "outputs": [], 446 | "source": [ 447 | "def tf_gradient_tape(x):\n", 448 | " \"\"\"\n", 449 | " Args:\n", 450 | " x (EagerTensor): a tensor.\n", 451 | "\n", 452 | " Returns:\n", 453 | " EagerTensor: Derivative of z with respect to the input tensor x.\n", 454 | " \"\"\"\n", 455 | " with tf.GradientTape() as t:\n", 456 | " \n", 457 | " ### START CODE HERE ###\n", 458 | " # Record the actions performed on tensor x with `watch`\n", 459 | " t.watch(x) \n", 460 | "\n", 461 | " # Define a polynomial of form 3x^3 - 2x^2 + x\n", 462 | " y = 3*(x**3) - 2*(x**2) + x \n", 463 | "\n", 464 | " # Obtain the sum of the elements in variable y\n", 465 | " z = tf.reduce_sum(y) \n", 466 | " \n", 467 | " # Get the derivative of z with respect to the original input tensor x\n", 468 | " dz_dx = t.gradient(z,x)\n", 469 | " ### END CODE HERE\n", 470 | " \n", 471 | " return dz_dx" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": 15, 477 | "metadata": {}, 478 | "outputs": [ 479 | { 480 | "data": { 481 | "text/plain": [ 482 | "29.0" 483 | ] 484 | }, 485 | "execution_count": 15, 486 | "metadata": {}, 487 | "output_type": "execute_result" 488 | } 489 | ], 490 | "source": [ 491 | "# Check your function\n", 492 | "tmp_x = tf.constant(2.0)\n", 493 | "dz_dx = tf_gradient_tape(tmp_x)\n", 494 | "result = dz_dx.numpy()\n", 495 | "result\n", 496 | "\n", 497 | "# Expected output:\n", 498 | "# 29.0" 499 | ] 500 | }, 501 | { 502 | "cell_type": "markdown", 503 | "metadata": {}, 504 | "source": [ 505 | "**Congratulations on finishing this week's assignment!**\n", 506 | "\n", 507 | "**Keep it up!**" 508 | ] 509 | } 510 | ], 511 | "metadata": { 512 | "coursera": { 513 | "schema_names": [ 514 | "TF3C2W1-1", 515 | "TF3C2W1-2", 516 | "TF3C2W1-3", 517 | "TF3C2W1-4", 518 | "TF3C2W1-5", 519 | "TF3C2W1-6", 520 | "TF3C2W1-7" 521 | ] 522 | }, 523 | "kernelspec": { 524 | "display_name": "Python 3", 525 | "language": "python", 526 | "name": "python3" 527 | }, 528 | "language_info": { 529 | "codemirror_mode": { 530 | "name": "ipython", 531 | "version": 3 532 | }, 533 | "file_extension": ".py", 534 | "mimetype": "text/x-python", 535 | "name": "python", 536 | "nbconvert_exporter": "python", 537 | "pygments_lexer": "ipython3", 538 | "version": "3.7.6" 539 | } 540 | }, 541 | "nbformat": 4, 542 | "nbformat_minor": 4 543 | } 544 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-1/C2_W1_Lab_1_basic-tensors.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Basic Tensors" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this ungraded lab, you will try some of the basic operations you can perform on tensors." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "## Imports" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": { 28 | "id": "jqev488WJ9-R" 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "try:\n", 33 | " # %tensorflow_version only exists in Colab.\n", 34 | " %tensorflow_version 2.x\n", 35 | "except Exception:\n", 36 | " pass\n", 37 | "\n", 38 | "import tensorflow as tf\n", 39 | "import numpy as np" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": { 45 | "id": "sIIXMZboUw-P" 46 | }, 47 | "source": [ 48 | "## Exercise on basic Tensor operations\n", 49 | "\n", 50 | "Lets create a single dimension numpy array on which you can perform some operation. You'll make an array of size 25, holding values from 0 to 24." 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 2, 56 | "metadata": { 57 | "id": "GkBZyS8hKNXX" 58 | }, 59 | "outputs": [ 60 | { 61 | "data": { 62 | "text/plain": [ 63 | "array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,\n", 64 | " 17, 18, 19, 20, 21, 22, 23, 24])" 65 | ] 66 | }, 67 | "execution_count": 2, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | } 71 | ], 72 | "source": [ 73 | "# Create a 1D uint8 NumPy array comprising of first 25 natural numbers\n", 74 | "x = np.arange(0, 25)\n", 75 | "x" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "Now that you have your 1-D array, next you'll change that array into a `tensor`. After running the code block below, take a moment to inspect the information of your tensor." 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 3, 88 | "metadata": { 89 | "id": "MYdVyiSoLPgO" 90 | }, 91 | "outputs": [ 92 | { 93 | "data": { 94 | "text/plain": [ 95 | "" 98 | ] 99 | }, 100 | "execution_count": 3, 101 | "metadata": {}, 102 | "output_type": "execute_result" 103 | } 104 | ], 105 | "source": [ 106 | "# Convert NumPy array to Tensor using `tf.constant`\n", 107 | "x = tf.constant(x)\n", 108 | "x" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "As the first operation to be performed, you'll square (element-wise) all the values in the tensor `x`" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 4, 121 | "metadata": { 122 | "id": "W6BTwNJCLjV8" 123 | }, 124 | "outputs": [ 125 | { 126 | "data": { 127 | "text/plain": [ 128 | "" 131 | ] 132 | }, 133 | "execution_count": 4, 134 | "metadata": {}, 135 | "output_type": "execute_result" 136 | } 137 | ], 138 | "source": [ 139 | "# Square the input tensor x\n", 140 | "x = tf.square(x)\n", 141 | "x" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "metadata": {}, 147 | "source": [ 148 | "One feature of tensors is that they can be reshaped. When reshpaing, make sure you consider dimensions that will include all of the values of the tensor." 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 6, 154 | "metadata": { 155 | "id": "7nzBSX8-L0Xt" 156 | }, 157 | "outputs": [ 158 | { 159 | "data": { 160 | "text/plain": [ 161 | "" 167 | ] 168 | }, 169 | "execution_count": 6, 170 | "metadata": {}, 171 | "output_type": "execute_result" 172 | } 173 | ], 174 | "source": [ 175 | "# Reshape tensor x into a 5 x 5 matrix. \n", 176 | "x = tf.reshape(x, shape = (5, 5))\n", 177 | "x" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "Notice that you'll get an error message if you choose a shape that cannot be exactly filled with the values of the given tensor. \n", 185 | "* Run the cell below and look at the error message\n", 186 | "* Try to change the tuple that is passed to `shape` to avoid an error." 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 8, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "data": { 196 | "text/plain": [ 197 | "" 200 | ] 201 | }, 202 | "execution_count": 8, 203 | "metadata": {}, 204 | "output_type": "execute_result" 205 | } 206 | ], 207 | "source": [ 208 | "# Try this and look at the error\n", 209 | "# Try to change the input to `shape` to avoid an error\n", 210 | "tmp = tf.constant([1,2,3,4])\n", 211 | "tf.reshape(tmp, shape=(2,2))" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "Like reshaping, you can also change the data type of the values within the tensor. Run the cell below to change the data type from `int` to `float`" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 9, 224 | "metadata": { 225 | "id": "VoT-jiAIL8x5" 226 | }, 227 | "outputs": [ 228 | { 229 | "data": { 230 | "text/plain": [ 231 | "" 237 | ] 238 | }, 239 | "execution_count": 9, 240 | "metadata": {}, 241 | "output_type": "execute_result" 242 | } 243 | ], 244 | "source": [ 245 | "# Cast tensor x into float32. Notice the change in the dtype.\n", 246 | "x = tf.cast(x, tf.float32)\n", 247 | "x" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "metadata": {}, 253 | "source": [ 254 | "Next, you'll create a single value float tensor by the help of which you'll see `broadcasting` in action" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 10, 260 | "metadata": {}, 261 | "outputs": [ 262 | { 263 | "data": { 264 | "text/plain": [ 265 | "" 266 | ] 267 | }, 268 | "execution_count": 10, 269 | "metadata": {}, 270 | "output_type": "execute_result" 271 | } 272 | ], 273 | "source": [ 274 | "# Let's define a constant and see how broadcasting works in the following cell.\n", 275 | "y = tf.constant(2, dtype=tf.float32)\n", 276 | "y" 277 | ] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "metadata": {}, 282 | "source": [ 283 | "Multiply the tensors `x` and `y` together, and notice how multiplication was done and its result." 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": 11, 289 | "metadata": { 290 | "id": "ivepGtD5MKP5" 291 | }, 292 | "outputs": [ 293 | { 294 | "data": { 295 | "text/plain": [ 296 | "" 302 | ] 303 | }, 304 | "execution_count": 11, 305 | "metadata": {}, 306 | "output_type": "execute_result" 307 | } 308 | ], 309 | "source": [ 310 | "# Multiply tensor `x` and `y`. `y` is multiplied to each element of x.\n", 311 | "result = tf.multiply(x, y)\n", 312 | "result" 313 | ] 314 | }, 315 | { 316 | "cell_type": "markdown", 317 | "metadata": {}, 318 | "source": [ 319 | "Re-Initialize `y` to a tensor having more values." 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": 12, 325 | "metadata": { 326 | "id": "8wzZ5NcMMPzD" 327 | }, 328 | "outputs": [ 329 | { 330 | "data": { 331 | "text/plain": [ 332 | "" 333 | ] 334 | }, 335 | "execution_count": 12, 336 | "metadata": {}, 337 | "output_type": "execute_result" 338 | } 339 | ], 340 | "source": [ 341 | "# Now let's define an array that matches the number of row elements in the `x` array.\n", 342 | "y = tf.constant([1, 2, 3, 4, 5], dtype=tf.float32)\n", 343 | "y" 344 | ] 345 | }, 346 | { 347 | "cell_type": "code", 348 | "execution_count": 13, 349 | "metadata": {}, 350 | "outputs": [ 351 | { 352 | "data": { 353 | "text/plain": [ 354 | "" 360 | ] 361 | }, 362 | "execution_count": 13, 363 | "metadata": {}, 364 | "output_type": "execute_result" 365 | } 366 | ], 367 | "source": [ 368 | "# Let's see first the contents of `x` again.\n", 369 | "x" 370 | ] 371 | }, 372 | { 373 | "cell_type": "markdown", 374 | "metadata": {}, 375 | "source": [ 376 | "Add the tensors `x` and `y` together, and notice how addition was done and its result." 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": 14, 382 | "metadata": { 383 | "id": "BVlntdYnMboh" 384 | }, 385 | "outputs": [ 386 | { 387 | "data": { 388 | "text/plain": [ 389 | "" 395 | ] 396 | }, 397 | "execution_count": 14, 398 | "metadata": {}, 399 | "output_type": "execute_result" 400 | } 401 | ], 402 | "source": [ 403 | "# Add tensor `x` and `y`. `y` is added element wise to each row of `x`.\n", 404 | "result = x + y\n", 405 | "result" 406 | ] 407 | }, 408 | { 409 | "cell_type": "markdown", 410 | "metadata": {}, 411 | "source": [ 412 | "### The shape parameter for tf.constant\n", 413 | "\n", 414 | "When using `tf.constant()`, you can pass in a 1D array (a vector) and set the `shape` parameter to turn this vector into a multi-dimensional array." 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": 15, 420 | "metadata": {}, 421 | "outputs": [ 422 | { 423 | "data": { 424 | "text/plain": [ 425 | "" 428 | ] 429 | }, 430 | "execution_count": 15, 431 | "metadata": {}, 432 | "output_type": "execute_result" 433 | } 434 | ], 435 | "source": [ 436 | "tf.constant([1,2,3,4], shape=(2,2))" 437 | ] 438 | }, 439 | { 440 | "cell_type": "markdown", 441 | "metadata": {}, 442 | "source": [ 443 | "### The shape parameter for tf.Variable\n", 444 | "\n", 445 | "Note, however, that for `tf.Variable()`, the shape of the tensor is derived from the shape given by the input array. Setting `shape` to something other than `None` will not reshape a 1D array into a multi-dimensional array, and will give a `ValueError`." 446 | ] 447 | }, 448 | { 449 | "cell_type": "code", 450 | "execution_count": 16, 451 | "metadata": {}, 452 | "outputs": [ 453 | { 454 | "name": "stdout", 455 | "output_type": "stream", 456 | "text": [ 457 | "The initial value's shape ((4,)) is not compatible with the explicitly supplied `shape` argument ((2, 2)).\n" 458 | ] 459 | } 460 | ], 461 | "source": [ 462 | "try:\n", 463 | " # This will produce a ValueError\n", 464 | " tf.Variable([1,2,3,4], shape=(2,2))\n", 465 | "except ValueError as v:\n", 466 | " # See what the ValueError says\n", 467 | " print(v)" 468 | ] 469 | } 470 | ], 471 | "metadata": { 472 | "colab": { 473 | "name": "basictensors.ipynb", 474 | "provenance": [] 475 | }, 476 | "kernelspec": { 477 | "display_name": "Python 3", 478 | "language": "python", 479 | "name": "python3" 480 | }, 481 | "language_info": { 482 | "codemirror_mode": { 483 | "name": "ipython", 484 | "version": 3 485 | }, 486 | "file_extension": ".py", 487 | "mimetype": "text/x-python", 488 | "name": "python", 489 | "nbconvert_exporter": "python", 490 | "pygments_lexer": "ipython3", 491 | "version": "3.7.6" 492 | } 493 | }, 494 | "nbformat": 4, 495 | "nbformat_minor": 4 496 | } 497 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-2/data/fashion_mnist/3.0.1/dataset_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "citation": "@article{DBLP:journals/corr/abs-1708-07747,\n author = {Han Xiao and\n Kashif Rasul and\n Roland Vollgraf},\n title = {Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning\n Algorithms},\n journal = {CoRR},\n volume = {abs/1708.07747},\n year = {2017},\n url = {http://arxiv.org/abs/1708.07747},\n archivePrefix = {arXiv},\n eprint = {1708.07747},\n timestamp = {Mon, 13 Aug 2018 16:47:27 +0200},\n biburl = {https://dblp.org/rec/bib/journals/corr/abs-1708-07747},\n bibsource = {dblp computer science bibliography, https://dblp.org}\n}\n", 3 | "description": "Fashion-MNIST is a dataset of Zalando's article images consisting of a training set of 60,000 examples and a test set of 10,000 examples. Each example is a 28x28 grayscale image, associated with a label from 10 classes.", 4 | "downloadSize": "30878645", 5 | "location": { 6 | "urls": [ 7 | "https://github.com/zalandoresearch/fashion-mnist" 8 | ] 9 | }, 10 | "name": "fashion_mnist", 11 | "schema": { 12 | "feature": [ 13 | { 14 | "name": "image", 15 | "shape": { 16 | "dim": [ 17 | { 18 | "size": "28" 19 | }, 20 | { 21 | "size": "28" 22 | }, 23 | { 24 | "size": "1" 25 | } 26 | ] 27 | }, 28 | "type": "INT" 29 | }, 30 | { 31 | "name": "label", 32 | "type": "INT" 33 | } 34 | ] 35 | }, 36 | "sizeInBytes": "30878645", 37 | "splits": [ 38 | { 39 | "name": "test", 40 | "numShards": "1", 41 | "shardLengths": [ 42 | "10000" 43 | ], 44 | "statistics": { 45 | "features": [ 46 | { 47 | "name": "image", 48 | "numStats": { 49 | "commonStats": { 50 | "numNonMissing": "10000" 51 | }, 52 | "max": 255.0 53 | } 54 | }, 55 | { 56 | "name": "label", 57 | "numStats": { 58 | "commonStats": { 59 | "numNonMissing": "10000" 60 | }, 61 | "max": 9.0 62 | } 63 | } 64 | ], 65 | "numExamples": "10000" 66 | } 67 | }, 68 | { 69 | "name": "train", 70 | "numShards": "10", 71 | "shardLengths": [ 72 | "60000" 73 | ], 74 | "statistics": { 75 | "features": [ 76 | { 77 | "name": "image", 78 | "numStats": { 79 | "commonStats": { 80 | "numNonMissing": "60000" 81 | }, 82 | "max": 255.0 83 | } 84 | }, 85 | { 86 | "name": "label", 87 | "numStats": { 88 | "commonStats": { 89 | "numNonMissing": "60000" 90 | }, 91 | "max": 9.0 92 | } 93 | } 94 | ], 95 | "numExamples": "60000" 96 | } 97 | } 98 | ], 99 | "supervisedKeys": { 100 | "input": "image", 101 | "output": "label" 102 | }, 103 | "version": "3.0.0" 104 | } -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-2/data/fashion_mnist/3.0.1/fashion_mnist-test.tfrecord-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/2-custom_and_distributed_training/week-2/data/fashion_mnist/3.0.1/fashion_mnist-test.tfrecord-00000-of-00001 -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-3/C2_W3_Lab_1_autograph-basics.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "colab_type": "text", 7 | "id": "hrXv0rU9sIma" 8 | }, 9 | "source": [ 10 | "# Autograph: Basic\n", 11 | "In this ungraded lab, you will go through some of the basics of autograph so you can explore what the generated code looks like." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## Imports" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": { 25 | "colab": {}, 26 | "colab_type": "code", 27 | "id": "NiolgWMPgpwI" 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "import tensorflow as tf" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "## Addition in autograph\n", 39 | "You can use the `@tf.function` decorator to automatically generate the graph-style code as shown below:" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "metadata": { 46 | "tags": [] 47 | }, 48 | "outputs": [ 49 | { 50 | "name": "stdout", 51 | "output_type": "stream", 52 | "text": [ 53 | "tf.Tensor(\n", 54 | "[[5. 2.]\n", 55 | " [4. 9.]], shape=(2, 2), dtype=float32)\n", 56 | "def tf__add(a, b):\n", 57 | " with ag__.FunctionScope('add', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n", 58 | " do_return = False\n", 59 | " retval_ = ag__.UndefinedReturnValue()\n", 60 | " try:\n", 61 | " do_return = True\n", 62 | " retval_ = (ag__.ld(a) + ag__.ld(b))\n", 63 | " except:\n", 64 | " do_return = False\n", 65 | " raise\n", 66 | " return fscope.ret(retval_, do_return)\n", 67 | "\n" 68 | ] 69 | } 70 | ], 71 | "source": [ 72 | "@tf.function\n", 73 | "def add(a, b):\n", 74 | " return a + b\n", 75 | "\n", 76 | "\n", 77 | "a = tf.Variable([[1.,2.],[3.,4.]])\n", 78 | "b = tf.Variable([[4.,0.],[1.,5.]])\n", 79 | "print(tf.add(a, b))\n", 80 | "\n", 81 | "# See what the generated code looks like\n", 82 | "print(tf.autograph.to_code(add.python_function))" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "## if-statements in autograph\n", 90 | "Control flow statements which are very intuitive to write in eager mode can look very complex in graph mode. You can see that in the next examples: first a simple function, then a more complicated one that involves lots of ops and conditionals (fizzbuzz)." 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 3, 96 | "metadata": { 97 | "tags": [] 98 | }, 99 | "outputs": [ 100 | { 101 | "name": "stdout", 102 | "output_type": "stream", 103 | "text": [ 104 | "def tf__f(x):\n", 105 | " with ag__.FunctionScope('f', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n", 106 | " do_return = False\n", 107 | " retval_ = ag__.UndefinedReturnValue()\n", 108 | "\n", 109 | " def get_state():\n", 110 | " return (x,)\n", 111 | "\n", 112 | " def set_state(vars_):\n", 113 | " nonlocal x\n", 114 | " (x,) = vars_\n", 115 | "\n", 116 | " def if_body():\n", 117 | " nonlocal x\n", 118 | " x = (ag__.ld(x) * ag__.ld(x))\n", 119 | "\n", 120 | " def else_body():\n", 121 | " nonlocal x\n", 122 | " pass\n", 123 | " ag__.if_stmt((ag__.ld(x) > 0), if_body, else_body, get_state, set_state, ('x',), 1)\n", 124 | " try:\n", 125 | " do_return = True\n", 126 | " retval_ = ag__.ld(x)\n", 127 | " except:\n", 128 | " do_return = False\n", 129 | " raise\n", 130 | " return fscope.ret(retval_, do_return)\n", 131 | "\n" 132 | ] 133 | } 134 | ], 135 | "source": [ 136 | "# simple function that returns the square if the input is greater than zero\n", 137 | "@tf.function\n", 138 | "def f(x):\n", 139 | " if x>0:\n", 140 | " x = x * x\n", 141 | " return x\n", 142 | "\n", 143 | "print(tf.autograph.to_code(f.python_function))" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "## Fizzbuzz in autograph\n", 151 | "\n", 152 | "You may remember implementing [fizzbuzz](http://wiki.c2.com/?FizzBuzzTest) in preparation for a coding interview. \n", 153 | "- Imagine how much fun it would be if you were asked to impement the graph mode version of that code! \n", 154 | "\n", 155 | "Fortunately, you can just use `@tf.function` and then call `tf.autograph.to_code`!" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": 4, 161 | "metadata": { 162 | "tags": [] 163 | }, 164 | "outputs": [ 165 | { 166 | "name": "stdout", 167 | "output_type": "stream", 168 | "text": [ 169 | "def tf__fizzbuzz(max_num):\n", 170 | " with ag__.FunctionScope('fizzbuzz', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n", 171 | " do_return = False\n", 172 | " retval_ = ag__.UndefinedReturnValue()\n", 173 | " counter = 0\n", 174 | "\n", 175 | " def get_state_3():\n", 176 | " return (counter,)\n", 177 | "\n", 178 | " def set_state_3(vars_):\n", 179 | " nonlocal counter\n", 180 | " (counter,) = vars_\n", 181 | "\n", 182 | " def loop_body(itr):\n", 183 | " nonlocal counter\n", 184 | " num = itr\n", 185 | "\n", 186 | " def get_state_2():\n", 187 | " return ()\n", 188 | "\n", 189 | " def set_state_2(block_vars):\n", 190 | " pass\n", 191 | "\n", 192 | " def if_body_2():\n", 193 | " ag__.ld(print)('FizzBuzz')\n", 194 | "\n", 195 | " def else_body_2():\n", 196 | "\n", 197 | " def get_state_1():\n", 198 | " return ()\n", 199 | "\n", 200 | " def set_state_1(block_vars):\n", 201 | " pass\n", 202 | "\n", 203 | " def if_body_1():\n", 204 | " ag__.ld(print)('Fizz')\n", 205 | "\n", 206 | " def else_body_1():\n", 207 | "\n", 208 | " def get_state():\n", 209 | " return ()\n", 210 | "\n", 211 | " def set_state(block_vars):\n", 212 | " pass\n", 213 | "\n", 214 | " def if_body():\n", 215 | " ag__.ld(print)('Buzz')\n", 216 | "\n", 217 | " def else_body():\n", 218 | " ag__.ld(print)(ag__.ld(num))\n", 219 | " ag__.if_stmt(((ag__.ld(num) % 5) == 0), if_body, else_body, get_state, set_state, (), 0)\n", 220 | " ag__.if_stmt(((ag__.ld(num) % 3) == 0), if_body_1, else_body_1, get_state_1, set_state_1, (), 0)\n", 221 | " ag__.if_stmt(ag__.and_((lambda : ((ag__.ld(num) % 3) == 0)), (lambda : ((ag__.ld(num) % 5) == 0))), if_body_2, else_body_2, get_state_2, set_state_2, (), 0)\n", 222 | " counter = ag__.ld(counter)\n", 223 | " counter += 1\n", 224 | " num = ag__.Undefined('num')\n", 225 | " ag__.for_stmt(ag__.converted_call(ag__.ld(range), (ag__.ld(max_num),), None, fscope), None, loop_body, get_state_3, set_state_3, ('counter',), {'iterate_names': 'num'})\n", 226 | " try:\n", 227 | " do_return = True\n", 228 | " retval_ = ag__.ld(counter)\n", 229 | " except:\n", 230 | " do_return = False\n", 231 | " raise\n", 232 | " return fscope.ret(retval_, do_return)\n", 233 | "\n" 234 | ] 235 | } 236 | ], 237 | "source": [ 238 | "@tf.function\n", 239 | "def fizzbuzz(max_num):\n", 240 | " counter = 0\n", 241 | " for num in range(max_num):\n", 242 | " if num % 3 == 0 and num % 5 == 0:\n", 243 | " print('FizzBuzz')\n", 244 | " elif num % 3 == 0:\n", 245 | " print('Fizz')\n", 246 | " elif num % 5 == 0:\n", 247 | " print('Buzz')\n", 248 | " else:\n", 249 | " print(num)\n", 250 | " counter += 1\n", 251 | " return counter\n", 252 | "\n", 253 | "print(tf.autograph.to_code(fizzbuzz.python_function))" 254 | ] 255 | } 256 | ], 257 | "metadata": { 258 | "colab": { 259 | "collapsed_sections": [], 260 | "include_colab_link": true, 261 | "name": "Autograph Basics.ipynb", 262 | "private_outputs": true, 263 | "provenance": [] 264 | }, 265 | "kernelspec": { 266 | "display_name": "Python 3", 267 | "language": "python", 268 | "name": "python3" 269 | }, 270 | "language_info": { 271 | "codemirror_mode": { 272 | "name": "ipython", 273 | "version": 3 274 | }, 275 | "file_extension": ".py", 276 | "mimetype": "text/x-python", 277 | "name": "python", 278 | "nbconvert_exporter": "python", 279 | "pygments_lexer": "ipython3", 280 | "version": "3.7.6" 281 | } 282 | }, 283 | "nbformat": 4, 284 | "nbformat_minor": 4 285 | } 286 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-3/C2_W3_Lab_2-graphs-for-complex-code.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\"Open" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": { 13 | "colab_type": "text", 14 | "id": "hrXv0rU9sIma" 15 | }, 16 | "source": [ 17 | "# Autograph: Graphs for complex code\n", 18 | "\n", 19 | "In this ungraded lab, you'll go through some of the scenarios from the lesson `Creating graphs for complex code`." 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "## Imports" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": { 33 | "colab": {}, 34 | "colab_type": "code", 35 | "id": "NiolgWMPgpwI" 36 | }, 37 | "outputs": [], 38 | "source": [ 39 | "try:\n", 40 | " # %tensorflow_version only exists in Colab.\n", 41 | " %tensorflow_version 2.x\n", 42 | "except Exception:\n", 43 | " pass\n", 44 | "\n", 45 | "import tensorflow as tf" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "As you saw in the lectures, seemingly simple functions can sometimes be difficult to write in graph mode. Fortunately, Autograph generates this complex graph code for us.\n", 53 | "\n", 54 | "- Here is a function that does some multiplication and additon." 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": { 61 | "tags": [] 62 | }, 63 | "outputs": [], 64 | "source": [ 65 | "a = tf.Variable(1.0)\n", 66 | "b = tf.Variable(2.0)\n", 67 | "\n", 68 | "@tf.function\n", 69 | "def f(x,y):\n", 70 | " a.assign(y * b)\n", 71 | " b.assign_add(x * a)\n", 72 | " return a + b\n", 73 | "\n", 74 | "print(f(1.0, 2.0))\n", 75 | "\n", 76 | "print(tf.autograph.to_code(f.python_function))" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "- Here is a function that checks if the sign of a number is positive or not." 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": { 90 | "tags": [] 91 | }, 92 | "outputs": [], 93 | "source": [ 94 | "@tf.function\n", 95 | "def sign(x):\n", 96 | " if x > 0:\n", 97 | " return 'Positive'\n", 98 | " else:\n", 99 | " return 'Negative or zero'\n", 100 | "\n", 101 | "print(\"Sign = {}\".format(sign(tf.constant(2))))\n", 102 | "print(\"Sign = {}\".format(sign(tf.constant(-2))))\n", 103 | "\n", 104 | "print(tf.autograph.to_code(sign.python_function))" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "- Here is another function that includes a while loop." 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "metadata": { 118 | "tags": [] 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "@tf.function\n", 123 | "def f(x):\n", 124 | " while tf.reduce_sum(x) > 1:\n", 125 | " tf.print(x)\n", 126 | " x = tf.tanh(x)\n", 127 | " return x\n", 128 | "\n", 129 | "print(tf.autograph.to_code(f.python_function))" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "- Here is a function that uses a for loop and an if statement." 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": { 143 | "tags": [] 144 | }, 145 | "outputs": [], 146 | "source": [ 147 | "@tf.function\n", 148 | "def sum_even(items):\n", 149 | " s = 0\n", 150 | " for c in items:\n", 151 | " if c % 2 > 0:\n", 152 | " continue\n", 153 | " s += c\n", 154 | " return s\n", 155 | "\n", 156 | "print(tf.autograph.to_code(sum_even.python_function))" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "## Print statements\n", 164 | "\n", 165 | "Tracing also behaves differently in graph mode. First, here is a function (not decorated with `@tf.function` yet) that prints the value of the input parameter. `f(2)` is called in a for loop 5 times, and then `f(3)` is called." 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": { 172 | "tags": [] 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "def f(x):\n", 177 | " print(\"Traced with\", x)\n", 178 | "\n", 179 | "for i in range(5):\n", 180 | " f(2)\n", 181 | " \n", 182 | "f(3)" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "If you were to decorate this function with `@tf.function` and run it, notice that the print statement only appears once for `f(2)` even though it is called in a loop." 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "@tf.function\n", 199 | "def f(x):\n", 200 | " print(\"Traced with\", x)\n", 201 | "\n", 202 | "for i in range(5):\n", 203 | " f(2)\n", 204 | " \n", 205 | "f(3)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "Now compare `print` to `tf.print`.\n", 213 | "- `tf.print` is graph aware and will run as expected in loops. \n", 214 | "\n", 215 | "Try running the same code where `tf.print()` is added in addition to the regular `print`.\n", 216 | "- Note how `tf.print` behaves compared to `print` in graph mode." 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "metadata": { 223 | "tags": [] 224 | }, 225 | "outputs": [], 226 | "source": [ 227 | "@tf.function\n", 228 | "def f(x):\n", 229 | " print(\"Traced with\", x)\n", 230 | " # added tf.print\n", 231 | " tf.print(\"Executed with\", x)\n", 232 | "\n", 233 | "for i in range(5):\n", 234 | " f(2)\n", 235 | " \n", 236 | "f(3)" 237 | ] 238 | }, 239 | { 240 | "cell_type": "markdown", 241 | "metadata": {}, 242 | "source": [ 243 | "## Avoid defining variables inside the function\n", 244 | "\n", 245 | "This function (not decorated yet) defines a tensor `v` and adds the input `x` to it. \n", 246 | "\n", 247 | "Here, it runs fine." 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "metadata": {}, 254 | "outputs": [], 255 | "source": [ 256 | "def f(x):\n", 257 | " v = tf.Variable(1.0)\n", 258 | " v.assign_add(x)\n", 259 | " return v\n", 260 | "\n", 261 | "print(f(1))" 262 | ] 263 | }, 264 | { 265 | "cell_type": "markdown", 266 | "metadata": {}, 267 | "source": [ 268 | "Now if you decorate the function with `@tf.function`.\n", 269 | "\n", 270 | "The cell below will throw an error because `tf.Variable` is defined within the function. The graph mode function should only contain operations." 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": null, 276 | "metadata": { 277 | "tags": [] 278 | }, 279 | "outputs": [], 280 | "source": [ 281 | "@tf.function\n", 282 | "def f(x):\n", 283 | " v = tf.Variable(1.0)\n", 284 | " v.assign_add(x)\n", 285 | " return v\n", 286 | "\n", 287 | "print(f(1))" 288 | ] 289 | }, 290 | { 291 | "cell_type": "markdown", 292 | "metadata": {}, 293 | "source": [ 294 | "To get around the error above, simply move `v = tf.Variable(1.0)` to the top of the cell before the `@tf.function` decorator." 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": null, 300 | "metadata": { 301 | "tags": [] 302 | }, 303 | "outputs": [], 304 | "source": [ 305 | "# define the variables outside of the decorated function\n", 306 | "v = tf.Variable(1.0)\n", 307 | "\n", 308 | "@tf.function\n", 309 | "def f(x):\n", 310 | " return v.assign_add(x)\n", 311 | "\n", 312 | "print(f(5))" 313 | ] 314 | } 315 | ], 316 | "metadata": { 317 | "colab": { 318 | "collapsed_sections": [], 319 | "include_colab_link": true, 320 | "name": "Autograph Lesson 2.ipynb", 321 | "private_outputs": true, 322 | "provenance": [] 323 | }, 324 | "kernelspec": { 325 | "display_name": "Python 3", 326 | "language": "python", 327 | "name": "python3" 328 | }, 329 | "language_info": { 330 | "codemirror_mode": { 331 | "name": "ipython", 332 | "version": 3 333 | }, 334 | "file_extension": ".py", 335 | "mimetype": "text/x-python", 336 | "name": "python", 337 | "nbconvert_exporter": "python", 338 | "pygments_lexer": "ipython3", 339 | "version": "3.7.6" 340 | } 341 | }, 342 | "nbformat": 4, 343 | "nbformat_minor": 4 344 | } 345 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-3/data/horses_or_humans/3.0.0/dataset_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "citation": "@ONLINE {horses_or_humans,\nauthor = \"Laurence Moroney\",\ntitle = \"Horses or Humans Dataset\",\nmonth = \"feb\",\nyear = \"2019\",\nurl = \"http://laurencemoroney.com/horses-or-humans-dataset\"\n}\n", 3 | "description": "A large set of images of horses and humans.", 4 | "downloadSize": "161055054", 5 | "location": { 6 | "urls": [ 7 | "http://laurencemoroney.com/horses-or-humans-dataset" 8 | ] 9 | }, 10 | "name": "horses_or_humans", 11 | "schema": { 12 | "feature": [ 13 | { 14 | "name": "image", 15 | "shape": { 16 | "dim": [ 17 | { 18 | "size": "300" 19 | }, 20 | { 21 | "size": "300" 22 | }, 23 | { 24 | "size": "3" 25 | } 26 | ] 27 | }, 28 | "type": "INT" 29 | }, 30 | { 31 | "name": "label", 32 | "type": "INT" 33 | } 34 | ] 35 | }, 36 | "sizeInBytes": "161055054", 37 | "splits": [ 38 | { 39 | "name": "test", 40 | "numShards": "10", 41 | "shardLengths": [ 42 | "256" 43 | ], 44 | "statistics": { 45 | "features": [ 46 | { 47 | "name": "image", 48 | "numStats": { 49 | "commonStats": { 50 | "numNonMissing": "256" 51 | }, 52 | "max": 255.0 53 | } 54 | }, 55 | { 56 | "name": "label", 57 | "numStats": { 58 | "commonStats": { 59 | "numNonMissing": "256" 60 | }, 61 | "max": 1.0 62 | } 63 | } 64 | ], 65 | "numExamples": "256" 66 | } 67 | }, 68 | { 69 | "name": "train", 70 | "numShards": "10", 71 | "shardLengths": [ 72 | "514", 73 | "513" 74 | ], 75 | "statistics": { 76 | "features": [ 77 | { 78 | "name": "image", 79 | "numStats": { 80 | "commonStats": { 81 | "numNonMissing": "1027" 82 | }, 83 | "max": 255.0 84 | } 85 | }, 86 | { 87 | "name": "label", 88 | "numStats": { 89 | "commonStats": { 90 | "numNonMissing": "1027" 91 | }, 92 | "max": 1.0 93 | } 94 | } 95 | ], 96 | "numExamples": "1027" 97 | } 98 | } 99 | ], 100 | "supervisedKeys": { 101 | "input": "image", 102 | "output": "label" 103 | }, 104 | "version": "3.0.0" 105 | } -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-4/C2_W4_Lab_1_basic-mirrored-strategy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\"Open" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# Mirrored Strategy: Basic\n", 15 | "\n", 16 | "In this ungraded lab, you'll go through some of the basics of applying [Mirrored Strategy](https://www.tensorflow.org/api_docs/python/tf/distribute/MirroredStrategy)." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Imports" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "colab": {}, 31 | "colab_type": "code", 32 | "id": "TkUjfmKkflCd" 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "# Import TensorFlow and TensorFlow Datasets\n", 37 | "\n", 38 | "import tensorflow_datasets as tfds\n", 39 | "import tensorflow as tf\n", 40 | "tfds.disable_progress_bar()\n", 41 | "\n", 42 | "import os" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "Load the MNIST dataset and split it into training and test chunks." 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": { 56 | "colab": {}, 57 | "colab_type": "code", 58 | "id": "eQN-PtIGgFtH" 59 | }, 60 | "outputs": [], 61 | "source": [ 62 | "# Load the dataset we'll use for this lab\n", 63 | "datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True, data_dir='./data')\n", 64 | "\n", 65 | "mnist_train, mnist_test = datasets['train'], datasets['test']" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "Next, you define `strategy` using the `MirroredStrategy()` class. Print to see the number of devices available.\n", 73 | "\n", 74 | "**Note:** \n", 75 | "- If you are running this on Coursera, you'll see it gives a warning about no presence of GPU devices. \n", 76 | "- If you are running this in Colab, make sure you have selected your `Runtime` to be `GPU` for it to detect it. \n", 77 | "- In both these cases, you'll see there's only 1 device that is available. \n", 78 | "- One device is sufficient for helping you understand these distribution strategies." 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": { 85 | "colab": {}, 86 | "colab_type": "code", 87 | "id": "eCsDqWnDgNHr" 88 | }, 89 | "outputs": [], 90 | "source": [ 91 | "# Define the strategy to use and print the number of devices found\n", 92 | "strategy = tf.distribute.MirroredStrategy()\n", 93 | "print('Number of devices: {}'.format(strategy.num_replicas_in_sync))" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "Next, you create your training and test examples, define your batch size and also define `BATCH_SIZE_PER_REPLICA` which is the distribution you are making for each available device." 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": { 107 | "colab": {}, 108 | "colab_type": "code", 109 | "id": "p1xWxKcnhar9" 110 | }, 111 | "outputs": [], 112 | "source": [ 113 | "# Get the number of examples in the train and test sets\n", 114 | "num_train_examples = info.splits['train'].num_examples\n", 115 | "num_test_examples = info.splits['test'].num_examples\n", 116 | "\n", 117 | "BUFFER_SIZE = 10000\n", 118 | "\n", 119 | "BATCH_SIZE_PER_REPLICA = 64\n", 120 | "# Use for Mirrored Strategy\n", 121 | "BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n", 122 | "# Use for No Strategy\n", 123 | "# BATCH_SIZE = BATCH_SIZE_PER_REPLICA * 1" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "A mapping function which normalizes your images:" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": { 137 | "colab": {}, 138 | "colab_type": "code", 139 | "id": "aPIU8E7BhyYq" 140 | }, 141 | "outputs": [], 142 | "source": [ 143 | "# Function for normalizing the image\n", 144 | "def scale(image, label):\n", 145 | " image = tf.cast(image, tf.float32)\n", 146 | " image /= 255\n", 147 | "\n", 148 | " return image, label" 149 | ] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": {}, 154 | "source": [ 155 | "Next, you create your training and evaluation datesets in the batch size you want by shuffling through your buffer size." 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": { 162 | "colab": {}, 163 | "colab_type": "code", 164 | "id": "ByTZB2AYh0nA" 165 | }, 166 | "outputs": [], 167 | "source": [ 168 | "# Set up the train and eval data set\n", 169 | "train_dataset = mnist_train.map(scale).cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", 170 | "eval_dataset = mnist_test.map(scale).batch(BATCH_SIZE)" 171 | ] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": {}, 176 | "source": [ 177 | "For your model to follow the strategy, define your model within the strategy's scope.\n", 178 | "- Run all the cells below and notice the results. \n", 179 | "- Afterwards comment out `with strategy.scope():` and run everything again, without the strategy. \n", 180 | "Then you can compare the results. \n", 181 | "The important thing to notice and compare is the time taken for each epoch to complete. As mentioned in the lecture, doing a mirrored strategy on a single device (which are lab environment has) might take longer to train because of the overhead in implementing the strategy. With that, the advantages of using this strategy is more evident if you will use it on multiple devices." 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": { 188 | "colab": {}, 189 | "colab_type": "code", 190 | "id": "7rRzY5ojh51B" 191 | }, 192 | "outputs": [], 193 | "source": [ 194 | "# Use for Mirrored Strategy -- comment out `with strategy.scope():` and deindent for no strategy\n", 195 | "with strategy.scope():\n", 196 | " model = tf.keras.Sequential([\n", 197 | " tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)),\n", 198 | " tf.keras.layers.MaxPooling2D(),\n", 199 | " tf.keras.layers.Flatten(),\n", 200 | " tf.keras.layers.Dense(64, activation='relu'),\n", 201 | " tf.keras.layers.Dense(10)\n", 202 | " ])" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": null, 208 | "metadata": { 209 | "colab": {}, 210 | "colab_type": "code", 211 | "id": "NWOJWLENphod" 212 | }, 213 | "outputs": [], 214 | "source": [ 215 | "model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", 216 | " optimizer=tf.keras.optimizers.Adam(),\n", 217 | " metrics=['accuracy'])" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": null, 223 | "metadata": { 224 | "colab": {}, 225 | "colab_type": "code", 226 | "id": "-8voqkush_Bx" 227 | }, 228 | "outputs": [], 229 | "source": [ 230 | "model.fit(train_dataset, epochs=12)" 231 | ] 232 | } 233 | ], 234 | "metadata": { 235 | "accelerator": "GPU", 236 | "colab": { 237 | "name": "Basic Mirrored Strategy", 238 | "provenance": [] 239 | }, 240 | "kernelspec": { 241 | "display_name": "Python 3", 242 | "language": "python", 243 | "name": "python3" 244 | }, 245 | "language_info": { 246 | "codemirror_mode": { 247 | "name": "ipython", 248 | "version": 3 249 | }, 250 | "file_extension": ".py", 251 | "mimetype": "text/x-python", 252 | "name": "python", 253 | "nbconvert_exporter": "python", 254 | "pygments_lexer": "ipython3", 255 | "version": "3.7.6" 256 | } 257 | }, 258 | "nbformat": 4, 259 | "nbformat_minor": 4 260 | } 261 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-4/C2_W4_Lab_2_multi-GPU-mirrored-strategy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\"Open" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# Multi-GPU Mirrored Strategy\n", 15 | "\n", 16 | "In this ungraded lab, you'll go through how to set up a Multi-GPU Mirrored Strategy. The lab environment only has a CPU but we placed the code here in case you want to try this out for yourself in a multiGPU device.\n", 17 | "\n", 18 | "**Notes:** \n", 19 | "- If you are running this on Coursera, you'll see it gives a warning about no presence of GPU devices. \n", 20 | "- If you are running this in Colab, make sure you have selected your `runtime` to be `GPU`. \n", 21 | "- In both these cases, you'll see there's only 1 device that is available. \n", 22 | "- One device is sufficient for helping you understand these distribution strategies." 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "## Imports" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "import tensorflow as tf\n", 39 | "import numpy as np\n", 40 | "import os" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "## Setup Distribution Strategy" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "# Note that it generally has a minimum of 8 cores, but if your GPU has\n", 57 | "# less, you need to set this. In this case one of my GPUs has 4 cores\n", 58 | "os.environ[\"TF_MIN_GPU_MULTIPROCESSOR_COUNT\"] = \"4\"\n", 59 | "\n", 60 | "# If the list of devices is not specified in the\n", 61 | "# `tf.distribute.MirroredStrategy` constructor, it will be auto-detected.\n", 62 | "# If you have *different* GPUs in your system, you probably have to set up cross_device_ops like this\n", 63 | "strategy = tf.distribute.MirroredStrategy(cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())\n", 64 | "print ('Number of devices: {}'.format(strategy.num_replicas_in_sync))" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "## Prepare the Data" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "# Get the data\n", 81 | "fashion_mnist = tf.keras.datasets.fashion_mnist\n", 82 | "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\n", 83 | "\n", 84 | "# Adding a dimension to the array -> new shape == (28, 28, 1)\n", 85 | "# We are doing this because the first layer in our model is a convolutional\n", 86 | "# layer and it requires a 4D input (batch_size, height, width, channels).\n", 87 | "# batch_size dimension will be added later on.\n", 88 | "train_images = train_images[..., None]\n", 89 | "test_images = test_images[..., None]\n", 90 | "\n", 91 | "# Normalize the images to [0, 1] range.\n", 92 | "train_images = train_images / np.float32(255)\n", 93 | "test_images = test_images / np.float32(255)\n", 94 | "\n", 95 | "# Batch the input data\n", 96 | "BUFFER_SIZE = len(train_images)\n", 97 | "BATCH_SIZE_PER_REPLICA = 64\n", 98 | "GLOBAL_BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n", 99 | "\n", 100 | "# Create Datasets from the batches\n", 101 | "train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(BUFFER_SIZE).batch(GLOBAL_BATCH_SIZE)\n", 102 | "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE)\n", 103 | "\n", 104 | "# Create Distributed Datasets from the datasets\n", 105 | "train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)\n", 106 | "test_dist_dataset = strategy.experimental_distribute_dataset(test_dataset)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "## Define the Model" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "# Create the model architecture\n", 123 | "def create_model():\n", 124 | " model = tf.keras.Sequential([\n", 125 | " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", 126 | " tf.keras.layers.MaxPooling2D(),\n", 127 | " tf.keras.layers.Conv2D(64, 3, activation='relu'),\n", 128 | " tf.keras.layers.MaxPooling2D(),\n", 129 | " tf.keras.layers.Flatten(),\n", 130 | " tf.keras.layers.Dense(64, activation='relu'),\n", 131 | " tf.keras.layers.Dense(10)\n", 132 | " ])\n", 133 | " return model" 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": {}, 139 | "source": [ 140 | "## Configure custom training\n", 141 | "\n", 142 | "Instead of `model.compile()`, we're going to do custom training, so let's do that within a strategy scope." 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "with strategy.scope():\n", 152 | " # We will use sparse categorical crossentropy as always. But, instead of having the loss function\n", 153 | " # manage the map reduce across GPUs for us, we'll do it ourselves with a simple algorithm.\n", 154 | " # Remember -- the map reduce is how the losses get aggregated\n", 155 | " # Set reduction to `none` so we can do the reduction afterwards and divide byglobal batch size.\n", 156 | " loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)\n", 157 | "\n", 158 | " def compute_loss(labels, predictions):\n", 159 | " # Compute Loss uses the loss object to compute the loss\n", 160 | " # Notice that per_example_loss will have an entry per GPU\n", 161 | " # so in this case there'll be 2 -- i.e. the loss for each replica\n", 162 | " per_example_loss = loss_object(labels, predictions)\n", 163 | " # You can print it to see it -- you'll get output like this:\n", 164 | " # Tensor(\"sparse_categorical_crossentropy/weighted_loss/Mul:0\", shape=(48,), dtype=float32, device=/job:localhost/replica:0/task:0/device:GPU:0)\n", 165 | " # Tensor(\"replica_1/sparse_categorical_crossentropy/weighted_loss/Mul:0\", shape=(48,), dtype=float32, device=/job:localhost/replica:0/task:0/device:GPU:1)\n", 166 | " # Note in particular that replica_0 isn't named in the weighted_loss -- the first is unnamed, the second is replica_1 etc\n", 167 | " print(per_example_loss)\n", 168 | " return tf.nn.compute_average_loss(per_example_loss, global_batch_size=GLOBAL_BATCH_SIZE)\n", 169 | "\n", 170 | " # We'll just reduce by getting the average of the losses\n", 171 | " test_loss = tf.keras.metrics.Mean(name='test_loss')\n", 172 | "\n", 173 | " # Accuracy on train and test will be SparseCategoricalAccuracy\n", 174 | " train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", 175 | " test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')\n", 176 | "\n", 177 | " # Optimizer will be Adam\n", 178 | " optimizer = tf.keras.optimizers.Adam()\n", 179 | "\n", 180 | " # Create the model within the scope\n", 181 | " model = create_model()" 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "## Train and Test Steps Functions\n", 189 | "\n", 190 | "Let's define a few utilities to facilitate the training." 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": { 197 | "colab": {}, 198 | "colab_type": "code", 199 | "id": "k0VOdqKP2NEz" 200 | }, 201 | "outputs": [], 202 | "source": [ 203 | "# `run` replicates the provided computation and runs it\n", 204 | "# with the distributed input.\n", 205 | "@tf.function\n", 206 | "def distributed_train_step(dataset_inputs):\n", 207 | " per_replica_losses = strategy.run(train_step, args=(dataset_inputs,))\n", 208 | " #tf.print(per_replica_losses.values)\n", 209 | " return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n", 210 | "\n", 211 | "def train_step(inputs):\n", 212 | " images, labels = inputs\n", 213 | " with tf.GradientTape() as tape:\n", 214 | " predictions = model(images, training=True)\n", 215 | " loss = compute_loss(labels, predictions)\n", 216 | "\n", 217 | " gradients = tape.gradient(loss, model.trainable_variables)\n", 218 | " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", 219 | "\n", 220 | " train_accuracy.update_state(labels, predictions)\n", 221 | " return loss\n", 222 | "\n", 223 | "#######################\n", 224 | "# Test Steps Functions\n", 225 | "#######################\n", 226 | "@tf.function\n", 227 | "def distributed_test_step(dataset_inputs):\n", 228 | " return strategy.run(test_step, args=(dataset_inputs,))\n", 229 | "\n", 230 | "def test_step(inputs):\n", 231 | " images, labels = inputs\n", 232 | "\n", 233 | " predictions = model(images, training=False)\n", 234 | " t_loss = loss_object(labels, predictions)\n", 235 | "\n", 236 | " test_loss.update_state(t_loss)\n", 237 | " test_accuracy.update_state(labels, predictions)\n", 238 | "\n", 239 | "\n" 240 | ] 241 | }, 242 | { 243 | "cell_type": "markdown", 244 | "metadata": {}, 245 | "source": [ 246 | "## Training Loop\n", 247 | "\n", 248 | "We can now start training the model." 249 | ] 250 | }, 251 | { 252 | "cell_type": "code", 253 | "execution_count": null, 254 | "metadata": {}, 255 | "outputs": [], 256 | "source": [ 257 | "EPOCHS = 10\n", 258 | "for epoch in range(EPOCHS):\n", 259 | " # Do Training\n", 260 | " total_loss = 0.0\n", 261 | " num_batches = 0\n", 262 | " for batch in train_dist_dataset:\n", 263 | " total_loss += distributed_train_step(batch)\n", 264 | " num_batches += 1\n", 265 | " train_loss = total_loss / num_batches\n", 266 | "\n", 267 | " # Do Testing\n", 268 | " for batch in test_dist_dataset:\n", 269 | " distributed_test_step(batch)\n", 270 | "\n", 271 | " template = (\"Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, \" \"Test Accuracy: {}\")\n", 272 | "\n", 273 | " print (template.format(epoch+1, train_loss, train_accuracy.result()*100, test_loss.result(), test_accuracy.result()*100))\n", 274 | "\n", 275 | " test_loss.reset_states()\n", 276 | " train_accuracy.reset_states()\n", 277 | " test_accuracy.reset_states()" 278 | ] 279 | } 280 | ], 281 | "metadata": { 282 | "colab": { 283 | "name": "MultiGPUMirroredStrategy.ipynb", 284 | "provenance": [] 285 | }, 286 | "kernelspec": { 287 | "display_name": "Python 3", 288 | "language": "python", 289 | "name": "python3" 290 | }, 291 | "language_info": { 292 | "codemirror_mode": { 293 | "name": "ipython", 294 | "version": 3 295 | }, 296 | "file_extension": ".py", 297 | "mimetype": "text/x-python", 298 | "name": "python", 299 | "nbconvert_exporter": "python", 300 | "pygments_lexer": "ipython3", 301 | "version": "3.7.6" 302 | } 303 | }, 304 | "nbformat": 4, 305 | "nbformat_minor": 4 306 | } 307 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-4/C2_W4_Lab_4_one-device-strategy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\"Open" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# One Device Strategy \n", 15 | "\n", 16 | "In this ungraded lab, you'll learn how to set up a [One Device Strategy](https://www.tensorflow.org/api_docs/python/tf/distribute/OneDeviceStrategy). This is typically used to deliberately test your code on a single device. This can be used before switching to a different strategy that distributes across multiple devices. Please click on the **Open in Colab** badge above so you can download the datasets and use a GPU-enabled lab environment." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Imports" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "colab": {}, 31 | "colab_type": "code", 32 | "id": "rFpbGH-egdhC" 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "try:\n", 37 | " # %tensorflow_version only exists in Colab.\n", 38 | " %tensorflow_version 2.x\n", 39 | "except Exception:\n", 40 | " pass\n", 41 | "\n", 42 | "import tensorflow as tf\n", 43 | "import tensorflow_hub as hub\n", 44 | "import tensorflow_datasets as tfds\n", 45 | "\n", 46 | "tfds.disable_progress_bar()" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "## Define the Distribution Strategy\n", 54 | "\n", 55 | "You can list available devices in your machine and specify a device type. This allows you to verify the device name to pass in `tf.distribute.OneDeviceStrategy()`." 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "# choose a device type such as CPU or GPU\n", 65 | "devices = tf.config.list_physical_devices('GPU')\n", 66 | "print(devices[0])\n", 67 | "\n", 68 | "# You'll see that the name will look something like \"/physical_device:GPU:0\"\n", 69 | "# Just take the GPU:0 part and use that as the name\n", 70 | "gpu_name = \"GPU:0\"\n", 71 | "\n", 72 | "# define the strategy and pass in the device name\n", 73 | "one_strategy = tf.distribute.OneDeviceStrategy(device=gpu_name)" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "## Parameters\n", 81 | "\n", 82 | "We'll define a few global variables for setting up the model and dataset." 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": { 89 | "colab": {}, 90 | "colab_type": "code", 91 | "id": "P_OV1G0J0bx8" 92 | }, 93 | "outputs": [], 94 | "source": [ 95 | "pixels = 224\n", 96 | "MODULE_HANDLE = 'https://tfhub.dev/tensorflow/resnet_50/feature_vector/1'\n", 97 | "IMAGE_SIZE = (pixels, pixels)\n", 98 | "BATCH_SIZE = 32\n", 99 | "\n", 100 | "print(\"Using {} with input size {}\".format(MODULE_HANDLE, IMAGE_SIZE))" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "## Download and Prepare the Dataset\n", 108 | "\n", 109 | "We will use the [Cats vs Dogs](https://www.tensorflow.org/datasets/catalog/cats_vs_dogs) dataset and we will fetch it via TFDS." 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "metadata": { 116 | "colab": {}, 117 | "colab_type": "code", 118 | "id": "1HCFBMh-1gaX" 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "splits = ['train[:80%]', 'train[80%:90%]', 'train[90%:]']\n", 123 | "\n", 124 | "(train_examples, validation_examples, test_examples), info = tfds.load('cats_vs_dogs', with_info=True, as_supervised=True, split=splits)\n", 125 | "\n", 126 | "num_examples = info.splits['train'].num_examples\n", 127 | "num_classes = info.features['label'].num_classes" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": { 134 | "colab": {}, 135 | "colab_type": "code", 136 | "id": "5jfUDRQN1kfk" 137 | }, 138 | "outputs": [], 139 | "source": [ 140 | "# resize the image and normalize pixel values\n", 141 | "def format_image(image, label):\n", 142 | " image = tf.image.resize(image, IMAGE_SIZE) / 255.0\n", 143 | " return image, label" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "metadata": { 150 | "colab": {}, 151 | "colab_type": "code", 152 | "id": "WRBF8Vp01uaE" 153 | }, 154 | "outputs": [], 155 | "source": [ 156 | "# prepare batches\n", 157 | "train_batches = train_examples.shuffle(num_examples // 4).map(format_image).batch(BATCH_SIZE).prefetch(1)\n", 158 | "validation_batches = validation_examples.map(format_image).batch(BATCH_SIZE).prefetch(1)\n", 159 | "test_batches = test_examples.map(format_image).batch(1)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": null, 165 | "metadata": { 166 | "colab": {}, 167 | "colab_type": "code", 168 | "id": "MxggWg9m11P1" 169 | }, 170 | "outputs": [], 171 | "source": [ 172 | "# check if the batches have the correct size and the images have the correct shape\n", 173 | "for image_batch, label_batch in train_batches.take(1):\n", 174 | " pass\n", 175 | "\n", 176 | "print(image_batch.shape)" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "## Define and Configure the Model\n", 184 | "\n", 185 | "As with other strategies, setting up the model requires minimal code changes. Let's first define a utility function to build and compile the model." 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": { 192 | "colab": {}, 193 | "colab_type": "code", 194 | "id": "Wx8MEwUl1300" 195 | }, 196 | "outputs": [], 197 | "source": [ 198 | "# tells if we want to freeze the layer weights of our feature extractor during training\n", 199 | "do_fine_tuning = False" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": { 206 | "colab": {}, 207 | "colab_type": "code", 208 | "id": "WKLytydu1_qt" 209 | }, 210 | "outputs": [], 211 | "source": [ 212 | "def build_and_compile_model():\n", 213 | " print(\"Building model with\", MODULE_HANDLE)\n", 214 | "\n", 215 | " # configures the feature extractor fetched from TF Hub\n", 216 | " feature_extractor = hub.KerasLayer(MODULE_HANDLE,\n", 217 | " input_shape=IMAGE_SIZE + (3,), \n", 218 | " trainable=do_fine_tuning)\n", 219 | "\n", 220 | " # define the model\n", 221 | " model = tf.keras.Sequential([\n", 222 | " feature_extractor,\n", 223 | " # append a dense with softmax for the number of classes\n", 224 | " tf.keras.layers.Dense(num_classes, activation='softmax')\n", 225 | " ])\n", 226 | "\n", 227 | " # display summary\n", 228 | " model.summary()\n", 229 | "\n", 230 | " # configure the optimizer, loss and metrics\n", 231 | " optimizer = tf.keras.optimizers.SGD(lr=0.002, momentum=0.9) if do_fine_tuning else 'adam'\n", 232 | " model.compile(optimizer=optimizer,\n", 233 | " loss='sparse_categorical_crossentropy',\n", 234 | " metrics=['accuracy'])\n", 235 | "\n", 236 | " return model" 237 | ] 238 | }, 239 | { 240 | "cell_type": "markdown", 241 | "metadata": {}, 242 | "source": [ 243 | "You can now call the function under the strategy scope. This places variables and computations on the device you specified earlier." 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": null, 249 | "metadata": { 250 | "colab": {}, 251 | "colab_type": "code", 252 | "id": "sDenpJX-2EhD" 253 | }, 254 | "outputs": [], 255 | "source": [ 256 | "# build and compile under the strategy scope\n", 257 | "with one_strategy.scope():\n", 258 | " model = build_and_compile_model()" 259 | ] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": {}, 264 | "source": [ 265 | "`model.fit()` can be run as usual." 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": null, 271 | "metadata": { 272 | "colab": {}, 273 | "colab_type": "code", 274 | "id": "7L4C5KKs3fal" 275 | }, 276 | "outputs": [], 277 | "source": [ 278 | "EPOCHS = 5\n", 279 | "hist = model.fit(train_batches,\n", 280 | " epochs=EPOCHS,\n", 281 | " validation_data=validation_batches)" 282 | ] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "metadata": {}, 287 | "source": [ 288 | "Once everything is working correctly, you can switch to a different device or a different strategy that distributes to multiple devices." 289 | ] 290 | } 291 | ], 292 | "metadata": { 293 | "accelerator": "GPU", 294 | "colab": { 295 | "collapsed_sections": [], 296 | "name": "C2W4_Lab_4_one-device-strategy.ipynb", 297 | "provenance": [] 298 | }, 299 | "kernelspec": { 300 | "display_name": "Python 3", 301 | "language": "python", 302 | "name": "python3" 303 | }, 304 | "language_info": { 305 | "codemirror_mode": { 306 | "name": "ipython", 307 | "version": 3 308 | }, 309 | "file_extension": ".py", 310 | "mimetype": "text/x-python", 311 | "name": "python", 312 | "nbconvert_exporter": "python", 313 | "pygments_lexer": "ipython3", 314 | "version": "3.7.6" 315 | } 316 | }, 317 | "nbformat": 4, 318 | "nbformat_minor": 4 319 | } 320 | -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-4/data/mnist/3.0.1/dataset_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "citation": "@article{lecun2010mnist,\n title={MNIST handwritten digit database},\n author={LeCun, Yann and Cortes, Corinna and Burges, CJ},\n journal={ATT Labs [Online]. Available: http://yann. lecun. com/exdb/mnist},\n volume={2},\n year={2010}\n}\n", 3 | "description": "The MNIST database of handwritten digits.", 4 | "location": { 5 | "urls": [ 6 | "https://storage.googleapis.com/cvdf-datasets/mnist/" 7 | ] 8 | }, 9 | "name": "mnist", 10 | "schema": { 11 | "feature": [ 12 | { 13 | "name": "image", 14 | "shape": { 15 | "dim": [ 16 | { 17 | "size": "28" 18 | }, 19 | { 20 | "size": "28" 21 | }, 22 | { 23 | "size": "1" 24 | } 25 | ] 26 | }, 27 | "type": "INT" 28 | }, 29 | { 30 | "name": "label", 31 | "type": "INT" 32 | } 33 | ] 34 | }, 35 | "sizeInBytes": "11594722", 36 | "splits": [ 37 | { 38 | "name": "test", 39 | "numShards": "1", 40 | "shardLengths": [ 41 | "10000" 42 | ], 43 | "statistics": { 44 | "features": [ 45 | { 46 | "name": "image", 47 | "numStats": { 48 | "commonStats": { 49 | "numNonMissing": "10000" 50 | }, 51 | "max": 255.0 52 | } 53 | }, 54 | { 55 | "name": "label", 56 | "numStats": { 57 | "commonStats": { 58 | "numNonMissing": "10000" 59 | }, 60 | "max": 9.0 61 | } 62 | } 63 | ], 64 | "numExamples": "10000" 65 | } 66 | }, 67 | { 68 | "name": "train", 69 | "numShards": "10", 70 | "shardLengths": [ 71 | "60000" 72 | ], 73 | "statistics": { 74 | "features": [ 75 | { 76 | "name": "image", 77 | "numStats": { 78 | "commonStats": { 79 | "numNonMissing": "60000" 80 | }, 81 | "max": 255.0 82 | } 83 | }, 84 | { 85 | "name": "label", 86 | "numStats": { 87 | "commonStats": { 88 | "numNonMissing": "60000" 89 | }, 90 | "max": 9.0 91 | } 92 | } 93 | ], 94 | "numExamples": "60000" 95 | } 96 | } 97 | ], 98 | "supervisedKeys": { 99 | "input": "image", 100 | "output": "label" 101 | }, 102 | "version": "3.0.0" 103 | } -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-4/data/mnist/3.0.1/image.image.json: -------------------------------------------------------------------------------- 1 | {"encoding_format": "png", "shape": [28, 28, 1]} -------------------------------------------------------------------------------- /2-custom_and_distributed_training/week-4/data/mnist/3.0.1/mnist-test.tfrecord-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/2-custom_and_distributed_training/week-4/data/mnist/3.0.1/mnist-test.tfrecord-00000-of-00001 -------------------------------------------------------------------------------- /3-Advance_computer_vision/week -1/C3_W1_Lab_2_Transfer_Learning_CIFAR_10.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "C3_W1_Lab_2_Transfer_Learning_CIFAR_10.ipynb", 8 | "private_outputs": true, 9 | "provenance": [], 10 | "collapsed_sections": [], 11 | "machine_shape": "hm" 12 | }, 13 | "kernelspec": { 14 | "display_name": "Python 3", 15 | "language": "python", 16 | "name": "python3" 17 | }, 18 | "language_info": { 19 | "codemirror_mode": { 20 | "name": "ipython", 21 | "version": 3 22 | }, 23 | "file_extension": ".py", 24 | "mimetype": "text/x-python", 25 | "name": "python", 26 | "nbconvert_exporter": "python", 27 | "pygments_lexer": "ipython3", 28 | "version": "3.7.4" 29 | } 30 | }, 31 | "cells": [ 32 | { 33 | "cell_type": "markdown", 34 | "metadata": { 35 | "id": "FStp_vbUkRz5" 36 | }, 37 | "source": [ 38 | "\n", 39 | "\n", 40 | "\n", 41 | "# Transfer Learning\n", 42 | "In this notebook, you will perform transfer learning to train CIFAR-10 dataset on ResNet50 model available in Keras.\n", 43 | "\n" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": { 49 | "id": "qpiJj8ym0v0-" 50 | }, 51 | "source": [ 52 | "## Imports" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "metadata": { 58 | "id": "AoilhmYe1b5t" 59 | }, 60 | "source": [ 61 | "import os, re, time, json\n", 62 | "import PIL.Image, PIL.ImageFont, PIL.ImageDraw\n", 63 | "import numpy as np\n", 64 | "try:\n", 65 | " # %tensorflow_version only exists in Colab.\n", 66 | " %tensorflow_version 2.x\n", 67 | "except Exception:\n", 68 | " pass\n", 69 | "import tensorflow as tf\n", 70 | "from tensorflow.keras.applications.resnet50 import ResNet50\n", 71 | "from matplotlib import pyplot as plt\n", 72 | "import tensorflow_datasets as tfds\n", 73 | "\n", 74 | "print(\"Tensorflow version \" + tf.__version__)" 75 | ], 76 | "execution_count": null, 77 | "outputs": [] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": { 82 | "id": "HuG_q_1jkaZ6" 83 | }, 84 | "source": [ 85 | "## Parameters" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": { 91 | "id": "v4ocPhg6J_xw" 92 | }, 93 | "source": [ 94 | "- Define the batch size\n", 95 | "- Define the class (category) names" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "metadata": { 101 | "id": "cCpkS9C_H7Tl" 102 | }, 103 | "source": [ 104 | "BATCH_SIZE = 32 \n", 105 | "classes = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']" 106 | ], 107 | "execution_count": null, 108 | "outputs": [] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": { 113 | "id": "O-o96NnyJ_xx" 114 | }, 115 | "source": [ 116 | "Define some functions that will help you to create some visualizations. (These will be used later)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "metadata": { 122 | "id": "CfFqJxrzoj5Q" 123 | }, 124 | "source": [ 125 | "#@title Visualization Utilities[RUN ME]\n", 126 | "#Matplotlib config\n", 127 | "plt.rc('image', cmap='gray')\n", 128 | "plt.rc('grid', linewidth=0)\n", 129 | "plt.rc('xtick', top=False, bottom=False, labelsize='large')\n", 130 | "plt.rc('ytick', left=False, right=False, labelsize='large')\n", 131 | "plt.rc('axes', facecolor='F8F8F8', titlesize=\"large\", edgecolor='white')\n", 132 | "plt.rc('text', color='a8151a')\n", 133 | "plt.rc('figure', facecolor='F0F0F0')# Matplotlib fonts\n", 134 | "MATPLOTLIB_FONT_DIR = os.path.join(os.path.dirname(plt.__file__), \"mpl-data/fonts/ttf\")\n", 135 | "# utility to display a row of digits with their predictions\n", 136 | "def display_images(digits, predictions, labels, title):\n", 137 | "\n", 138 | " n = 10\n", 139 | "\n", 140 | " indexes = np.random.choice(len(predictions), size=n)\n", 141 | " n_digits = digits[indexes]\n", 142 | " n_predictions = predictions[indexes]\n", 143 | " n_predictions = n_predictions.reshape((n,))\n", 144 | " n_labels = labels[indexes]\n", 145 | " \n", 146 | " fig = plt.figure(figsize=(20, 4))\n", 147 | " plt.title(title)\n", 148 | " plt.yticks([])\n", 149 | " plt.xticks([])\n", 150 | "\n", 151 | " for i in range(10):\n", 152 | " ax = fig.add_subplot(1, 10, i+1)\n", 153 | " class_index = n_predictions[i]\n", 154 | " \n", 155 | " plt.xlabel(classes[class_index])\n", 156 | " plt.xticks([])\n", 157 | " plt.yticks([])\n", 158 | " plt.imshow(n_digits[i])\n", 159 | "\n", 160 | "# utility to display training and validation curves\n", 161 | "def plot_metrics(metric_name, title, ylim=5):\n", 162 | " plt.title(title)\n", 163 | " plt.ylim(0,ylim)\n", 164 | " plt.plot(history.history[metric_name],color='blue',label=metric_name)\n", 165 | " plt.plot(history.history['val_' + metric_name],color='green',label='val_' + metric_name)" 166 | ], 167 | "execution_count": null, 168 | "outputs": [] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": { 173 | "id": "wPq4Sw5akosT" 174 | }, 175 | "source": [ 176 | "## Loading and Preprocessing Data\n", 177 | "[CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset has 32 x 32 RGB images belonging to 10 classes. You will load the dataset from Keras." 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "metadata": { 183 | "id": "E103YDdQ8NNq" 184 | }, 185 | "source": [ 186 | "(training_images, training_labels) , (validation_images, validation_labels) = tf.keras.datasets.cifar10.load_data()" 187 | ], 188 | "execution_count": null, 189 | "outputs": [] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": { 194 | "id": "prd944ThNavt" 195 | }, 196 | "source": [ 197 | "### Visualize Dataset\n", 198 | "\n", 199 | "Use the `display_image` to view some of the images and their class labels." 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "metadata": { 205 | "id": "UiokWTuKo88c" 206 | }, 207 | "source": [ 208 | "display_images(training_images, training_labels, training_labels, \"Training Data\" )" 209 | ], 210 | "execution_count": null, 211 | "outputs": [] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "metadata": { 216 | "id": "-q35q41KNfxH" 217 | }, 218 | "source": [ 219 | "display_images(validation_images, validation_labels, validation_labels, \"Training Data\" )" 220 | ], 221 | "execution_count": null, 222 | "outputs": [] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "metadata": { 227 | "id": "ltKfwrCVNuIu" 228 | }, 229 | "source": [ 230 | "### Preprocess Dataset\n", 231 | "Here, you'll perform normalization on images in training and validation set. \n", 232 | "- You'll use the function [preprocess_input](https://github.com/keras-team/keras-applications/blob/master/keras_applications/resnet50.py) from the ResNet50 model in Keras." 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "metadata": { 238 | "id": "JIxdiJVKArC6" 239 | }, 240 | "source": [ 241 | "def preprocess_image_input(input_images):\n", 242 | " input_images = input_images.astype('float32')\n", 243 | " output_ims = tf.keras.applications.resnet50.preprocess_input(input_images)\n", 244 | " return output_ims\n" 245 | ], 246 | "execution_count": null, 247 | "outputs": [] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "metadata": { 252 | "id": "QOqjKzgAEU-Z" 253 | }, 254 | "source": [ 255 | "train_X = preprocess_image_input(training_images)\n", 256 | "valid_X = preprocess_image_input(validation_images)" 257 | ], 258 | "execution_count": null, 259 | "outputs": [] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": { 264 | "id": "2fooPL9Gkuox" 265 | }, 266 | "source": [ 267 | "## Define the Network\n", 268 | "You will be performing transfer learning on **ResNet50** available in Keras.\n", 269 | "- You'll load pre-trained **imagenet weights** to the model.\n", 270 | "- You'll choose to retain all layers of **ResNet50** along with the final classification layers." 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "metadata": { 276 | "id": "56y8UNFQIVwj" 277 | }, 278 | "source": [ 279 | "'''\n", 280 | "Feature Extraction is performed by ResNet50 pretrained on imagenet weights. \n", 281 | "Input size is 224 x 224.\n", 282 | "'''\n", 283 | "def feature_extractor(inputs):\n", 284 | "\n", 285 | " feature_extractor = tf.keras.applications.resnet.ResNet50(input_shape=(224, 224, 3),\n", 286 | " include_top=False,\n", 287 | " weights='imagenet')(inputs)\n", 288 | " return feature_extractor\n", 289 | "\n", 290 | "\n", 291 | "'''\n", 292 | "Defines final dense layers and subsequent softmax layer for classification.\n", 293 | "'''\n", 294 | "def classifier(inputs):\n", 295 | " x = tf.keras.layers.GlobalAveragePooling2D()(inputs)\n", 296 | " x = tf.keras.layers.Flatten()(x)\n", 297 | " x = tf.keras.layers.Dense(1024, activation=\"relu\")(x)\n", 298 | " x = tf.keras.layers.Dense(512, activation=\"relu\")(x)\n", 299 | " x = tf.keras.layers.Dense(10, activation=\"softmax\", name=\"classification\")(x)\n", 300 | " return x\n", 301 | "\n", 302 | "'''\n", 303 | "Since input image size is (32 x 32), first upsample the image by factor of (7x7) to transform it to (224 x 224)\n", 304 | "Connect the feature extraction and \"classifier\" layers to build the model.\n", 305 | "'''\n", 306 | "def final_model(inputs):\n", 307 | "\n", 308 | " resize = tf.keras.layers.UpSampling2D(size=(7,7))(inputs)\n", 309 | "\n", 310 | " resnet_feature_extractor = feature_extractor(resize)\n", 311 | " classification_output = classifier(resnet_feature_extractor)\n", 312 | "\n", 313 | " return classification_output\n", 314 | "\n", 315 | "'''\n", 316 | "Define the model and compile it. \n", 317 | "Use Stochastic Gradient Descent as the optimizer.\n", 318 | "Use Sparse Categorical CrossEntropy as the loss function.\n", 319 | "''' \n", 320 | "\n", 321 | "\n", 322 | "def define_compile_model():\n", 323 | " inputs = tf.keras.layers.Input(shape=(32,32,3))\n", 324 | " \n", 325 | " classification_output = final_model(inputs) \n", 326 | " model = tf.keras.Model(inputs=inputs, outputs = classification_output)\n", 327 | " \n", 328 | " model.compile(optimizer='SGD', \n", 329 | " loss='sparse_categorical_crossentropy',\n", 330 | " metrics = ['accuracy'])\n", 331 | " \n", 332 | " return model\n", 333 | "\n", 334 | "\n", 335 | "model = define_compile_model()\n", 336 | "\n", 337 | "model.summary()" 338 | ], 339 | "execution_count": null, 340 | "outputs": [] 341 | }, 342 | { 343 | "cell_type": "markdown", 344 | "metadata": { 345 | "id": "CuhDh8ao8VyB" 346 | }, 347 | "source": [ 348 | "## Train the model" 349 | ] 350 | }, 351 | { 352 | "cell_type": "code", 353 | "metadata": { 354 | "id": "2K6RNDqtJ_xx" 355 | }, 356 | "source": [ 357 | "# this will take around 20 minutes to complete\n", 358 | "EPOCHS = 4\n", 359 | "history = model.fit(train_X, training_labels, epochs=EPOCHS, validation_data = (valid_X, validation_labels), batch_size=64)" 360 | ], 361 | "execution_count": null, 362 | "outputs": [] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "metadata": { 367 | "id": "CYb5sAEmk4ut" 368 | }, 369 | "source": [ 370 | "## Evaluate the Model\n", 371 | "\n", 372 | "Calculate the loss and accuracy metrics using the model's `.evaluate` function." 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "metadata": { 378 | "id": "io7Fuu-w3PZi" 379 | }, 380 | "source": [ 381 | "loss, accuracy = model.evaluate(valid_X, validation_labels, batch_size=64)" 382 | ], 383 | "execution_count": null, 384 | "outputs": [] 385 | }, 386 | { 387 | "cell_type": "markdown", 388 | "metadata": { 389 | "id": "yml-phRfPeOj" 390 | }, 391 | "source": [ 392 | "### Plot Loss and Accuracy Curves\n", 393 | "\n", 394 | "Plot the loss (in blue) and validation loss (in green)." 395 | ] 396 | }, 397 | { 398 | "cell_type": "code", 399 | "metadata": { 400 | "id": "b1ZMMJ6T921A" 401 | }, 402 | "source": [ 403 | "plot_metrics(\"loss\", \"Loss\")" 404 | ], 405 | "execution_count": null, 406 | "outputs": [] 407 | }, 408 | { 409 | "cell_type": "markdown", 410 | "metadata": { 411 | "id": "QbnWIbeJJ_xx" 412 | }, 413 | "source": [ 414 | "Plot the training accuracy (blue) as well as the validation accuracy (green)." 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "metadata": { 420 | "id": "P0YpFs3J99eO" 421 | }, 422 | "source": [ 423 | "plot_metrics(\"accuracy\", \"Accuracy\")" 424 | ], 425 | "execution_count": null, 426 | "outputs": [] 427 | }, 428 | { 429 | "cell_type": "markdown", 430 | "metadata": { 431 | "id": "9jFVovcUUVs1" 432 | }, 433 | "source": [ 434 | "### Visualize predictions\n", 435 | "You can take a look at the predictions on the validation set." 436 | ] 437 | }, 438 | { 439 | "cell_type": "code", 440 | "metadata": { 441 | "id": "NIQAqkMV9adq" 442 | }, 443 | "source": [ 444 | "probabilities = model.predict(valid_X, batch_size=64)\n", 445 | "probabilities = np.argmax(probabilities, axis = 1)\n", 446 | "\n", 447 | "display_images(validation_images, probabilities, validation_labels, \"Bad predictions indicated in red.\")" 448 | ], 449 | "execution_count": null, 450 | "outputs": [] 451 | } 452 | ] 453 | } -------------------------------------------------------------------------------- /3-Advance_computer_vision/week-2/results.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/3-Advance_computer_vision/week-2/results.data -------------------------------------------------------------------------------- /4-Generative-deeplearning-with-tensorflow/week-1/puppy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/4-Generative-deeplearning-with-tensorflow/week-1/puppy.png -------------------------------------------------------------------------------- /4-Generative-deeplearning-with-tensorflow/week-2/mymodel.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/4-Generative-deeplearning-with-tensorflow/week-2/mymodel.h5 -------------------------------------------------------------------------------- /4-Generative-deeplearning-with-tensorflow/week-4/my-signs.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasidhar-programmer/Tensorflow_Advance_Techniques/ead7af9aa2a07d60bcdd77919f6c83c8749076aa/4-Generative-deeplearning-with-tensorflow/week-4/my-signs.zip -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tensorflow_Advance_Techniques 2 | # This specialization from coursera consists of four courses 3 | ### 1 - Custom Models, Layers, and Loss Functions with TensorFlow 4 | 5 | 6 | • Compare Functional and Sequential APIs, discover new models you can build with the Functional API, and build a model that produces multiple outputs including a Siamese network. 7 | • Build custom loss functions (including the contrastive loss function used in a Siamese network) in order to measure how well a model is doing and help your neural network learn from training data. 8 | • Build off of existing standard layers to create custom layers for your models, customize a network layer with a lambda layer, understand the differences between them, learn what makes up a custom layer, and explore activation functions. 9 | • Build off of existing models to add custom functionality, learn how to define your own custom class instead of using the Functional or Sequential APIs, build models that can be inherited from the TensorFlow Model class, and build a residual network (ResNet) through defining a custom model class. 10 | 11 | 12 | The DeepLearning.AI TensorFlow: Advanced Techniques Specialization introduces the features of TensorFlow that provide learners with more control over their model architecture and tools that help them create and train advanced ML models. 13 | 14 | This Specialization is for early and mid-career software and machine learning engineers with a foundational understanding of TensorFlow who are looking to expand their knowledge and skill set by learning advanced TensorFlow features to build powerful models. 15 | ### 2 - Custom and Distributed Training with TensorFlow 16 | 17 | • Learn about Tensor objects, the fundamental building blocks of TensorFlow, understand the difference between the eager and graph modes in TensorFlow, and learn how to use a TensorFlow tool to calculate gradients. 18 | • Build your own custom training loops using GradientTape and TensorFlow Datasets to gain more flexibility and visibility with your model training. 19 | • Learn about the benefits of generating code that runs in graph mode, take a peek at what graph code looks like, and practice generating this more efficient code automatically with TensorFlow’s tools. 20 | • Harness the power of distributed training to process more data and train larger models, faster, get an overview of various distributed training strategies, and practice working with a strategy that trains on multiple GPU cores, and another that trains on multiple TPU cores. 21 | 22 | 23 | The DeepLearning.AI TensorFlow: Advanced Techniques Specialization introduces the features of TensorFlow that provide learners with more control over their model architecture and tools that help them create and train advanced ML models. 24 | 25 | This Specialization is for early and mid-career software and machine learning engineers with a foundational understanding of TensorFlow who are looking to expand their knowledge and skill set by learning advanced TensorFlow features to build powerful models. 26 | 27 | ### 3 - Advanced Computer Vision with TensorFlow 28 | 29 | In this course you will 30 | 31 | a) Explore image classification, image segmentation, object localization, and object detection. Apply transfer learning to object localization and detection. 32 | b) Apply object detection models such as regional-CNN and ResNet-50, customize existing models, and build your own models to detect, localize, and label your own rubber duck images. 33 | c) Implement image segmentation using variations of the fully convolutional network (FCN) including U-Net and d) Mask-RCNN to identify and detect numbers, pets, zombies, and more. 34 | d) Identify which parts of an image are being used by your model to make its predictions using class activation maps and saliency maps and apply these ML interpretation methods to inspect and improve the design of a famous network, AlexNet. 35 | 36 | 37 | The DeepLearning.AI TensorFlow: Advanced Techniques Specialization introduces the features of TensorFlow that provide learners with more control over their model architecture and tools that help them create and train advanced ML models. 38 | 39 | This Specialization is for early and mid-career software and machine learning engineers with a foundational understanding of TensorFlow who are looking to expand their knowledge and skill set by learning advanced TensorFlow features to build powerful models. 40 | 41 | ### 4 - Generative Deep Learning with TensorFlow 42 | 43 | In this course, you will 44 | 45 | a) Learn neural style transfer using transfer learning: extract the content of an image (eg. swan), and the style of a painting (eg. cubist or impressionist), and combine the content and style into a new image. 46 | b) Build simple AutoEncoders on the familiar MNIST dataset, and more complex deep and convolutional architectures on the Fashion MNIST dataset, understand the difference in results of the DNN and CNN AutoEncoder models, identify ways to de-noise noisy images, and build a CNN AutoEncoder using TensorFlow to output a clean image from a noisy one. 47 | c) Explore Variational AutoEncoders (VAEs) to generate entirely new data, and generate anime faces to compare them against reference images. 48 | d) Learn about GANs; their invention, properties, architecture, and how they vary from VAEs, understand the function of the generator and the discriminator within the model, the concept of 2 training phases and the role of introduced noise, and build your own GAN that can generate faces. 49 | 50 | The DeepLearning.AI TensorFlow: Advanced Techniques Specialization introduces the features of TensorFlow that provide learners with more control over their model architecture, and gives them the tools to create and train advanced ML models. 51 | 52 | This Specialization is for early and mid-career software and machine learning engineers with a foundational understanding of TensorFlow who are looking to expand their knowledge and skill set by learning advanced TensorFlow features to build powerful models. 53 | --------------------------------------------------------------------------------