├── A_tale_of_quantization.ipynb ├── Aerial Cactus Identification using TFHub.ipynb ├── Basic_AutoEncoder.ipynb ├── Bayesian_Search_W&B ├── Bayesian_Search_W&B.ipynb ├── sweep.yaml └── train.py ├── Benchmarking_TRT_TFLite.ipynb ├── Cats_vs_Dogs_TFRecords.ipynb ├── Character_Level_LSTM_Model ├── Character_Level_LSTM_TF_2.ipynb ├── README.md └── anna.txt ├── Custom_Image_Classification_EdgeTPU.ipynb ├── DSC_NSEC ├── Training a neural network.ipynb └── neural_network.py ├── EfficientNet.ipynb ├── Embedding_Preproc_Functions_in_Models.ipynb ├── GANs_with_TF_2_0.ipynb ├── GradientTape, Model Subclassing Experiments ├── Simple Experiments with TF 2.0 └── Simple Experiments with TF 2.0.ipynb ├── Hyperparameter Sweeps with TF 2.0 & W&B ├── Bayesian_Search_W&B.ipynb ├── Custom_TF_Training_Loops_WnB.ipynb ├── Demo_tensorboard_ii.ipynb ├── Hyperparameter Sweeps (part I) with TF 2.0 and W&B.ipynb └── TensorBoard Integration in W&B.ipynb ├── Installing_TensorFlow_Object_Detection_API.ipynb ├── Jigsaw Multilingual Toxic Comment Classification.ipynb ├── Keras_tuner_with_TF_2_0 ├── Keras_tuner_with_TF_2_0.ipynb └── diabetes.csv ├── LICENSE ├── ML GDE Dev Challenge ├── Predicting the publisher's name from an article.ipynb └── README.md ├── NALU in TF 2.0 ├── NALU_using_TF_2_0.ipynb ├── README.md ├── __pycache__ │ └── utils.cpython-37.pyc ├── figures │ ├── download (1).png │ ├── download (2).png │ └── download.png └── utils.py ├── NNLM_using_tf_keras.ipynb ├── Neural_Networks_as_Feature_Extractors.ipynb ├── Neural_Style_Transfer_(without_TV_loss)_in_TF_2_0.ipynb ├── Predicting publisher's name from an article with TF 2.0 ├── Data_Collection_and_basic_EDA.ipynb ├── Modeling_with_TensorFlow_2_0_and_Keras.ipynb └── data │ ├── test.csv │ ├── train.csv │ └── valid.csv ├── Prototypical_Networks_in_TensorFlow_2_0.ipynb ├── README.md ├── Running inference with a custom TFOD API model.ipynb ├── Simple_CBOW_in_TF_2.0.ipynb ├── Speed comparison between TF 1.x and TF 2.0 ├── FashionMNIST_TF1.ipynb ├── FashionMNIST_TF2.ipynb └── README.md ├── TF 2.0 Hack 1.ipynb ├── TF_2_0_and_cloud_functions.ipynb ├── Training_a_pets_detector_model_within_minutes_with_TFOD_API.ipynb ├── Transfer_Learning_EfficientNet.ipynb ├── Transfer_Learning_with_NSL_and_TF_2_0.ipynb ├── Universal_Sentence_Encoder_using_tf_keras.ipynb ├── Variational_AutoEncoder.ipynb ├── Weight_Initialization_in_Neural_Nets.ipynb ├── Weights_Visualization_TensorBoard.ipynb ├── data2vec_vision_image_classification.ipynb ├── hf_vision_model_tfserving.ipynb └── tf.data exploration ├── TFRecords_with_tf_data_Advanced.ipynb ├── TFRecords_with_tf_data_Advanced_(One_hot).ipynb ├── TFRecords_with_tf_data_Basics.ipynb ├── tf.data - Initial Experiments.ipynb └── tf_data_with_tf_keras.ipynb /Bayesian_Search_W&B/sweep.yaml: -------------------------------------------------------------------------------- 1 | program: train.py 2 | method: random 3 | metric: 4 | name: loss 5 | goal: minimize 6 | parameters: 7 | epochs: 8 | distribution: int_uniform 9 | min: 3 10 | max: 20 11 | layers: 12 | distribution: int_uniform 13 | min: 16 14 | max: 512 15 | method: 16 | distribution: categorical 17 | values: 18 | - '"grid"' 19 | batch_size: 20 | distribution: int_uniform 21 | min: 16 22 | max: 64 23 | -------------------------------------------------------------------------------- /Bayesian_Search_W&B/train.py: -------------------------------------------------------------------------------- 1 | # Imports 2 | from tensorflow.keras.models import * 3 | from tensorflow.keras.layers import * 4 | from wandb.keras import WandbCallback 5 | import tensorflow as tf 6 | import numpy as np 7 | import wandb 8 | import time 9 | 10 | # Fix the random generator seeds for better reproducibility 11 | tf.random.set_seed(67) 12 | np.random.seed(67) 13 | 14 | # Load the dataset 15 | fashion_mnist = tf.keras.datasets.fashion_mnist 16 | (train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data() 17 | 18 | # Scale the pixel values of the images to 19 | train_images = train_images / 255.0 20 | test_images = test_images / 255.0 21 | 22 | # Reshape the pixel values so that they are compatible with 23 | # the conv layers 24 | train_images = train_images.reshape(-1, 28, 28, 1) 25 | test_images = test_images.reshape(-1, 28, 28, 1) 26 | 27 | # Specify the labels of FashionMNIST dataset, it would 28 | # be needed later 😉 29 | labels = ["T-shirt/top","Trouser","Pullover","Dress","Coat", 30 | "Sandal","Shirt","Sneaker","Bag","Ankle boot"] 31 | 32 | # Prepare data tuples 33 | (X_train, y_train) = train_images, train_labels 34 | (X_test, y_test) = test_images, test_labels 35 | 36 | # Default values for hyper-parameters we're going to sweep over 37 | configs = { 38 | 'layers': 128, 39 | 'batch_size': 64, 40 | 'epochs': 5 41 | } 42 | 43 | # Initilize a new wandb run 44 | wandb.init(project='hyperparameter-sweeps-comparison', config=configs) 45 | 46 | # Config is a variable that holds and saves hyperparameters and inputs 47 | config = wandb.config 48 | 49 | # Add the config items to wandb (make sure you have logged wandb in) 50 | if wandb.run: 51 | wandb.config.update({k: v for k, v in configs.items() if k not in dict(wandb.config.user_items())}) 52 | 53 | # Define the model 54 | model = Sequential([ 55 | Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)), 56 | MaxPooling2D((2,2)), 57 | Conv2D(64, (3, 3), activation='relu'), 58 | MaxPooling2D((2,2)), 59 | Conv2D(64, (3, 3), activation='relu'), 60 | GlobalAveragePooling2D(), 61 | Dense(config.layers, activation=tf.nn.relu), 62 | Dense(10, activation='softmax') 63 | ]) 64 | 65 | # Compile the model 66 | model.compile(optimizer='adam', 67 | loss='sparse_categorical_crossentropy', 68 | metrics=['accuracy']) 69 | 70 | # Train the model 71 | model.fit(X_train, y_train, 72 | epochs=config.epochs, 73 | batch_size=config.batch_size, 74 | validation_data=(X_test, y_test), 75 | callbacks=[WandbCallback(data_type="image", 76 | validation_data=(X_test, y_test), labels=labels)]) 77 | -------------------------------------------------------------------------------- /Character_Level_LSTM_Model/README.md: -------------------------------------------------------------------------------- 1 | While taking the [Intro to Deep Learning with PyTorch](https://classroom.udacity.com/courses/ud188) course by Udacity, I really liked exercise that was based on building a character-level language model using LSTMs. I was unable to complete all on my own since NLP is still a very new field to me. I decided to give the exercise a try with `tensorflow 2.0` and because of the ease of use you get in `keras`, I could develop a very simple LSTM-based language model able to predict a single character given a set of characters. 2 | 3 | The exercise uses the **Anna Karenina** novel, written by Leo Tolstoy as its data. I used a small subset of it in this notebook, though. 4 | -------------------------------------------------------------------------------- /DSC_NSEC/Training a neural network.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "2.0.0\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "# Dependencies\n", 18 | "import tensorflow as tf \n", 19 | "import matplotlib.pyplot as plt\n", 20 | "\n", 21 | "print(tf.__version__)" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": {}, 27 | "source": [ 28 | "If you don't have TensorFlow 2.0 installed install it by uncommenting and executing the following cell:" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "# pip install tensorflow" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 3, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "# Prepare dataset\n", 47 | "def prepare_dataset(m=0.1, b=0.3, num_samples=100):\n", 48 | " x = tf.random.uniform(shape=(num_samples,))\n", 49 | " noise = tf.random.normal(shape=(len(x),), stddev=0.01)\n", 50 | " y = m*x + b + noise\n", 51 | " return (x, y)\n", 52 | "\n", 53 | "\n", 54 | "X_train, y_train = prepare_dataset()" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 4, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "data": { 64 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGghJREFUeJzt3X+MXedd5/H3x+PaS0K7RY5XorYHB+RIpDUS1ZB0pGXxpknXaqQYKezKWJCwqmICNluTwjYWKRulqtqyu8ZdrdXgds3arIIpVLsa1LCWmsUqVJPiCaUFG4UYtzgOaJuabKNQOv4x3/3jnJvc3NyZe+6d8/t8XlI099x77p3n2M7nPPf7POc5igjMzKwb1lTdADMzK49D38ysQxz6ZmYd4tA3M+sQh76ZWYc49M3MOsShb2bWIQ59M7MOceibmXXI2qobMOimm26KrVu3Vt0MM7NGeeaZZ74ZERtH7Ve70N+6dSsLCwtVN8PMrFEk/U2W/VzeMTPrEIe+mVmHOPTNzDrEoW9m1iEOfTOzDnHom5l1iEPfzKxE8/Pw0Y8mP6tQu3n6ZmZtNT8P7343XLkC69bBU0/B7Gy5bXBP38ysJKdPJ4F//Xry8/Tp8tvg0DczK8mOHUkPf2oq+bljR/ltcHnHzKwks7NJSef06STwyy7tgEPfzKxUs7PVhH2PyztmZh3i0Dcz6xCHvplZhzj0zcw6xKFvZtYhDn0zsxxVvczCKJ6yaWaWkzosszCKe/pmZjmpwzILozj0zcxyUodlFkZxecfMLCd1WGZhFIe+mVmOql5mYZRM5R1JOyU9K+m8pIdX2O9eSSFppu+5g+n7npX0r/JotJmZTWZkT1/SFHAEuAu4BJyRNBcR5wb2ezPwfuBLfc/dCuwG3g68Dfi8pFsi4np+h2BmZlll6enfBpyPiAsRcQU4Cewast+HgY8D3+l7bhdwMiIWI+JrwPn088zMrAJZQn8T8Hzf9qX0uVdJeiewJSI+N+57zcysPKuesilpDXAI+MAqPmOvpAVJCy+++OJqm2RmZsvIEvovAFv6tjenz/W8GXgHcFrS14F3AXPpYO6o9wIQEUcjYiYiZjZu3DjeEZiZFaTuSypMIsuUzTPANkk3kwT2bmBP78WI+BZwU29b0mnglyJiQdI/Ak9IOkQykLsN+JP8mm9mVowmLKkwiZGhHxHXJO0HTgFTwLGIOCvpMWAhIuZWeO9ZSZ8BzgHXgH2euWNmTTC4pMKJE/W+6CorRUTVbXidmZmZWFhYqLoZZtZS8/PZwru/p792LUQkJ4Berx/qdRKQ9ExEzIzaz1fkmllnjFOymZ2Fw4fhs5+FG26A3//91/f6jx9vZunHoW9mnTFsFczlwnp+Hg4ceK2nPzWVPL9uXfIz6+cMfmbV3w4c+mbWGb1VMHs99JVWwew/QQA88ABMT7/2nv6efpbVNOsyMOzQN7POGGcVzMETxH33vX7/cVfTHOdbRpEc+mbWKVlXwRx1ghh3Nc1R3zLKKv049M2sMcquiee5TPJKJ5EySz8OfTNrhLrUxFdjuZNImaUf3y7RzBqhCfefndSGDSDBmjXF32bRoW9mjdCE+89Oojc1dGkpObbDh13TNzNrxP1nJ9H7BrO0lPT2L18u9vc59M2sMep+/9lJjHPtQB4c+mZmFSr7G4xD38waq8gpnGVODy3zG4xD38xqb1gAFzmFsw3TQ5fj2TtmVmu9AP7Qh5KfvbtYFTmFs+zpoWXeocs9fTOrteUuXCpyALTMwdWyv1U49M2s1pYL4CIHQMscXC17ITaHvpnV2nIBPMlA6zjvKWtw1VM2zcwGDAbwciWRlUK9roOznrJpZq3WC+YNG5KrTycJuuUGWlcK9bqsZz+Mp2yaWSv1etuLi8myA2vWwPr14/e6h5VERoV62WWUunLom1lp+teZgeTnJL3u5UoiK4V6W9fuGZdD38xK0+tt9/f0J+11D5ZEsoR6G9fuGZdD38xK0x/Mq6npr/T5XQ/1URz6ZlYqB3O1vAyDmVmHZAp9STslPSvpvKSHh7z+oKQ/l/Rnkv5Y0q3p82+SdDx97S8lHcz7AMysfVZai6bMdWraaGR5R9IUcAS4C7gEnJE0FxHn+nZ7IiIeT/e/BzgE7AT+NbA+IrZLugE4J+m3I+LrOR+HmTVAlitiV7qIqogLrMpcQrkOstT0bwPOR8QFAEkngV3Aq6EfES/37X8jEL2XgBslrQW+C7gC9O9rZh0xGNiHDw8fyF1pvn3eF1jV9SrdImUJ/U3A833bl4DbB3eStA94CFgH3JE+/XskJ4i/A24AfjEi/n7Ie/cCewGmp6fHaL6ZNUV/YC8uwr59EPHGsF3pIqq8L7Cq81W6RcltIDcijkTEDwAfBB5Jn74NuA68DbgZ+ICk7x/y3qMRMRMRMxs3bsyrSWaN1rbadS+wp6aS/5aWhq9X35vW+eEPv7HnvdJrq21TV67SzdLTfwHY0re9OX1uOSeBT6aP9wD/OyKuAt+Q9EVgBrgwQVvNOqONZYfBOfoHDqx89exyx5vnlM8uXqWbJfTPANsk3UwS9rtJwvxVkrZFxHPp5t1A7/FFklLPb0m6EXgXcDiPhpu1WVvLDv2BvX17PcK2a9cNjAz9iLgmaT9wCpgCjkXEWUmPAQsRMQfsl3QncBV4Cbg/ffsR4DclnQUE/GZEfLWIAzFrky4sDta1sK0LRcTovUo0MzMTCwsLVTfDrHJdm0poqyPpmYiYGbWfl2Ewqyn3hFfPJ843cuibWSu1cTA8D157x6xCbZuWWSfL3V2r69zTN6uIe6LF6sJg+CQc+mYVaeu0zLro4hz8LBz6ZhVpa0+0ToOnHgx/I4e+WUXa2BOtY8mqTiehOnDom1WoyT3RYWFaVskqa5DX8SRUNYe+mY1tuTAto2Q1TpB73OSNPGXTzMa23HTIvFfBHOd3D9PFVTRHcU/fzMa2Uo++6JJVlm8T/eWfto2brJZD36wAbR88rHIQetTvHlb+Oei7c7/KoW+Ws6IGD+t2Iunv0ZfdtpW+TbiOvzKHvlkG44RaEaFT51koRbdt3BNKW69/yItD32yEcUOtiNCpc++1yLZNckJp4/UPeXLom40wbqgVETp17r0W2bZJTyhNvv6haA59sxEmCbW8Q2fwRALJ6px16MkW2bOu88muqXznLLMM6jSIWqf6fhl/LnX6s68z3znLLEd1KhfUpb5f1smnTn/2beArcs0api5XmfomJc3knr5Zw9Rldorr7c3k0DdroDqUPOpy8rHxOPTNxjDpoGJbByPrcPKx8Tj0zTKadOCyTrNtzDyQa5bR4MDliRPJXPn5+fHeV4cBz/n5bG239snU05e0E/gEMAV8OiI+NvD6g8A+4DrwCrA3Is6lr/0Q8BvAW4Al4Eci4ju5HYFZSfoHLteuhWPHkiAf1Xuv24DnsG8e0M7yk73RyNCXNAUcAe4CLgFnJM31Qj31REQ8nu5/D3AI2ClpLfA/gJ+OiK9I2gBczfsgzMrQP3B58SJ86lMrz5Wv65ruw76xHD/u8lNXZOnp3wacj4gLAJJOAruAV0M/Il7u2/9GoHeZ73uAr0bEV9L9LufRaLOq9AYu5+dfH5SDvfc6r+k++M0D6nGxl5UjS+hvAp7v274E3D64k6R9wEPAOuCO9OlbgJB0CtgInIyIX1tVi80GVDEzZtR0xbpcNTvMsHV8VjqBWbvkNnsnIo4ARyTtAR4B7k8//58DPwJ8G3gqXR/iqf73StoL7AWYnp7Oq0nWAVXOjFlpumLd6viDBttep/KTFStL6L8AbOnb3pw+t5yTwCfTx5eAL0TENwEkPQm8E3hd6EfEUeAoJAuuZWq5NV4ePfS69qibduGS59t3R5bQPwNsk3QzSdjvBvb07yBpW0Q8l27eDfQenwL+vaQbgCvAjwG/nkfDrdny6qHXuUftILU6Ghn6EXFN0n6SAJ8CjkXEWUmPAQsRMQfsl3Qnycycl0hKO0TES5IOkZw4AngyIj5X0LFYg+TVQ29aj9qsal5P3yrhq1TN8uX19K3W3EM3q4ZD3yrjmrdZ+bz2jplZhzj0zUbw4mTWJi7vmK3AA87WNu7pm62gjssim62GQ99sBePehLxOpaA6tcXqw+UdsxWMM7W0TqWgOrXF6sWhbzZC1qmldVoHqE5tsXpxeccstdpyyLiloCLl3RaXitrDPX0zVlcOGbxD1okTRbY0mzyveHapqF0c+mZMXg4ZDMTDh1+7Icnx49UGZF5XPLtU1C4u75gxeTlkMBA/+9n2TfGsU9nKVs89fWutcW7SMmk5ZHA9/3vvhT/6ozeu71/FLR3z4sXx2sVLK1srDIZqmXXoYb+7qrZYd3lpZeuMYaFaZh16sHY+uO2auNWJa/rWeMNCNY86dF7TFF0TtzpxT98ab9h9cldbh86zJOOauNWJQ98ab7lQXc2UxeUWWps0uH3DGKsLh761Qt6hOvjtYcOGYgdjmzy7x5rFoW82xOC3hyIHYz27x8rk0DdbxuC3h8Fxg7x4do+VyaFvlkGRg7HDBqLNiuLQt9bo1cU3bIDLl/MP56IGYz27x8rk0LdW6NXFFxdhaQnWrIH165tTH/fsHiuLL86yUgxe6JT3+uy9uvjSUrK9tFTfBc+8Nr1VKVNPX9JO4BPAFPDpiPjYwOsPAvuA68ArwN6IONf3+jRwDng0Iv5TTm23hhi2/PCBA/nOVunVxft7+nWsj3umjlVtZOhLmgKOAHcBl4Azkub6Qx14IiIeT/e/BzgE7Ox7/RDwB7m12holy/LDqw2+/rp4UTX9PHimjlUtS0//NuB8RFwAkHQS2EXScwcgIl7u2/9G4NWlOyX9OPA14B/yaLA1T9blh1er7nXx+Xm4eBHWpv/X1fGbiLVfltDfBDzft30JuH1wJ0n7gIeAdcAd6XPfDXyQ5FvCLy33CyTtBfYCTE9PZ2y6NcWw2Snbt3drtkp/WWdqCh54AO67rxvHbvWS2+ydiDgCHJG0B3gEuB94FPj1iHhF0krvPQochWQ9/bzaZPUxavnhtusv6wBMT3fr+K0+soT+C8CWvu3N6XPLOQl8Mn18O/ATkn4NeCuwJOk7EfFfJ2ms2TBNWLfGF2BZXWQJ/TPANkk3k4T9bmBP/w6StkXEc+nm3cBzABHxo337PAq84sC3PDVlNowvwLK6GBn6EXFN0n7gFMmUzWMRcVbSY8BCRMwB+yXdCVwFXiIp7dgymtAzbYomzYbpWknL6ilTTT8ingSeHHjuV/sevz/DZzw6buPaqCk906Zw2cRsPF6GoWRN6pk2gcsmZuNx6JfMPdP8uWxilp1Dv2TumZpZlRz6FXDP1Myq4lU2zcw6xKFvZtYhDn0zsw5x6JuZdYhD31olz7tS+Q5X1kaevWOtkefVzr5y2trKPX1rjWFXO+f9We79W9O5p98CXsAtkefVzsM+y71/awOHfsM5iF6T59XOwz7rox/1uknWfA79hvMCbq+X59XOg5/ldZOsDRz6DVd2EPVKSRs2wOXL3Soped0kawOHfsOVGUS9UtLiIiwtwZo1sH59t0pKXjfJms6h3wJlBVGvlLS0lGwvLbmkZNY0nrJpmfVKSWvSfzVr1ri2bdY07ulbZv2lpC7W9M3awKFvY3FN26zZXN4xM+sQh76ZWYc49DusynVkvIaNWTVc0++oKpdv8NIRZtVxT7+j8lyRskm/26zrMoW+pJ2SnpV0XtLDQ15/UNKfS/ozSX8s6db0+bskPZO+9oykO/I+AJtMb8791FT5c+2r/N2juOxkbaeIWHkHaQr4K+Au4BJwBvjJiDjXt89bIuLl9PE9wM9HxE5JPwz834j4W0nvAE5FxKaVft/MzEwsLCys6qAsmyqXZK7jctAuO1mTSXomImZG7Zelpn8bcD4iLqQffBLYBbwa+r3AT90IRPr8l/uePwt8l6T1EbGY4fdawcqccz8Y8nWc7+8VS60LsoT+JuD5vu1LwO2DO0naBzwErAOGlXHuBf60ToFfZG+zjj3ZqjSlB+2lk60Lcpu9ExFHgCOS9gCPAPf3XpP0duDjwHuGvVfSXmAvwPT0dF5NWlGRQdSUkCtLU3rQXjrZuiDLQO4LwJa+7c3pc8s5Cfx4b0PSZuB/AvdFxF8Pe0NEHI2ImYiY2bhxY4YmrV6RM0g8O+X16jxwO2h2Fg4edOBbe2Xp6Z8Btkm6mSTsdwN7+neQtC0inks37waeS59/K/A54OGI+GJurc5BkV/lXSZ4Pfegzepj5OwdAEnvBQ4DU8CxiPiIpMeAhYiYk/QJ4E7gKvASsD8izkp6BDhIehJIvScivrHc7ypz9o5r+mbWFlln72QK/TJ5yqaZ2fiyhr6vyDUz6xCHfkGKvLJzks/2laZmBl5wrRBHj8L+/cnsnbxvHD5sOiisPH7gKaRm1uOefs7m52HfPrh6Nblx+OIiPPpofj3swemgJ04kgf6hDyU/h/0eTyE1sx6Hfs5On07CvmdpCT7/+eUDeTnLlWMG57zD6EBv0jx5MyuWyzs527EjKekspotNRCTBP86VqCuVYwbnvAMcP77yNQGeJ29mPQ79nPUH7IYNcODA+BdpjVq2YHCxsiyBXscFzsDXM5iVzaFfgP6A3b59/FAb94reugb6KB5gNiufQ79gkwRyV8oxTVmIzaxNHPo11dTe+zi8RpFZ+Rz6VpmufKMxqxOHvlWqC99ozOrE8/TNzDrEoW9m1iEOfTOzDnHom5l1iEN/Ql6q2MyayLN3JlDUlaReksDMiubQn0ARV5J6SQIzK4PLOxMoYqlir3lvZmVwT38CRVxJ6iUJzKwMDv0J5X0lqZckMLMyOPT7VD2Q6iUJzKxoDv2UB1LNrAs8kJvyQKqZdYFDP9XEm4f7AjEzG1em8o6kncAngCng0xHxsYHXHwT2AdeBV4C9EXEufe0g8L70tX8XEafya35+mjaQ6nKUmU1iZOhLmgKOAHcBl4AzkuZ6oZ56IiIeT/e/BzgE7JR0K7AbeDvwNuDzkm6JiOs5H0cumjSQ6lsNmtkkspR3bgPOR8SFiLgCnAR29e8QES/3bd4IRPp4F3AyIhYj4mvA+fTzaq0JZZMmlqPMrHpZyjubgOf7ti8Btw/uJGkf8BCwDrij771PD7x305D37gX2AkxPT2dpd2GaUjZpWjnKzOoht4HciDgSET8AfBB4ZMz3Ho2ImYiY2bhx48RtyKOH3qRZPLOzcPCgA9/MssvS038B2NK3vTl9bjkngU9O+N6J5dFDn5+HixdhbfqnslLZpOoLuczMJpEl9M8A2yTdTBLYu4E9/TtI2hYRz6WbdwO9x3PAE5IOkQzkbgP+JI+GD1rtwGb/SWNqCh54AO67b/hnNKUEZGY2aGToR8Q1SfuBUyRTNo9FxFlJjwELETEH7Jd0J3AVeAm4P33vWUmfAc4B14B9Rc3cmWTBsvl5OHHite3eSQNgenr5IPfMGTNrqkzz9CPiSeDJged+te/x+1d470eAj0zawKzGHdicn0/2u3Il2X7Tm7KVdeC1E8ziIkiwYcOqm29mVopWrb0zzjz706fh6tXXtq9dg5/92aSHP+qkMTsLhw/D/v1Jb//AAdi+3b19M6u/VoX+OHbsSHr3vZ7+unXL1/CHuXwZlpaS/1ziMbOm6Gzoz84mQd2r6Y8T+OCbnphZM3U29GF1yy744igza6JOh/5qNWmtHjMz8NLKZmad4tA3M+sQh76ZWYc49M3MOsShb2bWIQ59M7MOUUSM3qtEkl4E/mbMt90EfLOA5tSdj7tbfNzdMu5xf19EjLwhSe1CfxKSFiJipup2lM3H3S0+7m4p6rhd3jEz6xCHvplZh7Ql9I9W3YCK+Li7xcfdLYUcdytq+mZmlk1bevpmZpZBY0Jf0k5Jz0o6L+nhIa+vl/Q76etfkrS1/FbmL8NxPyTpnKSvSnpK0vdV0c68jTruvv3ulRSSWjG7I8txS/o36d/5WUlPlN3GImT4dz4t6Q8lfTn9t/7eKtqZN0nHJH1D0l8s87ok/Zf0z+Wrkt656l8aEbX/j+SG7H8NfD+wDvgKcOvAPj8PPJ4+3g38TtXtLum4/yVwQ/r457py3Ol+bwa+ADwNzFTd7pL+vrcBXwa+J93+Z1W3u6TjPgr8XPr4VuDrVbc7p2P/F8A7gb9Y5vX3An8ACHgX8KXV/s6m9PRvA85HxIWIuAKcBHYN7LMLOJ4+/j3g3ZJUYhuLMPK4I+IPI+Lb6ebTwOaS21iELH/fAB8GPg58p8zGFSjLcT8AHImIlwAi4hslt7EIWY47gLekj/8p8Lcltq8wEfEF4O9X2GUXcCISTwNvlfS9q/mdTQn9TcDzfduX0ueG7hMR14BvARtKaV1xshx3v/eR9AqabuRxp19zt0TE58psWMGy/H3fAtwi6YuSnpa0s7TWFSfLcT8K/JSkS8CTwC+U07TKjZsBI/nOWS0h6aeAGeDHqm5L0SStAQ4BP1NxU6qwlqTEs4PkW90XJG2PiP9XaauK95PAf4+I/yxpFvgtSe+IiKWqG9Y0TenpvwBs6dvenD43dB9Ja0m+Al4upXXFyXLcSLoT+BXgnohYLKltRRp13G8G3gGclvR1klrnXAsGc7P8fV8C5iLiakR8DfgrkpNAk2U57vcBnwGIiHngn5CsTdN2mTJgHE0J/TPANkk3S1pHMlA7N7DPHHB/+vgngP8T6UhIg408bkk/DPwGSeC3ob4LI447Ir4VETdFxNaI2EoylnFPRCxU09zcZPl3/r9IevlIuomk3HOhzEYWIMtxXwTeDSDpB0lC/8VSW1mNOeC+dBbPu4BvRcTfreYDG1HeiYhrkvYDp0hG+o9FxFlJjwELETEH/DeSr3znSQZGdlfX4nxkPO7/CHw38LvpuPXFiLinskbnIONxt07G4z4FvEfSOeA68MsR0ehvtBmP+wPApyT9Ismg7s+0oFOHpN8mOYnflI5X/AfgTQAR8TjJ+MV7gfPAt4F/u+rf2YI/NzMzy6gp5R0zM8uBQ9/MrEMc+mZmHeLQNzPrEIe+mVmHOPTNzDrEoW9m1iEOfTOzDvn/ENNr8kZjxQMAAAAASUVORK5CYII=\n", 65 | "text/plain": [ 66 | "
" 67 | ] 68 | }, 69 | "metadata": { 70 | "needs_background": "light" 71 | }, 72 | "output_type": "display_data" 73 | } 74 | ], 75 | "source": [ 76 | "# Plot the dataset\n", 77 | "plt.plot(X_train, y_train, 'b.')\n", 78 | "plt.show()" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 5, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "# Initialize m and b\n", 88 | "m = tf.Variable(0.)\n", 89 | "b = tf.Variable(0.)" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 17, 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "# Forward pass\n", 99 | "def calculate_y(x):\n", 100 | " return x*m + b\n", 101 | "\n", 102 | "# Calculate the error using MSE\n", 103 | "def mse(y_pred, y_true):\n", 104 | " return tf.reduce_mean(tf.square(y_pred - y_true))" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 18, 110 | "metadata": {}, 111 | "outputs": [ 112 | { 113 | "name": "stdout", 114 | "output_type": "stream", 115 | "text": [ 116 | "Loss: 0.12479435\n" 117 | ] 118 | } 119 | ], 120 | "source": [ 121 | "# A sample forward pass\n", 122 | "y_pred = calculate_y(X_train)\n", 123 | "\n", 124 | "# Calculate the error/loss and log\n", 125 | "loss = mse(y_pred, y_train)\n", 126 | "print(\"Loss: \",loss.numpy())" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": {}, 132 | "source": [ 133 | "To be continued. " 134 | ] 135 | } 136 | ], 137 | "metadata": { 138 | "kernelspec": { 139 | "display_name": "Python 3", 140 | "language": "python", 141 | "name": "python3" 142 | }, 143 | "language_info": { 144 | "codemirror_mode": { 145 | "name": "ipython", 146 | "version": 3 147 | }, 148 | "file_extension": ".py", 149 | "mimetype": "text/x-python", 150 | "name": "python", 151 | "nbconvert_exporter": "python", 152 | "pygments_lexer": "ipython3", 153 | "version": "3.7.1" 154 | } 155 | }, 156 | "nbformat": 4, 157 | "nbformat_minor": 2 158 | } 159 | -------------------------------------------------------------------------------- /DSC_NSEC/neural_network.py: -------------------------------------------------------------------------------- 1 | def neural_network(inputs, weight): 2 | prediction = weighted_sum(inputs, weight) 3 | return prediction 4 | 5 | def weighted_sum(inputs, weight): 6 | output = 0 7 | for i in range(len(inputs)): 8 | output += inputs[i] * weights[i] 9 | return output 10 | 11 | number_of_all_rounders = 3 12 | number_of_batsmen = 4 13 | number_of_bowlers = 3 14 | 15 | inputs = [number_of_all_rounders, number_of_batsmen, number_of_bowlers] 16 | 17 | weights = [0.01, 0.2, 0.03] 18 | 19 | print(neural_network(inputs, weights)) -------------------------------------------------------------------------------- /GANs_with_TF_2_0.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "GANs with TF 2.0.ipynb", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "id": "view-in-github", 23 | "colab_type": "text" 24 | }, 25 | "source": [ 26 | "\"Open" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": { 32 | "id": "s9vaOlDrphRN", 33 | "colab_type": "text" 34 | }, 35 | "source": [ 36 | "This notebook follows [this amazing tutorial on GANs](https://medium.com/@devnag/generative-adversarial-networks-gans-in-50-lines-of-code-pytorch-e81b79659e3f) and tries to port the code to TensorFlow 2.0. \n" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": { 42 | "id": "qn636Xcba-Q1", 43 | "colab_type": "text" 44 | }, 45 | "source": [ 46 | "## Install `Tensorflow 2.0`" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "metadata": { 52 | "id": "Qp5esV6aANHl", 53 | "colab_type": "code", 54 | "colab": {} 55 | }, 56 | "source": [ 57 | "!pip install tensorflow-gpu==2.0.0-beta1" 58 | ], 59 | "execution_count": 0, 60 | "outputs": [] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": { 65 | "id": "6jIXCCVsbC-K", 66 | "colab_type": "text" 67 | }, 68 | "source": [ 69 | "## Imports" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "metadata": { 75 | "id": "jGB-ehhCAsO0", 76 | "colab_type": "code", 77 | "colab": {} 78 | }, 79 | "source": [ 80 | "import numpy as np\n", 81 | "import tensorflow as tf\n", 82 | "import matplotlib.pyplot as plt\n", 83 | "\n", 84 | "from tensorflow import keras\n", 85 | "\n", 86 | "%matplotlib inline" 87 | ], 88 | "execution_count": 0, 89 | "outputs": [] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": { 94 | "id": "l7EUQq5NbFSe", 95 | "colab_type": "text" 96 | }, 97 | "source": [ 98 | "## Helper function to generate a distribution (normal) for the real data" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "metadata": { 104 | "id": "7G6PMW-RBTpC", 105 | "colab_type": "code", 106 | "colab": {} 107 | }, 108 | "source": [ 109 | "def get_distribution_sampler(mu, sigma):\n", 110 | " return lambda n: tf.convert_to_tensor(np.random.normal(mu, sigma, (1, n)))" 111 | ], 112 | "execution_count": 0, 113 | "outputs": [] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": { 118 | "id": "loyhvfRxbLX2", 119 | "colab_type": "text" 120 | }, 121 | "source": [ 122 | "## Helper function to generate a uniform distribution for the generator network" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "metadata": { 128 | "id": "NzGT3Q8zBzht", 129 | "colab_type": "code", 130 | "colab": {} 131 | }, 132 | "source": [ 133 | "def get_generator_input_sampler():\n", 134 | " return lambda m, n: tf.convert_to_tensor(np.random.rand(m, n))" 135 | ], 136 | "execution_count": 0, 137 | "outputs": [] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "metadata": { 142 | "id": "ZG6Py0Xmbczy", 143 | "colab_type": "text" 144 | }, 145 | "source": [ 146 | "## The Generator network class" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "metadata": { 152 | "id": "4ifARasTCMw0", 153 | "colab_type": "code", 154 | "colab": {} 155 | }, 156 | "source": [ 157 | "class Generator(keras.Model):\n", 158 | " def __init__(self, input_size, hidden_size, output_size):\n", 159 | " super(Generator, self).__init__()\n", 160 | " self.map1 = keras.layers.Dense(hidden_size, input_shape=input_size, activation='tanh')\n", 161 | " self.map2 = keras.layers.Dense(hidden_size, activation='tanh')\n", 162 | " self.map3 = keras.layers.Dense(output_size, activation='linear')\n", 163 | " \n", 164 | " def call(self, inputs):\n", 165 | " x = self.map1(inputs)\n", 166 | " x = self.map2(x)\n", 167 | " x = self.map3(x)\n", 168 | " return x" 169 | ], 170 | "execution_count": 0, 171 | "outputs": [] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": { 176 | "id": "DZSWuojAbgDE", 177 | "colab_type": "text" 178 | }, 179 | "source": [ 180 | "## The Discriminator network class" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "metadata": { 186 | "id": "FAAzJHCFEp9e", 187 | "colab_type": "code", 188 | "colab": {} 189 | }, 190 | "source": [ 191 | "class Discriminator(keras.Model):\n", 192 | " def __init__(self, input_size, hidden_size, output_size):\n", 193 | " super(Discriminator, self).__init__()\n", 194 | " self.map1 = keras.layers.Dense(hidden_size, input_shape=input_size, activation='sigmoid')\n", 195 | " self.map2 = keras.layers.Dense(hidden_size, activation='sigmoid')\n", 196 | " self.map3 = keras.layers.Dense(output_size, activation='sigmoid')\n", 197 | " \n", 198 | " def call(self, inputs):\n", 199 | " x = self.map1(inputs)\n", 200 | " return self.map3(self.map2(x))" 201 | ], 202 | "execution_count": 0, 203 | "outputs": [] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "metadata": { 208 | "id": "9-PY9BZuFgqr", 209 | "colab_type": "code", 210 | "colab": {} 211 | }, 212 | "source": [ 213 | "def get_moments(d):\n", 214 | " # https://stats.stackexchange.com/questions/126346/why-kurtosis-of-a-normal-distribution-is-3-instead-of-0\n", 215 | " # Return the first 4 moments of the data provided\n", 216 | " d = tf.transpose(d, (1, 0))\n", 217 | " mean = tf.reduce_mean(d)\n", 218 | " diffs = (d - mean)\n", 219 | " var = tf.reduce_mean(tf.pow(diffs, 2.0))\n", 220 | " std = tf.sqrt(var)\n", 221 | " zscores = diffs / std\n", 222 | " skews = tf.reduce_mean(tf.pow(zscores, 3.0))\n", 223 | " kurtoses = tf.reduce_mean(tf.pow(zscores, 4.0)) - 3.0 # excess kurtosis, should be 0 for Gaussian\n", 224 | " return tf.stack([mean, std, skews, kurtoses], axis=0)" 225 | ], 226 | "execution_count": 0, 227 | "outputs": [] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "metadata": { 232 | "id": "3OV4KWgRJy7i", 233 | "colab_type": "code", 234 | "colab": {} 235 | }, 236 | "source": [ 237 | "def stats(d):\n", 238 | " return [np.mean(d), np.std(d)]" 239 | ], 240 | "execution_count": 0, 241 | "outputs": [] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "metadata": { 246 | "id": "WzKl011RXlLU", 247 | "colab_type": "text" 248 | }, 249 | "source": [ 250 | "## Model hyperparameters and other constants" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "metadata": { 256 | "id": "-qj45CReOW9c", 257 | "colab_type": "code", 258 | "colab": {} 259 | }, 260 | "source": [ 261 | "# Model parameters\n", 262 | "g_input_size = 1 # Random noise dimension coming into generator, per output vector\n", 263 | "g_hidden_size = 5 # Generator complexity\n", 264 | "g_output_size = 1 # Size of generated output vector\n", 265 | "d_input_size = 500 # Minibatch size - cardinality of distributions\n", 266 | "d_hidden_size = 10 # Discriminator complexity\n", 267 | "d_output_size = 1 # Single dimension for 'real' vs. 'fake' classification\n", 268 | "minibatch_size = d_input_size\n", 269 | "\n", 270 | "d_learning_rate = 1e-3\n", 271 | "g_learning_rate = 1e-3\n", 272 | "sgd_momentum = 0.9\n", 273 | "\n", 274 | "num_epochs = 5000\n", 275 | "print_interval = 100\n", 276 | "d_steps = 20\n", 277 | "g_steps = 20\n", 278 | "\n", 279 | "dfe, dre, ge = 0, 0, 0\n", 280 | "d_real_data, d_fake_data, g_fake_data = None, None, None" 281 | ], 282 | "execution_count": 0, 283 | "outputs": [] 284 | }, 285 | { 286 | "cell_type": "markdown", 287 | "metadata": { 288 | "id": "OdRJkiDAXwQk", 289 | "colab_type": "text" 290 | }, 291 | "source": [ 292 | "## Data generation parameters" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "metadata": { 298 | "id": "CM9ZeUMqOt8j", 299 | "colab_type": "code", 300 | "colab": {} 301 | }, 302 | "source": [ 303 | "data_mean = 4\n", 304 | "data_stddev = 1.25\n", 305 | "\n", 306 | "d_sampler = get_distribution_sampler(data_mean, data_stddev)\n", 307 | "gi_sampler = get_generator_input_sampler()" 308 | ], 309 | "execution_count": 0, 310 | "outputs": [] 311 | }, 312 | { 313 | "cell_type": "markdown", 314 | "metadata": { 315 | "id": "UIYpY99oXyiQ", 316 | "colab_type": "text" 317 | }, 318 | "source": [ 319 | "## Initialize the networks" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "metadata": { 325 | "id": "8LN_PKnbO4Ag", 326 | "colab_type": "code", 327 | "colab": {} 328 | }, 329 | "source": [ 330 | "G = Generator(input_size=(500, 1),\n", 331 | " hidden_size=g_hidden_size,\n", 332 | " output_size=g_output_size)\n", 333 | "\n", 334 | "D = Discriminator(input_size=(1,4),\n", 335 | " hidden_size=d_hidden_size,\n", 336 | " output_size=d_output_size)" 337 | ], 338 | "execution_count": 0, 339 | "outputs": [] 340 | }, 341 | { 342 | "cell_type": "markdown", 343 | "metadata": { 344 | "id": "FWvzGpJcX2AQ", 345 | "colab_type": "text" 346 | }, 347 | "source": [ 348 | "## Declare the loss and optimizers" 349 | ] 350 | }, 351 | { 352 | "cell_type": "code", 353 | "metadata": { 354 | "id": "Lkc9zVjCPlaW", 355 | "colab_type": "code", 356 | "colab": {} 357 | }, 358 | "source": [ 359 | "criterion = tf.keras.losses.BinaryCrossentropy(from_logits=True) \n", 360 | "d_optimizer = tf.keras.optimizers.SGD(learning_rate=d_learning_rate, momentum=sgd_momentum)\n", 361 | "g_optimizer = tf.keras.optimizers.SGD(learning_rate=g_learning_rate, momentum=sgd_momentum)" 362 | ], 363 | "execution_count": 0, 364 | "outputs": [] 365 | }, 366 | { 367 | "cell_type": "markdown", 368 | "metadata": { 369 | "id": "TeBmj8g9X8FV", 370 | "colab_type": "text" 371 | }, 372 | "source": [ 373 | "## One forward and backward pass with the Discriminator network with real data\n", 374 | "\n", 375 | "*We do not update the parameters with these gradients.*" 376 | ] 377 | }, 378 | { 379 | "cell_type": "code", 380 | "metadata": { 381 | "id": "2Kw8MU7Vgr1E", 382 | "colab_type": "code", 383 | "colab": {} 384 | }, 385 | "source": [ 386 | "# d_real_data = d_sampler(d_input_size)\n", 387 | "\n", 388 | "# with tf.GradientTape() as tape:\n", 389 | "# d_real_decision = D(get_moments(d_real_data).reshape((1,4)))\n", 390 | "# d_real_error = criterion(d_real_decision, np.ones((1,1))) # ones = true\n", 391 | "# d_real_grads = tape.gradient(d_real_error, D.trainable_weights) # compute/store gradients, but don't change params\n", 392 | "# d_real_grads[0].numpy()" 393 | ], 394 | "execution_count": 0, 395 | "outputs": [] 396 | }, 397 | { 398 | "cell_type": "markdown", 399 | "metadata": { 400 | "id": "81YBUAh6YGc2", 401 | "colab_type": "text" 402 | }, 403 | "source": [ 404 | "## One forward and backward pass with the Discriminator network with the fake data" 405 | ] 406 | }, 407 | { 408 | "cell_type": "code", 409 | "metadata": { 410 | "id": "IjcWC0gNibld", 411 | "colab_type": "code", 412 | "colab": {} 413 | }, 414 | "source": [ 415 | "# d_gen_input = gi_sampler(minibatch_size, g_input_size)\n", 416 | "# with tf.GradientTape() as tape:\n", 417 | "# with tape.stop_recording():\n", 418 | "# d_fake_data = G(d_gen_input)\n", 419 | "# d_fake_decision = D(get_moments(d_fake_data.numpy().T).reshape((1,4)))\n", 420 | "# d_fake_error = criterion(d_fake_decision, np.zeros((1,1)))\n", 421 | "# d_fake_grads = tape.gradient(d_fake_error, D.trainable_weights) \n", 422 | "# print(d_fake_grads[0].numpy())\n", 423 | "# d_optimizer.apply_gradients(zip(d_fake_grads, D.trainable_weights)) # Only optimizes D's parameters" 424 | ], 425 | "execution_count": 0, 426 | "outputs": [] 427 | }, 428 | { 429 | "cell_type": "markdown", 430 | "metadata": { 431 | "id": "dS7mg238Yf3M", 432 | "colab_type": "text" 433 | }, 434 | "source": [ 435 | "## One forward and backward pass with the Generator network" 436 | ] 437 | }, 438 | { 439 | "cell_type": "code", 440 | "metadata": { 441 | "id": "yp3eWwowQsfA", 442 | "colab_type": "code", 443 | "colab": {} 444 | }, 445 | "source": [ 446 | "# gen_input = gi_sampler(minibatch_size, g_input_size)\n", 447 | "# with tf.GradientTape() as tape:\n", 448 | "# g_fake_data = G(gen_input)\n", 449 | "# dg_fake_decision = D(tf.reshape(get_moments_tf(g_fake_data), (1, 4)))\n", 450 | "# g_error = criterion(dg_fake_decision, np.ones((1,1)))\n", 451 | "# g_grads = tape.gradient(g_error, G.trainable_weights)\n", 452 | "# g_optimizer.apply_gradients(zip(g_grads, G.trainable_weights))" 453 | ], 454 | "execution_count": 0, 455 | "outputs": [] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "metadata": { 460 | "id": "as0XOQuhQVai", 461 | "colab_type": "code", 462 | "colab": { 463 | "base_uri": "https://localhost:8080/", 464 | "height": 496 465 | }, 466 | "outputId": "fc0c169d-9b00-407d-e97d-de07469d4147" 467 | }, 468 | "source": [ 469 | "for epoch in range(num_epochs):\n", 470 | " for d_index in range(d_steps):\n", 471 | " # 1A: Train D on real\n", 472 | " d_real_data = tf.convert_to_tensor(d_sampler(d_input_size))\n", 473 | " with tf.GradientTape() as tape:\n", 474 | " d_real_decision = D(tf.reshape(get_moments(d_real_data), (1,4)))\n", 475 | " d_real_error = criterion(d_real_decision, tf.convert_to_tensor(np.ones((1,1)))) # ones = true\n", 476 | " d_real_grads = tape.gradient(d_real_error, D.trainable_weights) # compute/store gradients, but don't change params\n", 477 | " \n", 478 | " # 1B: Train D on fake\n", 479 | " d_gen_input = tf.convert_to_tensor(gi_sampler(minibatch_size, g_input_size))\n", 480 | " with tf.GradientTape() as tape:\n", 481 | " with tape.stop_recording():\n", 482 | " d_fake_data = G(d_gen_input)\n", 483 | " d_fake_decision = D(tf.reshape(get_moments(d_fake_data), (1, 4)))\n", 484 | " d_fake_error = criterion(d_fake_decision, tf.convert_to_tensor(np.zeros((1,1))))\n", 485 | " d_fake_grads = tape.gradient(d_fake_error, D.trainable_weights) \n", 486 | " d_optimizer.apply_gradients(zip(d_fake_grads, D.trainable_weights)) # Only optimizes D's parameters\n", 487 | "\n", 488 | " dre, dfe = d_real_error.numpy(), d_fake_error.numpy()\n", 489 | "\n", 490 | " for g_index in range(g_steps):\n", 491 | " # 2. Train G on D's response (but DO NOT train D on these labels)\n", 492 | " gen_input = tf.convert_to_tensor(gi_sampler(minibatch_size, g_input_size))\n", 493 | " with tf.GradientTape() as tape:\n", 494 | " g_fake_data = G(gen_input)\n", 495 | " dg_fake_decision = D(tf.reshape(get_moments(g_fake_data), (1, 4)))\n", 496 | " g_error = criterion(dg_fake_decision, tf.convert_to_tensor(np.ones((1,1))))\n", 497 | " g_grads = tape.gradient(g_error, G.trainable_weights)\n", 498 | " g_optimizer.apply_gradients(zip(g_grads, G.trainable_weights))\n", 499 | "\n", 500 | " ge = g_error.numpy()\n", 501 | "\n", 502 | " if epoch % print_interval == 0:\n", 503 | " print(\"Epoch %s: D (%s real_err, %s fake_err) G (%s err); Real Dist (%s), Fake Dist (%s) \" %\n", 504 | " (epoch, dre, dfe, ge, stats(d_real_data.numpy()), stats(d_fake_data.numpy())))" 505 | ], 506 | "execution_count": 0, 507 | "outputs": [ 508 | { 509 | "output_type": "stream", 510 | "text": [ 511 | "WARNING: Logging before flag parsing goes to stderr.\n", 512 | "W0825 16:46:38.575693 140618269017984 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_impl.py:182: add_dispatch_support..wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n", 513 | "Instructions for updating:\n", 514 | "Use tf.where in 2.0, which has the same broadcast rule as np.where\n" 515 | ], 516 | "name": "stderr" 517 | }, 518 | { 519 | "output_type": "stream", 520 | "text": [ 521 | "Epoch 0: D (1.0886667966842651 real_err, 0.6931471824645996 fake_err) G (1.05702543258667 err); Real Dist ([3.969248032721655, 1.2093934306513305]), Fake Dist ([-0.16539627014683225, 0.07419386361400125]) \n", 522 | "Epoch 100: D (1.087760329246521 real_err, 0.6931471824645996 fake_err) G (1.0470203161239624 err); Real Dist ([4.066130571911628, 1.2346445557152335]), Fake Dist ([-0.30376876966542266, 0.11773658791402217]) \n", 523 | "Epoch 200: D (1.0887233018875122 real_err, 0.6931471824645996 fake_err) G (1.0439269542694092 err); Real Dist ([3.932937238259309, 1.2277380207905655]), Fake Dist ([-0.45581279916255424, 0.05699663380595091]) \n", 524 | "Epoch 300: D (1.089498519897461 real_err, 0.6931471824645996 fake_err) G (1.0407992601394653 err); Real Dist ([3.9936716043970866, 1.26235733971329]), Fake Dist ([-0.7447241589696177, 0.008942774214919355]) \n", 525 | "Epoch 400: D (1.0871673822402954 real_err, 0.6931471824645996 fake_err) G (1.0361649990081787 err); Real Dist ([3.984654450017385, 1.2337837894636985]), Fake Dist ([-1.0351879749603823, 0.002109290445755961]) \n", 526 | "Epoch 500: D (1.0883467197418213 real_err, 0.6931471824645996 fake_err) G (1.0363883972167969 err); Real Dist ([3.988741519040915, 1.2769630954792424]), Fake Dist ([-1.3187153942607712, 0.01051046466078226]) \n", 527 | "Epoch 600: D (1.0887354612350464 real_err, 0.6931471824645996 fake_err) G (1.0328069925308228 err); Real Dist ([4.0354432685602495, 1.2639294650160453]), Fake Dist ([-1.5836209810851478, 0.0016828689300378383]) \n", 528 | "Epoch 700: D (1.086708903312683 real_err, 0.6931471824645996 fake_err) G (1.0311508178710938 err); Real Dist ([4.031299864056426, 1.2657557941081317]), Fake Dist ([-1.8416030141383084, 0.0013018275670126536]) \n", 529 | "Epoch 800: D (1.0865145921707153 real_err, 0.6931471824645996 fake_err) G (1.0310169458389282 err); Real Dist ([3.932973583748997, 1.2239113147645544]), Fake Dist ([-2.0886074957547605, 0.0013422930905573522]) \n", 530 | "Epoch 900: D (1.0907894372940063 real_err, 0.6931471824645996 fake_err) G (1.0300602912902832 err); Real Dist ([3.9778720529350466, 1.2423837728686176]), Fake Dist ([-2.3256808724775326, 0.0012080414429308768]) \n", 531 | "Epoch 1000: D (1.0880186557769775 real_err, 0.6931471824645996 fake_err) G (1.028603434562683 err); Real Dist ([3.9940469689561717, 1.2715260343372974]), Fake Dist ([-2.551267113414703, 0.0011714463221389963]) \n", 532 | "Epoch 1100: D (1.0890711545944214 real_err, 0.6931471824645996 fake_err) G (1.0281318426132202 err); Real Dist ([3.9846537807237072, 1.2359541446670275]), Fake Dist ([-2.7675533651296775, 0.0012208285143596883]) \n", 533 | "Epoch 1200: D (1.08739173412323 real_err, 0.6931471824645996 fake_err) G (1.0269633531570435 err); Real Dist ([3.9974220963831475, 1.1644718718455738]), Fake Dist ([-2.9766773087855567, 0.0011700479420714103]) \n", 534 | "Epoch 1300: D (1.0886634588241577 real_err, 0.6931471824645996 fake_err) G (1.0266574621200562 err); Real Dist ([4.083303671966571, 1.2616810053521563]), Fake Dist ([-3.1772018743392696, 0.0014295534361247573]) \n", 535 | "Epoch 1400: D (1.08675217628479 real_err, 0.6931471824645996 fake_err) G (1.0262341499328613 err); Real Dist ([3.9615456319438453, 1.2818510111661996]), Fake Dist ([-3.369814808413284, 0.0013407713700283883]) \n", 536 | "Epoch 1500: D (1.087263584136963 real_err, 0.6931471824645996 fake_err) G (1.025339961051941 err); Real Dist ([3.904134046604551, 1.1628845651645383]), Fake Dist ([-3.5559914304462503, 0.001369007176916181]) \n", 537 | "Epoch 1600: D (1.0901052951812744 real_err, 0.6931471824645996 fake_err) G (1.025075912475586 err); Real Dist ([4.059169023803737, 1.2646959132035673]), Fake Dist ([-3.7347559002639072, 0.0014854473813596168]) \n", 538 | "Epoch 1700: D (1.0903997421264648 real_err, 0.6931471824645996 fake_err) G (1.0246912240982056 err); Real Dist ([4.073631580767418, 1.307118740402538]), Fake Dist ([-3.9077235078747, 0.00145042718194546]) \n", 539 | "Epoch 1800: D (1.0890417098999023 real_err, 0.6931471824645996 fake_err) G (1.0242844820022583 err); Real Dist ([4.013299261717157, 1.2009866954408044]), Fake Dist ([-4.074154845765458, 0.0014768626935643397]) \n", 540 | "Epoch 1900: D (1.0875056982040405 real_err, 0.6931471824645996 fake_err) G (1.0239284038543701 err); Real Dist ([3.932749537333192, 1.3076516555178603]), Fake Dist ([-4.234803080252499, 0.0016178279077750752]) \n", 541 | "Epoch 2000: D (1.0874297618865967 real_err, 0.6931471824645996 fake_err) G (1.0234853029251099 err); Real Dist ([3.9801396846719146, 1.2030596669445992]), Fake Dist ([-4.3905574960095075, 0.0014821050846723775]) \n", 542 | "Epoch 2100: D (1.0854185819625854 real_err, 0.6931471824645996 fake_err) G (1.023307204246521 err); Real Dist ([3.9527717265787685, 1.2955137559221834]), Fake Dist ([-4.5413157765176875, 0.0013579439905201547]) \n", 543 | "Epoch 2200: D (1.0870475769042969 real_err, 0.6931471824645996 fake_err) G (1.022774577140808 err); Real Dist ([4.032350958520261, 1.2474040559753414]), Fake Dist ([-4.6874410387605545, 0.00129060017867163]) \n" 544 | ], 545 | "name": "stdout" 546 | } 547 | ] 548 | }, 549 | { 550 | "cell_type": "markdown", 551 | "metadata": { 552 | "id": "H9ggO7OcwgMV", 553 | "colab_type": "text" 554 | }, 555 | "source": [ 556 | "The network is still not properly configured. " 557 | ] 558 | }, 559 | { 560 | "cell_type": "code", 561 | "metadata": { 562 | "id": "zB1GPjYvT8SY", 563 | "colab_type": "code", 564 | "colab": {} 565 | }, 566 | "source": [ 567 | "" 568 | ], 569 | "execution_count": 0, 570 | "outputs": [] 571 | } 572 | ] 573 | } -------------------------------------------------------------------------------- /Hyperparameter Sweeps with TF 2.0 & W&B/Custom_TF_Training_Loops_WnB.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Custom_TF_Training_Loops_WnB.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [] 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "codemirror_mode": { 17 | "name": "ipython", 18 | "version": 3 19 | }, 20 | "file_extension": ".py", 21 | "mimetype": "text/x-python", 22 | "name": "python", 23 | "nbconvert_exporter": "python", 24 | "pygments_lexer": "ipython3", 25 | "version": "3.7.1" 26 | } 27 | }, 28 | "cells": [ 29 | { 30 | "cell_type": "code", 31 | "metadata": { 32 | "colab_type": "code", 33 | "id": "11OpHzjyDV_R", 34 | "outputId": "5592067b-1d45-401f-fcf7-de2f2845f61b", 35 | "colab": { 36 | "base_uri": "https://localhost:8080/", 37 | "height": 34 38 | } 39 | }, 40 | "source": [ 41 | "# Select the TensorFlow 2.0 runtime\n", 42 | "%tensorflow_version 2.x" 43 | ], 44 | "execution_count": 1, 45 | "outputs": [ 46 | { 47 | "output_type": "stream", 48 | "text": [ 49 | "TensorFlow 2.x selected.\n" 50 | ], 51 | "name": "stdout" 52 | } 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "metadata": { 58 | "colab_type": "code", 59 | "id": "ARaFUj2ZDsoO", 60 | "colab": {} 61 | }, 62 | "source": [ 63 | "# Install Weights and Biases (WnB)\n", 64 | "!pip install wandb" 65 | ], 66 | "execution_count": 0, 67 | "outputs": [] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "metadata": { 72 | "colab_type": "code", 73 | "id": "pfi5G8ZWDeox", 74 | "colab": {} 75 | }, 76 | "source": [ 77 | "# Primary imports\n", 78 | "import tensorflow as tf\n", 79 | "import numpy as np\n", 80 | "import wandb" 81 | ], 82 | "execution_count": 0, 83 | "outputs": [] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "metadata": { 88 | "colab_type": "code", 89 | "id": "UUK3cHkqDovW", 90 | "colab": {} 91 | }, 92 | "source": [ 93 | "# Authorize Weights and Biases\n", 94 | "!wandb login" 95 | ], 96 | "execution_count": 0, 97 | "outputs": [] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "metadata": { 102 | "colab_type": "code", 103 | "id": "1RqRfkKJDzXp", 104 | "outputId": "8b17678f-f36e-4388-f1c1-8b2d2373630a", 105 | "colab": { 106 | "base_uri": "https://localhost:8080/", 107 | "height": 85 108 | } 109 | }, 110 | "source": [ 111 | "# Intialize WnB with a project name of your choice\n", 112 | "wandb.init(project=\"custom_training_loops_tf\")" 113 | ], 114 | "execution_count": 5, 115 | "outputs": [ 116 | { 117 | "output_type": "display_data", 118 | "data": { 119 | "text/html": [ 120 | "\n", 121 | " Logging results to Weights & Biases (Documentation).
\n", 122 | " Project page: https://app.wandb.ai/sayakpaul/custom_training_loops_tf
\n", 123 | " Run page: https://app.wandb.ai/sayakpaul/custom_training_loops_tf/runs/3c1cfpm9
\n", 124 | " " 125 | ], 126 | "text/plain": [ 127 | "" 128 | ] 129 | }, 130 | "metadata": { 131 | "tags": [] 132 | } 133 | }, 134 | { 135 | "output_type": "execute_result", 136 | "data": { 137 | "text/plain": [ 138 | "W&B Run: https://app.wandb.ai/sayakpaul/custom_training_loops_tf/runs/3c1cfpm9" 139 | ] 140 | }, 141 | "metadata": { 142 | "tags": [] 143 | }, 144 | "execution_count": 5 145 | } 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "metadata": { 151 | "colab_type": "code", 152 | "id": "-TkzuZXSD_MN", 153 | "outputId": "13417acd-fe1e-40e9-db0f-a252c8f17086", 154 | "colab": { 155 | "base_uri": "https://localhost:8080/", 156 | "height": 170 157 | } 158 | }, 159 | "source": [ 160 | "# Load the FashionMNIST dataset, scale the pixel values\n", 161 | "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n", 162 | "X_train = X_train/255.\n", 163 | "X_test = X_test/255.\n", 164 | "\n", 165 | "X_train.shape, X_test.shape, y_train.shape, y_test.shape" 166 | ], 167 | "execution_count": 6, 168 | "outputs": [ 169 | { 170 | "output_type": "stream", 171 | "text": [ 172 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n", 173 | "32768/29515 [=================================] - 0s 0us/step\n", 174 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n", 175 | "26427392/26421880 [==============================] - 0s 0us/step\n", 176 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n", 177 | "8192/5148 [===============================================] - 0s 0us/step\n", 178 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n", 179 | "4423680/4422102 [==============================] - 0s 0us/step\n" 180 | ], 181 | "name": "stdout" 182 | }, 183 | { 184 | "output_type": "execute_result", 185 | "data": { 186 | "text/plain": [ 187 | "((60000, 28, 28), (10000, 28, 28), (60000,), (10000,))" 188 | ] 189 | }, 190 | "metadata": { 191 | "tags": [] 192 | }, 193 | "execution_count": 6 194 | } 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "metadata": { 200 | "colab_type": "code", 201 | "id": "uNfUBY4mErkY", 202 | "colab": {} 203 | }, 204 | "source": [ 205 | "# Define the labels of the dataset\n", 206 | "CLASSES=[\"T-shirt/top\",\"Trouser\",\"Pullover\",\"Dress\",\"Coat\",\n", 207 | " \"Sandal\",\"Shirt\",\"Sneaker\",\"Bag\",\"Ankle boot\"]" 208 | ], 209 | "execution_count": 0, 210 | "outputs": [] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "metadata": { 215 | "colab_type": "code", 216 | "id": "ofiUxCQpEu0p", 217 | "colab": {} 218 | }, 219 | "source": [ 220 | "# Reshape input data\n", 221 | "X_train = X_train.astype(\"float32\").reshape(-1, 28, 28, 1)\n", 222 | "X_test = X_test.astype(\"float32\").reshape(-1, 28, 28, 1)" 223 | ], 224 | "execution_count": 0, 225 | "outputs": [] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "metadata": { 230 | "colab_type": "code", 231 | "id": "vWsjTjvwLVvL", 232 | "outputId": "da3856d3-e93b-4c5f-92b1-19df42d2898e", 233 | "colab": { 234 | "base_uri": "https://localhost:8080/", 235 | "height": 34 236 | } 237 | }, 238 | "source": [ 239 | "y_train.shape, y_test.shape" 240 | ], 241 | "execution_count": 9, 242 | "outputs": [ 243 | { 244 | "output_type": "execute_result", 245 | "data": { 246 | "text/plain": [ 247 | "((60000,), (10000,))" 248 | ] 249 | }, 250 | "metadata": { 251 | "tags": [] 252 | }, 253 | "execution_count": 9 254 | } 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "metadata": { 260 | "colab_type": "code", 261 | "id": "jA2ES9N7FOo4", 262 | "colab": {} 263 | }, 264 | "source": [ 265 | "# TensorFlow imports\n", 266 | "from tensorflow.keras.models import *\n", 267 | "from tensorflow.keras.layers import *" 268 | ], 269 | "execution_count": 0, 270 | "outputs": [] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "metadata": { 275 | "colab_type": "code", 276 | "id": "UMXRRKcbE9vE", 277 | "colab": {} 278 | }, 279 | "source": [ 280 | "# Define utility function for building a basic shallow Convnet \n", 281 | "def get_training_model():\n", 282 | " model = Sequential()\n", 283 | " model.add(Conv2D(16, (5, 5), activation=\"relu\",\n", 284 | " input_shape=(28, 28,1)))\n", 285 | " model.add(MaxPooling2D(pool_size=(2, 2)))\n", 286 | " model.add(Conv2D(32, (5, 5), activation=\"relu\"))\n", 287 | " model.add(MaxPooling2D(pool_size=(2, 2)))\n", 288 | " model.add(Dropout(0.2))\n", 289 | " model.add(Flatten())\n", 290 | " model.add(Dense(128, activation=\"relu\"))\n", 291 | " model.add(Dense(len(CLASSES), activation=\"softmax\"))\n", 292 | " \n", 293 | " return model" 294 | ], 295 | "execution_count": 0, 296 | "outputs": [] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "metadata": { 301 | "colab_type": "code", 302 | "id": "DDXOr6FhFvXw", 303 | "colab": {} 304 | }, 305 | "source": [ 306 | "# Define loass function and optimizer\n", 307 | "loss_func = tf.keras.losses.SparseCategoricalCrossentropy()\n", 308 | "optimizer = tf.keras.optimizers.Adam()" 309 | ], 310 | "execution_count": 0, 311 | "outputs": [] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "metadata": { 316 | "colab_type": "code", 317 | "id": "uXfw22IZF5Rw", 318 | "colab": {} 319 | }, 320 | "source": [ 321 | "# Average the loss across the batch size within an epoch\n", 322 | "train_loss = tf.keras.metrics.Mean(name=\"train_loss\")\n", 323 | "valid_loss = tf.keras.metrics.Mean(name=\"test_loss\")\n", 324 | "\n", 325 | "# Specify the performance metric\n", 326 | "train_acc = tf.keras.metrics.SparseCategoricalAccuracy(name=\"train_acc\")\n", 327 | "valid_acc = tf.keras.metrics.SparseCategoricalAccuracy(name=\"valid_acc\")" 328 | ], 329 | "execution_count": 0, 330 | "outputs": [] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "metadata": { 335 | "colab_type": "code", 336 | "id": "w_CcwaadLI8e", 337 | "colab": {} 338 | }, 339 | "source": [ 340 | "# Batches of 64\n", 341 | "train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).batch(64)\n", 342 | "test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(64)" 343 | ], 344 | "execution_count": 0, 345 | "outputs": [] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "metadata": { 350 | "colab_type": "code", 351 | "id": "-JgN5TWzLroE", 352 | "colab": {} 353 | }, 354 | "source": [ 355 | "# Train the model\n", 356 | "@tf.function\n", 357 | "def model_train(features, labels):\n", 358 | " # Define the GradientTape context\n", 359 | " with tf.GradientTape() as tape:\n", 360 | " # Get the probabilities\n", 361 | " predictions = model(features)\n", 362 | " # Calculate the loss\n", 363 | " loss = loss_func(labels, predictions)\n", 364 | " # Get the gradients\n", 365 | " gradients = tape.gradient(loss, model.trainable_variables)\n", 366 | " # Update the weights\n", 367 | " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", 368 | "\n", 369 | " # Update the loss and accuracy\n", 370 | " train_loss(loss)\n", 371 | " train_acc(labels, predictions)" 372 | ], 373 | "execution_count": 0, 374 | "outputs": [] 375 | }, 376 | { 377 | "cell_type": "code", 378 | "metadata": { 379 | "colab_type": "code", 380 | "id": "LBPBn762LwZ1", 381 | "colab": {} 382 | }, 383 | "source": [ 384 | "# Validating the model\n", 385 | "@tf.function\n", 386 | "def model_validate(features, labels):\n", 387 | " predictions = model(features)\n", 388 | " v_loss = loss_func(labels, predictions)\n", 389 | "\n", 390 | " valid_loss(v_loss)\n", 391 | " valid_acc(labels, predictions)" 392 | ], 393 | "execution_count": 0, 394 | "outputs": [] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "metadata": { 399 | "colab_type": "code", 400 | "id": "jleWU_WoPXfN", 401 | "colab": {} 402 | }, 403 | "source": [ 404 | "# A shallow Convnet\n", 405 | "model = get_training_model()" 406 | ], 407 | "execution_count": 0, 408 | "outputs": [] 409 | }, 410 | { 411 | "cell_type": "code", 412 | "metadata": { 413 | "colab_type": "code", 414 | "id": "946iV6qSXZXo", 415 | "colab": {} 416 | }, 417 | "source": [ 418 | "# Grab random images from the test and make predictions using \n", 419 | "# the model *while it is training* and log them using WnB\n", 420 | "def get_sample_predictions():\n", 421 | " predictions = []\n", 422 | " images = []\n", 423 | " random_indices = np.random.choice(X_test.shape[0], 25)\n", 424 | " for index in random_indices:\n", 425 | " image = X_test[index].reshape(1, 28, 28, 1)\n", 426 | " prediction = np.argmax(model(image).numpy(), axis=1)\n", 427 | " prediction = CLASSES[int(prediction)]\n", 428 | " \n", 429 | " images.append(image)\n", 430 | " predictions.append(prediction)\n", 431 | " \n", 432 | " wandb.log({\"predictions\": [wandb.Image(image, caption=prediction) \n", 433 | " for (image, prediction) in zip(images, predictions)]})" 434 | ], 435 | "execution_count": 0, 436 | "outputs": [] 437 | }, 438 | { 439 | "cell_type": "code", 440 | "metadata": { 441 | "colab_type": "code", 442 | "id": "yZE9_IhjL4lM", 443 | "outputId": "8b4f3f61-5a85-4128-bf33-ea7753b4ab19", 444 | "scrolled": false, 445 | "colab": { 446 | "base_uri": "https://localhost:8080/", 447 | "height": 102 448 | } 449 | }, 450 | "source": [ 451 | "# Train the model for 5 epochs\n", 452 | "for epoch in range(5):\n", 453 | " # Run the model through train and test sets respectively\n", 454 | " for (features, labels) in train_ds:\n", 455 | " model_train(features, labels)\n", 456 | "\n", 457 | " for test_features, test_labels in test_ds:\n", 458 | " model_validate(test_features, test_labels)\n", 459 | " \n", 460 | " # Grab the results\n", 461 | " (loss, acc) = train_loss.result(), train_acc.result()\n", 462 | " (val_loss, val_acc) = valid_loss.result(), valid_acc.result()\n", 463 | " \n", 464 | " # Clear the current state of the metrics\n", 465 | " train_loss.reset_states(), train_acc.reset_states()\n", 466 | " valid_loss.reset_states(), valid_acc.reset_states()\n", 467 | " \n", 468 | " # Local logging\n", 469 | " template = \"Epoch {:.3f}, loss: {:.3f}, acc: {:.3f}, val_loss: {:.3f}, val_acc: {:.3f}\"\n", 470 | " print (template.format(epoch+1,\n", 471 | " loss,\n", 472 | " acc,\n", 473 | " val_loss,\n", 474 | " val_acc))\n", 475 | " \n", 476 | " # Logging with WnB\n", 477 | " wandb.log({\"train_loss\": loss.numpy(),\n", 478 | " \"train_accuracy\": acc.numpy(),\n", 479 | " \"val_loss\": val_loss.numpy(),\n", 480 | " \"val_accuracy\": val_acc.numpy()\n", 481 | " })\n", 482 | " get_sample_predictions()" 483 | ], 484 | "execution_count": 19, 485 | "outputs": [ 486 | { 487 | "output_type": "stream", 488 | "text": [ 489 | "Epoch 1.000, loss: 0.544, acc: 0.802, val_loss: 0.429, val_acc: 0.845\n", 490 | "Epoch 2.000, loss: 0.361, acc: 0.871, val_loss: 0.377, val_acc: 0.860\n", 491 | "Epoch 3.000, loss: 0.309, acc: 0.888, val_loss: 0.351, val_acc: 0.869\n", 492 | "Epoch 4.000, loss: 0.277, acc: 0.899, val_loss: 0.336, val_acc: 0.873\n", 493 | "Epoch 5.000, loss: 0.252, acc: 0.908, val_loss: 0.323, val_acc: 0.882\n" 494 | ], 495 | "name": "stdout" 496 | } 497 | ] 498 | } 499 | ] 500 | } -------------------------------------------------------------------------------- /Hyperparameter Sweeps with TF 2.0 & W&B/TensorBoard Integration in W&B.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "kernelspec": { 6 | "name": "python3", 7 | "display_name": "Python 3" 8 | }, 9 | "language_info": { 10 | "codemirror_mode": { 11 | "name": "ipython", 12 | "version": 3 13 | }, 14 | "file_extension": ".py", 15 | "mimetype": "text/x-python", 16 | "name": "python", 17 | "nbconvert_exporter": "python", 18 | "pygments_lexer": "ipython3", 19 | "version": "3.5.3" 20 | }, 21 | "colab": { 22 | "name": "Demo_tensorboard.ipynb", 23 | "provenance": [], 24 | "collapsed_sections": [] 25 | }, 26 | "accelerator": "GPU" 27 | }, 28 | "cells": [ 29 | { 30 | "cell_type": "code", 31 | "metadata": { 32 | "id": "Z5ZHrNOX6F8g", 33 | "colab_type": "code", 34 | "outputId": "3790fe51-2ce7-4473-fbfc-1493a87a3e83", 35 | "colab": { 36 | "base_uri": "https://localhost:8080/", 37 | "height": 34 38 | } 39 | }, 40 | "source": [ 41 | "# Select TensorFlow 2.0\n", 42 | "%tensorflow_version 2.x" 43 | ], 44 | "execution_count": 0, 45 | "outputs": [ 46 | { 47 | "output_type": "stream", 48 | "text": [ 49 | "TensorFlow 2.x selected.\n" 50 | ], 51 | "name": "stdout" 52 | } 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "metadata": { 58 | "id": "JiQDKKao6KoO", 59 | "colab_type": "code", 60 | "colab": {} 61 | }, 62 | "source": [ 63 | "# Install Weights and Biases\n", 64 | "!pip install wandb" 65 | ], 66 | "execution_count": 0, 67 | "outputs": [] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "metadata": { 72 | "id": "2xKhB06-6DSB", 73 | "colab_type": "code", 74 | "colab": {} 75 | }, 76 | "source": [ 77 | "# Imports\n", 78 | "from tensorflow.keras.datasets import fashion_mnist\n", 79 | "from tensorflow.keras.models import Sequential\n", 80 | "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dropout, Dense, Flatten\n", 81 | "from tensorflow.keras import utils\n", 82 | "from tensorflow.keras.optimizers import SGD\n", 83 | "from tensorflow.keras.callbacks import TensorBoard\n", 84 | "\n", 85 | "import wandb\n", 86 | "from wandb.keras import WandbCallback" 87 | ], 88 | "execution_count": 0, 89 | "outputs": [] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "metadata": { 94 | "id": "WrOqgXc16Pki", 95 | "colab_type": "code", 96 | "colab": {} 97 | }, 98 | "source": [ 99 | "# Login to your W&B account\n", 100 | "!wandb login" 101 | ], 102 | "execution_count": 0, 103 | "outputs": [] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "metadata": { 108 | "id": "rDUf-OBO6DSG", 109 | "colab_type": "code", 110 | "colab": {} 111 | }, 112 | "source": [ 113 | "# Initialize your W&B project allowing it to sync with TensorBoard\n", 114 | "wandb.init(project=\"tensorboard-integration\", sync_tensorboard=True)\n", 115 | "config = wandb.config" 116 | ], 117 | "execution_count": 0, 118 | "outputs": [] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "metadata": { 123 | "id": "6uBarjh76DSL", 124 | "colab_type": "code", 125 | "colab": {} 126 | }, 127 | "source": [ 128 | "# Specify the configuration variables\n", 129 | "config.dropout = 0.2\n", 130 | "config.hidden_layer_size = 128\n", 131 | "config.layer_1_size = 16\n", 132 | "config.layer_2_size = 32\n", 133 | "config.learn_rate = 0.01\n", 134 | "config.decay = 1e-6\n", 135 | "config.momentum = 0.9\n", 136 | "config.epochs = 25" 137 | ], 138 | "execution_count": 0, 139 | "outputs": [] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "metadata": { 144 | "id": "9dFCHnAO6DSO", 145 | "colab_type": "code", 146 | "outputId": "c9f4e8f3-98d5-4d05-bccc-6785ed87e11c", 147 | "colab": { 148 | "base_uri": "https://localhost:8080/", 149 | "height": 153 150 | } 151 | }, 152 | "source": [ 153 | "# Load and preprocess the data\n", 154 | "(X_train, y_train), (X_test, y_test) = fashion_mnist.load_data()\n", 155 | "labels=[\"T-shirt/top\",\"Trouser\",\"Pullover\",\"Dress\",\"Coat\",\n", 156 | " \"Sandal\",\"Shirt\",\"Sneaker\",\"Bag\",\"Ankle boot\"]\n", 157 | "\n", 158 | "img_width=28\n", 159 | "img_height=28\n", 160 | "\n", 161 | "X_train = X_train.astype('float32')\n", 162 | "X_train /= 255.\n", 163 | "X_test = X_test.astype('float32')\n", 164 | "X_test /= 255.\n", 165 | "\n", 166 | "# Reshape input data\n", 167 | "X_train = X_train.reshape(X_train.shape[0], img_width, img_height, 1)\n", 168 | "X_test = X_test.reshape(X_test.shape[0], img_width, img_height, 1)\n", 169 | "\n", 170 | "# One hot encode outputs\n", 171 | "y_train = utils.to_categorical(y_train)\n", 172 | "y_test = utils.to_categorical(y_test)\n", 173 | "num_classes = y_test.shape[1]" 174 | ], 175 | "execution_count": 0, 176 | "outputs": [ 177 | { 178 | "output_type": "stream", 179 | "text": [ 180 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n", 181 | "32768/29515 [=================================] - 0s 0us/step\n", 182 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n", 183 | "26427392/26421880 [==============================] - 0s 0us/step\n", 184 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n", 185 | "8192/5148 [===============================================] - 0s 0us/step\n", 186 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n", 187 | "4423680/4422102 [==============================] - 0s 0us/step\n" 188 | ], 189 | "name": "stdout" 190 | } 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "metadata": { 196 | "id": "0s5c6sIW6DSQ", 197 | "colab_type": "code", 198 | "outputId": "db0346e9-3bea-4c96-ebcf-34bb7f88f4db", 199 | "colab": { 200 | "base_uri": "https://localhost:8080/", 201 | "height": 901 202 | } 203 | }, 204 | "source": [ 205 | "# Train the model\n", 206 | "sgd = SGD(lr=config.learn_rate, decay=config.decay, momentum=config.momentum,\n", 207 | " nesterov=True)\n", 208 | "\n", 209 | "model = Sequential()\n", 210 | "model.add(Conv2D(config.layer_1_size, (5, 5), activation='relu',\n", 211 | " input_shape=(img_width, img_height,1)))\n", 212 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 213 | "model.add(Conv2D(config.layer_2_size, (5, 5), activation='relu'))\n", 214 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 215 | "model.add(Dropout(config.dropout))\n", 216 | "model.add(Flatten())\n", 217 | "model.add(Dense(config.hidden_layer_size, activation='relu'))\n", 218 | "model.add(Dense(num_classes, activation='softmax'))\n", 219 | "\n", 220 | "model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])\n", 221 | "\n", 222 | "# The WandbCallback logs metrics and some examples of the test data\n", 223 | "model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=config.epochs,\n", 224 | " callbacks=[WandbCallback(data_type=\"image\", labels=labels), \n", 225 | " TensorBoard(log_dir=wandb.run.dir)])" 226 | ], 227 | "execution_count": 0, 228 | "outputs": [ 229 | { 230 | "output_type": "stream", 231 | "text": [ 232 | "Train on 60000 samples, validate on 10000 samples\n", 233 | "Epoch 1/25\n", 234 | "60000/60000 [==============================] - 40s 664us/sample - loss: 0.5759 - accuracy: 0.7878 - val_loss: 0.3768 - val_accuracy: 0.8607\n", 235 | "Epoch 2/25\n", 236 | "60000/60000 [==============================] - 37s 615us/sample - loss: 0.3774 - accuracy: 0.8621 - val_loss: 0.3495 - val_accuracy: 0.8723\n", 237 | "Epoch 3/25\n", 238 | "60000/60000 [==============================] - 37s 612us/sample - loss: 0.3355 - accuracy: 0.8769 - val_loss: 0.3151 - val_accuracy: 0.8856\n", 239 | "Epoch 4/25\n", 240 | "60000/60000 [==============================] - 35s 585us/sample - loss: 0.3108 - accuracy: 0.8855 - val_loss: 0.2991 - val_accuracy: 0.8926\n", 241 | "Epoch 5/25\n", 242 | "60000/60000 [==============================] - 35s 579us/sample - loss: 0.2919 - accuracy: 0.8911 - val_loss: 0.3040 - val_accuracy: 0.8926\n", 243 | "Epoch 6/25\n", 244 | "60000/60000 [==============================] - 35s 579us/sample - loss: 0.2787 - accuracy: 0.8956 - val_loss: 0.2912 - val_accuracy: 0.8924\n", 245 | "Epoch 7/25\n", 246 | "60000/60000 [==============================] - 34s 569us/sample - loss: 0.2657 - accuracy: 0.9009 - val_loss: 0.2867 - val_accuracy: 0.8960\n", 247 | "Epoch 8/25\n", 248 | "60000/60000 [==============================] - 34s 560us/sample - loss: 0.2570 - accuracy: 0.9051 - val_loss: 0.2841 - val_accuracy: 0.8952\n", 249 | "Epoch 9/25\n", 250 | "60000/60000 [==============================] - 33s 555us/sample - loss: 0.2480 - accuracy: 0.9062 - val_loss: 0.2764 - val_accuracy: 0.8987\n", 251 | "Epoch 10/25\n", 252 | "60000/60000 [==============================] - 33s 558us/sample - loss: 0.2400 - accuracy: 0.9100 - val_loss: 0.2718 - val_accuracy: 0.9009\n", 253 | "Epoch 11/25\n", 254 | "60000/60000 [==============================] - 35s 586us/sample - loss: 0.2309 - accuracy: 0.9129 - val_loss: 0.2867 - val_accuracy: 0.8951\n", 255 | "Epoch 12/25\n", 256 | "60000/60000 [==============================] - 36s 593us/sample - loss: 0.2236 - accuracy: 0.9154 - val_loss: 0.2721 - val_accuracy: 0.9005\n", 257 | "Epoch 13/25\n", 258 | "60000/60000 [==============================] - 35s 584us/sample - loss: 0.2202 - accuracy: 0.9177 - val_loss: 0.2767 - val_accuracy: 0.9006\n", 259 | "Epoch 14/25\n", 260 | "60000/60000 [==============================] - 35s 591us/sample - loss: 0.2132 - accuracy: 0.9186 - val_loss: 0.2727 - val_accuracy: 0.9009\n", 261 | "Epoch 15/25\n", 262 | "60000/60000 [==============================] - 36s 599us/sample - loss: 0.2081 - accuracy: 0.9218 - val_loss: 0.2683 - val_accuracy: 0.9001\n", 263 | "Epoch 16/25\n", 264 | "60000/60000 [==============================] - 35s 583us/sample - loss: 0.2005 - accuracy: 0.9252 - val_loss: 0.2738 - val_accuracy: 0.9077\n", 265 | "Epoch 17/25\n", 266 | "60000/60000 [==============================] - 37s 617us/sample - loss: 0.1995 - accuracy: 0.9254 - val_loss: 0.2710 - val_accuracy: 0.9028\n", 267 | "Epoch 18/25\n", 268 | "60000/60000 [==============================] - 37s 623us/sample - loss: 0.1920 - accuracy: 0.9272 - val_loss: 0.2753 - val_accuracy: 0.9036\n", 269 | "Epoch 19/25\n", 270 | "60000/60000 [==============================] - 36s 604us/sample - loss: 0.1931 - accuracy: 0.9273 - val_loss: 0.2679 - val_accuracy: 0.9054\n", 271 | "Epoch 20/25\n", 272 | "60000/60000 [==============================] - 36s 605us/sample - loss: 0.1852 - accuracy: 0.9290 - val_loss: 0.2765 - val_accuracy: 0.9054\n", 273 | "Epoch 21/25\n", 274 | "60000/60000 [==============================] - 36s 603us/sample - loss: 0.1803 - accuracy: 0.9320 - val_loss: 0.2618 - val_accuracy: 0.9078\n", 275 | "Epoch 22/25\n", 276 | "60000/60000 [==============================] - 36s 601us/sample - loss: 0.1776 - accuracy: 0.9329 - val_loss: 0.2788 - val_accuracy: 0.9040\n", 277 | "Epoch 23/25\n", 278 | "60000/60000 [==============================] - 35s 589us/sample - loss: 0.1747 - accuracy: 0.9337 - val_loss: 0.2751 - val_accuracy: 0.9063\n", 279 | "Epoch 24/25\n", 280 | "60000/60000 [==============================] - 35s 580us/sample - loss: 0.1728 - accuracy: 0.9359 - val_loss: 0.2704 - val_accuracy: 0.9066\n", 281 | "Epoch 25/25\n", 282 | "60000/60000 [==============================] - 34s 574us/sample - loss: 0.1715 - accuracy: 0.9338 - val_loss: 0.2828 - val_accuracy: 0.9021\n" 283 | ], 284 | "name": "stdout" 285 | }, 286 | { 287 | "output_type": "execute_result", 288 | "data": { 289 | "text/plain": [ 290 | "" 291 | ] 292 | }, 293 | "metadata": { 294 | "tags": [] 295 | }, 296 | "execution_count": 8 297 | } 298 | ] 299 | }, 300 | { 301 | "cell_type": "markdown", 302 | "metadata": { 303 | "id": "qveZwzeS_t4B", 304 | "colab_type": "text" 305 | }, 306 | "source": [ 307 | "Run page: https://app.wandb.ai/sayakpaul/tensorboard-integration/runs/e8kv5zab" 308 | ] 309 | } 310 | ] 311 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /ML GDE Dev Challenge/README.md: -------------------------------------------------------------------------------- 1 | The folder contains the Jupyter Notebook of this blog [Predicting the publisher’s name from an article: A case study](https://medium.com/google-developer-experts/predicting-the-publishers-name-from-an-article-a-case-study-fd9e2ea8315d). 2 | -------------------------------------------------------------------------------- /NALU in TF 2.0/README.md: -------------------------------------------------------------------------------- 1 | In the `NALU_using_TF_2_0` notebook I present my implementation of [Neural Arithmetic Logic Units](https://arxiv.org/abs/1808.00508) that was proposed by Trask et al. last year. The paper presents a solution to a very important problem in neural networks. Despite having the capability of approximating any arbitrary functions, neural networks show very poor performance at counting. Put in other words, they fail to extrapolate to the values that were seen by them during the training process. 2 | 3 | The novelty of the paper lies in two main components as proposed by the authors: Neural Accumulator (NAC) and Neural Arithmetic Logic Gates (NALU) which build on top of NAC. I would encourage the readers to take a look at the paper once, it is extremely well written and most importantly written in plain English. I would also suggest you read [this article](https://medium.com/tensorflow/understanding-neural-arithmetic-logic-units-11b0f85c1d1d) on NALU alongside the original paper. 4 | 5 | I used the TensorFlow 2.0 (beta) for this implementation since in TF 2.0 you get a lot of flexibility in terms customization of layers, defining forward passes and so on. 6 | 7 | I have included a link to run the notebook on Google Colab. But make sure to include the `utils.py` file when you are running it. It contains a utility function for generating toy data. 8 | -------------------------------------------------------------------------------- /NALU in TF 2.0/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sayakpaul/TF-2.0-Hacks/3ec76db4a1a0a666f037863e8e2539dce15b1162/NALU in TF 2.0/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /NALU in TF 2.0/figures/download (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sayakpaul/TF-2.0-Hacks/3ec76db4a1a0a666f037863e8e2539dce15b1162/NALU in TF 2.0/figures/download (1).png -------------------------------------------------------------------------------- /NALU in TF 2.0/figures/download (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sayakpaul/TF-2.0-Hacks/3ec76db4a1a0a666f037863e8e2539dce15b1162/NALU in TF 2.0/figures/download (2).png -------------------------------------------------------------------------------- /NALU in TF 2.0/figures/download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sayakpaul/TF-2.0-Hacks/3ec76db4a1a0a666f037863e8e2539dce15b1162/NALU in TF 2.0/figures/download.png -------------------------------------------------------------------------------- /NALU in TF 2.0/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def generate_data(min_val, max_val, observations, op): 4 | data = np.random.uniform(min_val, max_val, size=(observations, 2)) 5 | if op == '+': 6 | target = data[:, 0] + data[:, 1] 7 | elif op == '-': 8 | target = data[:, 0] - data[:, 1] 9 | elif op == '*': 10 | target = data[:, 0] * data[:, 1] 11 | elif op == '/': 12 | target = data[:, 0] / data[:, 1] 13 | elif op == '^2': 14 | data = np.random.uniform(min_val, max_val, size=(observations, 1)) 15 | target = data ** 2 16 | elif op == 'sqrt': 17 | data = np.random.uniform(min_val, max_val, size=(observations, 1)) 18 | target = np.sqrt(data) 19 | 20 | return data, target 21 | -------------------------------------------------------------------------------- /Prototypical_Networks_in_TensorFlow_2_0.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Prototypical Networks in TensorFlow 2.0.ipynb", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "id": "view-in-github", 23 | "colab_type": "text" 24 | }, 25 | "source": [ 26 | "\"Open" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": { 32 | "id": "3weO3xt6-i1E", 33 | "colab_type": "text" 34 | }, 35 | "source": [ 36 | "## Data gathering" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "eROAs3aS6wHc", 43 | "colab_type": "code", 44 | "colab": {} 45 | }, 46 | "source": [ 47 | "!wget https://github.com/brendenlake/omniglot/raw/master/python/images_background.zip" 48 | ], 49 | "execution_count": 0, 50 | "outputs": [] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "metadata": { 55 | "id": "d78ikRIV-Y_N", 56 | "colab_type": "code", 57 | "colab": {} 58 | }, 59 | "source": [ 60 | "!wget https://github.com/brendenlake/omniglot/raw/master/python/images_evaluation.zip" 61 | ], 62 | "execution_count": 0, 63 | "outputs": [] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "metadata": { 68 | "id": "sL-MdFex8spV", 69 | "colab_type": "code", 70 | "colab": {} 71 | }, 72 | "source": [ 73 | "!unzip images_background.zip" 74 | ], 75 | "execution_count": 0, 76 | "outputs": [] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "id": "nIlL1x9_-bDt", 82 | "colab_type": "code", 83 | "colab": {} 84 | }, 85 | "source": [ 86 | "!unzip images_evaluation.zip" 87 | ], 88 | "execution_count": 0, 89 | "outputs": [] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "metadata": { 94 | "id": "pCrqYMNr8_8o", 95 | "colab_type": "code", 96 | "colab": {} 97 | }, 98 | "source": [ 99 | "from matplotlib import pyplot as plt\n", 100 | "from PIL import Image\n", 101 | "import multiprocessing as mp\n", 102 | "import pandas as pd \n", 103 | "import numpy as np \n", 104 | "import cv2\n", 105 | "import os" 106 | ], 107 | "execution_count": 0, 108 | "outputs": [] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "metadata": { 113 | "id": "YLP_-hY8c2xp", 114 | "colab_type": "code", 115 | "outputId": "4ed3cb39-de3a-4185-8efa-a2db8ed61b99", 116 | "colab": { 117 | "base_uri": "https://localhost:8080/", 118 | "height": 357 119 | } 120 | }, 121 | "source": [ 122 | "random_10_bengali = os.listdir('/content/images_background/Bengali/character01/')\n", 123 | "random_10_bengali" 124 | ], 125 | "execution_count": 0, 126 | "outputs": [ 127 | { 128 | "output_type": "execute_result", 129 | "data": { 130 | "text/plain": [ 131 | "['0132_16.png',\n", 132 | " '0132_01.png',\n", 133 | " '0132_12.png',\n", 134 | " '0132_17.png',\n", 135 | " '0132_08.png',\n", 136 | " '0132_15.png',\n", 137 | " '0132_18.png',\n", 138 | " '0132_19.png',\n", 139 | " '0132_04.png',\n", 140 | " '0132_06.png',\n", 141 | " '0132_07.png',\n", 142 | " '0132_11.png',\n", 143 | " '0132_03.png',\n", 144 | " '0132_10.png',\n", 145 | " '0132_09.png',\n", 146 | " '0132_14.png',\n", 147 | " '0132_05.png',\n", 148 | " '0132_13.png',\n", 149 | " '0132_20.png',\n", 150 | " '0132_02.png']" 151 | ] 152 | }, 153 | "metadata": { 154 | "tags": [] 155 | }, 156 | "execution_count": 6 157 | } 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "metadata": { 163 | "id": "JBl4HuTTceiz", 164 | "colab_type": "code", 165 | "outputId": "34a54a61-5272-4665-9c89-7ccf296bb571", 166 | "colab": { 167 | "base_uri": "https://localhost:8080/", 168 | "height": 122 169 | } 170 | }, 171 | "source": [ 172 | "Image.open('/content/images_background/Bengali/character01/0132_01.png')" 173 | ], 174 | "execution_count": 0, 175 | "outputs": [ 176 | { 177 | "output_type": "execute_result", 178 | "data": { 179 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGkAAABpAQAAAAAR+TCXAAAA9ElEQVR4nN2UMU7EMBBF3ziRktIH\noMhBKLgSJ8B7A44UbsARgkRBmZW22GKTobDXzpAN0grBSrh7+t/f8y3LoizWzmHWP0b1l9VJRETc\nXsRb9XjNQS0OKlVV1UZ1vH3fH+HpWzVcEzVvq+J5KDhxWKoHa35lRgv2KF3IGJ76aK7ThG/4Ym5M\nlKaAs9quZ24NhqLOxNyNRj7mFrXdNlfD+lyLL51VPyI+G7M+AnDMezuAMeMIMMTrqZOvD3sAJ/0J\nmHbnvvcT8J6SayqEfHnrRkNqnNAvOrnou8tRNItv5/bP++9wsDgarK3ZeYPyZe/vDXkBPwHQkznd\ntAZ8bwAAAABJRU5ErkJggg==\n", 180 | "text/plain": [ 181 | "" 182 | ] 183 | }, 184 | "metadata": { 185 | "tags": [] 186 | }, 187 | "execution_count": 7 188 | } 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "metadata": { 194 | "id": "9HCBC0COemXH", 195 | "colab_type": "code", 196 | "outputId": "dd21b59b-955b-4479-9b0d-8d0559e37ac4", 197 | "colab": { 198 | "base_uri": "https://localhost:8080/", 199 | "height": 122 200 | } 201 | }, 202 | "source": [ 203 | "Image.open('/content/images_background/Bengali/character01/0132_10.png')" 204 | ], 205 | "execution_count": 0, 206 | "outputs": [ 207 | { 208 | "output_type": "execute_result", 209 | "data": { 210 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGkAAABpAQAAAAAR+TCXAAABP0lEQVR4nO3TsU6EQBCA4X8BA4WJ\n2F3JI1jaeFL4UMbm9t7DwkdZLz6Aj4DmiisxsSAGGItdYDbxEqPtbcXHDLMzCxhBrW1CtE78iR/G\nXAZujTEltDUkMNiQ4flypJSFLFym/dt7iG5EpKe58TS+lAnPrn/T5O6PEw11xH4X8SC15hONpqVV\nHLCdYp/e9zo5y0fFbMUm7qq2EauoSUqnm6wLH/UnechDWwn0QJUdm3ckdYp+tRMFrqebvkTuZEm2\nUSkA1EQO+IyjbTmziTbqWoBujnZA2U8seqAYJp75TWxgov7ZBHAWDtXMCmhWSxsO3J3PFpHuQoQx\nFREbmhxSMydn3TQBCZB88Xru980AI9aU6nDs4/5WzVvvcYoVuT6rFcUyAuRcIeNciue1Tl4++Pkd\nmZiqFAAPMbOYnPgvfgM1ZFx1YvSO4wAAAABJRU5ErkJggg==\n", 211 | "text/plain": [ 212 | "" 213 | ] 214 | }, 215 | "metadata": { 216 | "tags": [] 217 | }, 218 | "execution_count": 8 219 | } 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "metadata": { 225 | "id": "3fWdY-TXeqTw", 226 | "colab_type": "code", 227 | "outputId": "4b590eed-bd73-4b46-ecaf-46a521485b65", 228 | "colab": { 229 | "base_uri": "https://localhost:8080/", 230 | "height": 122 231 | } 232 | }, 233 | "source": [ 234 | "Image.open('/content/images_background/Bengali/character01/0132_02.png')" 235 | ], 236 | "execution_count": 0, 237 | "outputs": [ 238 | { 239 | "output_type": "execute_result", 240 | "data": { 241 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGkAAABpAQAAAAAR+TCXAAABDklEQVR4nO2SMU7EMBREn73WkmLF\nrqjoCDfgCOEmHIGS0tyEY1B6b+KCA3glCiMl+RQ4ji1BWAQUSLh7mu/R/LGVUJx7TXX+Gu4NGF6a\nH7DaCmhORERE+ovwa5k/xINqSww8legZS3RIiRbsjLISW6ij4bYcznVpgL6OEeM3Vrgj1KovUm0t\nrlZtjbWzqL5ST2mPiTHU2FuAmIddfdcvWZkAuR+d9CEPR0Dl2lPax0kdAEn+GqwFUjsaugcYN3QJ\nW4Cz7Hwe5mfRYIB4mVE/T0ZgQI3gu2uZVhBwb8YYwCqQfV7wBlSx7w7WBTbz59DAGnZcZSslkBrM\nXTXvV3cc+iXVfMVKtxUqtzT8j5/iK4N3Rb2z1Ok9AAAAAElFTkSuQmCC\n", 242 | "text/plain": [ 243 | "" 244 | ] 245 | }, 246 | "metadata": { 247 | "tags": [] 248 | }, 249 | "execution_count": 9 250 | } 251 | ] 252 | }, 253 | { 254 | "cell_type": "markdown", 255 | "metadata": { 256 | "id": "rxb328gS9gJp", 257 | "colab_type": "text" 258 | }, 259 | "source": [ 260 | "## Data reading and augmentation\n", 261 | "The Omniglot data set is designed for developing more human-like learning algorithms. It contains 1623 different handwritten characters from 50 different alphabets. Then to increase the number of classes, all the images are rotated by 90, 180 and 270 degrees and each rotation resulted in one more class. Hence the total count of classes reached to 6492(1623 * 4) classes. We split images of 4200 classes to training data and the rest went to test set." 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "metadata": { 267 | "id": "rdMcXOli9bFl", 268 | "colab_type": "code", 269 | "colab": {} 270 | }, 271 | "source": [ 272 | "train_dir = os.listdir('images_background/')\n", 273 | "datax = np.array([])" 274 | ], 275 | "execution_count": 0, 276 | "outputs": [] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "metadata": { 281 | "id": "XEssKQ6f9z73", 282 | "colab_type": "code", 283 | "colab": {} 284 | }, 285 | "source": [ 286 | "def image_rotate(img, angle):\n", 287 | " \"\"\"\n", 288 | " Image rotation at certain angle. It is used for data augmentation \n", 289 | " \"\"\"\n", 290 | " rows,cols, _ = img.shape\n", 291 | " M = cv2.getRotationMatrix2D((cols/2 ,rows/2),angle,1)\n", 292 | " dst = cv2.warpAffine(img,M,(cols,rows))\n", 293 | " return np.expand_dims(dst, 0)" 294 | ], 295 | "execution_count": 0, 296 | "outputs": [] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "metadata": { 301 | "id": "h-b0wy9J953l", 302 | "colab_type": "code", 303 | "colab": {} 304 | }, 305 | "source": [ 306 | "def read_alphabets(alphabet_directory, directory):\n", 307 | " \"\"\"\n", 308 | " Reads all the characters from alphabet_directory and augment each image with 90, 180, 270 degrees of rotation.\n", 309 | " \"\"\"\n", 310 | " datax = None\n", 311 | " datay = []\n", 312 | " characters = os.listdir(alphabet_directory)\n", 313 | " for character in characters:\n", 314 | " images = os.listdir(alphabet_directory + character + '/')\n", 315 | " for img in images:\n", 316 | " image = cv2.resize(cv2.imread(alphabet_directory + character + '/' + img), (28,28))\n", 317 | " image90 = image_rotate(image, 90)\n", 318 | " image180 = image_rotate(image, 180)\n", 319 | " image270 = image_rotate(image, 270)\n", 320 | " image = np.expand_dims(image, 0)\n", 321 | " if datax is None:\n", 322 | " datax = np.vstack([image, image90, image180, image270])\n", 323 | " else:\n", 324 | " datax = np.vstack([datax, image, image90, image180, image270])\n", 325 | " datay.append(directory + '_' + character + '_0')\n", 326 | " datay.append(directory + '_' + character + '_90')\n", 327 | " datay.append(directory + '_' + character + '_180')\n", 328 | " datay.append(directory + '_' + character + '_270')\n", 329 | " return datax, np.array(datay)" 330 | ], 331 | "execution_count": 0, 332 | "outputs": [] 333 | }, 334 | { 335 | "cell_type": "code", 336 | "metadata": { 337 | "id": "3j4URJB_98pC", 338 | "colab_type": "code", 339 | "colab": {} 340 | }, 341 | "source": [ 342 | "def read_images(base_directory):\n", 343 | " \"\"\"\n", 344 | " Used multithreading for data reading to decrease the reading time drastically\n", 345 | " \"\"\"\n", 346 | " datax = None\n", 347 | " datay = []\n", 348 | " pool = mp.Pool(mp.cpu_count())\n", 349 | " results = [pool.apply(read_alphabets, args=(base_directory + '/' + directory + '/', directory, )) for directory in os.listdir(base_directory)]\n", 350 | " pool.close()\n", 351 | " for result in results:\n", 352 | " if datax is None:\n", 353 | " datax = result[0]\n", 354 | " datay = result[1]\n", 355 | " else:\n", 356 | " datax = np.vstack([datax, result[0]])\n", 357 | " datay = np.concatenate([datay, result[1]])\n", 358 | " return datax, datay" 359 | ], 360 | "execution_count": 0, 361 | "outputs": [] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "metadata": { 366 | "id": "yPpRsOd-9_A7", 367 | "colab_type": "code", 368 | "outputId": "71cc1d43-5291-41e4-d9f2-92f9f32f344d", 369 | "colab": { 370 | "base_uri": "https://localhost:8080/", 371 | "height": 51 372 | } 373 | }, 374 | "source": [ 375 | "%time trainx, trainy = read_images('images_background/')" 376 | ], 377 | "execution_count": 0, 378 | "outputs": [ 379 | { 380 | "output_type": "stream", 381 | "text": [ 382 | "CPU times: user 1.08 s, sys: 613 ms, total: 1.69 s\n", 383 | "Wall time: 15.5 s\n" 384 | ], 385 | "name": "stdout" 386 | } 387 | ] 388 | }, 389 | { 390 | "cell_type": "code", 391 | "metadata": { 392 | "id": "TvFXXdoT-Ew8", 393 | "colab_type": "code", 394 | "outputId": "3d6978a8-d6b4-4c3b-b1f6-abbac8eaa202", 395 | "colab": { 396 | "base_uri": "https://localhost:8080/", 397 | "height": 51 398 | } 399 | }, 400 | "source": [ 401 | "%time testx, testy = read_images('images_evaluation/')" 402 | ], 403 | "execution_count": 0, 404 | "outputs": [ 405 | { 406 | "output_type": "stream", 407 | "text": [ 408 | "CPU times: user 516 ms, sys: 321 ms, total: 838 ms\n", 409 | "Wall time: 10.5 s\n" 410 | ], 411 | "name": "stdout" 412 | } 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "metadata": { 418 | "id": "gR7DaBrh-uil", 419 | "colab_type": "code", 420 | "colab": {} 421 | }, 422 | "source": [ 423 | "assert(trainx.shape[0] == trainy.shape[0])\n", 424 | "assert(testx.shape[0] == testy.shape[0])" 425 | ], 426 | "execution_count": 0, 427 | "outputs": [] 428 | }, 429 | { 430 | "cell_type": "code", 431 | "metadata": { 432 | "id": "51WVpbxYi2-i", 433 | "colab_type": "code", 434 | "outputId": "a66924b9-78a2-4c54-b1fc-543bf20b104f", 435 | "colab": { 436 | "base_uri": "https://localhost:8080/", 437 | "height": 34 438 | } 439 | }, 440 | "source": [ 441 | "trainx.shape, trainy.shape, testx.shape, testy.shape" 442 | ], 443 | "execution_count": 0, 444 | "outputs": [ 445 | { 446 | "output_type": "execute_result", 447 | "data": { 448 | "text/plain": [ 449 | "((77120, 28, 28, 3), (77120,), (52720, 28, 28, 3), (52720,))" 450 | ] 451 | }, 452 | "metadata": { 453 | "tags": [] 454 | }, 455 | "execution_count": 17 456 | } 457 | ] 458 | }, 459 | { 460 | "cell_type": "markdown", 461 | "metadata": { 462 | "id": "TnIaQ2_O_B1M", 463 | "colab_type": "text" 464 | }, 465 | "source": [ 466 | "## Building the network in TensorFlow 2.0 using `tf.keras`" 467 | ] 468 | }, 469 | { 470 | "cell_type": "code", 471 | "metadata": { 472 | "id": "4tgJ2u9H-yDt", 473 | "colab_type": "code", 474 | "colab": {} 475 | }, 476 | "source": [ 477 | "!pip install tensorflow-gpu==2.0.0-beta1" 478 | ], 479 | "execution_count": 0, 480 | "outputs": [] 481 | }, 482 | { 483 | "cell_type": "code", 484 | "metadata": { 485 | "id": "Eu8Z5086_Low", 486 | "colab_type": "code", 487 | "outputId": "2ee996d7-7a24-4106-b195-c5a4d409e77c", 488 | "colab": { 489 | "base_uri": "https://localhost:8080/", 490 | "height": 34 491 | } 492 | }, 493 | "source": [ 494 | "import tensorflow as tf\n", 495 | "\n", 496 | "print(tf.__version__)" 497 | ], 498 | "execution_count": 0, 499 | "outputs": [ 500 | { 501 | "output_type": "stream", 502 | "text": [ 503 | "2.0.0-beta1\n" 504 | ], 505 | "name": "stdout" 506 | } 507 | ] 508 | }, 509 | { 510 | "cell_type": "code", 511 | "metadata": { 512 | "id": "GV02b54s_qEs", 513 | "colab_type": "code", 514 | "colab": {} 515 | }, 516 | "source": [ 517 | "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n", 518 | "from sklearn.metrics import pairwise_distances\n", 519 | "from tensorflow import keras" 520 | ], 521 | "execution_count": 0, 522 | "outputs": [] 523 | }, 524 | { 525 | "cell_type": "code", 526 | "metadata": { 527 | "id": "38oGcl3hpj7-", 528 | "colab_type": "code", 529 | "colab": {} 530 | }, 531 | "source": [ 532 | "conv_block = keras.models.Sequential([\n", 533 | " keras.layers.Conv2D(64, (3, 3), input_shape=(28, 28, 3)),\n", 534 | " keras.layers.BatchNormalization(),\n", 535 | " keras.layers.ReLU(),\n", 536 | " keras.layers.MaxPool2D(pool_size=(2,2))\n", 537 | "]) " 538 | ], 539 | "execution_count": 0, 540 | "outputs": [] 541 | }, 542 | { 543 | "cell_type": "code", 544 | "metadata": { 545 | "id": "WBJoDcx4sJIm", 546 | "colab_type": "code", 547 | "colab": {} 548 | }, 549 | "source": [ 550 | "flatten_layer = tf.keras.layers.Flatten()" 551 | ], 552 | "execution_count": 0, 553 | "outputs": [] 554 | }, 555 | { 556 | "cell_type": "code", 557 | "metadata": { 558 | "id": "SbfdNr-rqnOh", 559 | "colab_type": "code", 560 | "colab": {} 561 | }, 562 | "source": [ 563 | "def get_embeddings(support_set):\n", 564 | " x = conv_block(support_set)\n", 565 | " x = conv_block()\n", 566 | " x = conv_block()\n", 567 | " x = conv_block()\n", 568 | " return flatten_layer(x)" 569 | ], 570 | "execution_count": 0, 571 | "outputs": [] 572 | }, 573 | { 574 | "cell_type": "code", 575 | "metadata": { 576 | "id": "HR0E241E_rzU", 577 | "colab_type": "code", 578 | "colab": {} 579 | }, 580 | "source": [ 581 | "class Net(keras.Model):\n", 582 | " \"\"\"\n", 583 | " Image2Vector CNN which takes image of dimension (28x28x3) and return column vector length 64\n", 584 | " \"\"\"\n", 585 | " def sub_block(self, out_channels=64, kernel_size=(3,3)):\n", 586 | " block = keras.models.Sequential([\n", 587 | " keras.layers.Conv2D(filters=out_channels, \\\n", 588 | " kernel_size=kernel_size, input_shape=(28, 28, 3)),\n", 589 | " keras.layers.BatchNormalization(),\n", 590 | " keras.layers.ReLU(),\n", 591 | " keras.layers.MaxPool2D(pool_size=(2,2))\n", 592 | " ])\n", 593 | " return block\n", 594 | " \n", 595 | " def __init__(self):\n", 596 | " super(Net, self).__init__()\n", 597 | " self.convnet1 = self.sub_block()\n", 598 | " self.convnet2 = self.sub_block()\n", 599 | " self.convnet3 = self.sub_block()\n", 600 | " self.convnet4 = self.sub_block()\n", 601 | "\n", 602 | " def call(self, x):\n", 603 | " x = self.convnet1(x)\n", 604 | " x = self.convnet2(x)\n", 605 | " x = self.convnet3(x)\n", 606 | " x = self.convnet4(x)\n", 607 | " x = keras.layers.Flatten()(x)\n", 608 | " return x" 609 | ], 610 | "execution_count": 0, 611 | "outputs": [] 612 | }, 613 | { 614 | "cell_type": "code", 615 | "metadata": { 616 | "id": "1jfN3KEYbQuL", 617 | "colab_type": "code", 618 | "colab": {} 619 | }, 620 | "source": [ 621 | "" 622 | ], 623 | "execution_count": 0, 624 | "outputs": [] 625 | }, 626 | { 627 | "cell_type": "markdown", 628 | "metadata": { 629 | "id": "JEKvU-M_bN0l", 630 | "colab_type": "text" 631 | }, 632 | "source": [ 633 | "To be continued.\n", 634 | "\n", 635 | "https://blog.floydhub.com/n-shot-learning/" 636 | ] 637 | } 638 | ] 639 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](https://i.ibb.co/LJdwtJ8/TF-Full-Color-Primary-Horizontal.png) 2 | 3 | This respository contains my exploration of the newly released [TensorFlow 2.0](http://tensflow.org/). TensorFlow team introduced a lot of new and useful changes in this release; automatic mixed precision training, flexible custom training, distributed GPU training, enhanced ops for the high-level Keras API are some of my personal favorites. 4 | You can see all of the new changes [here](https://www.youtube.com/watch?v=kPweUtct2yY). 5 | 6 | Apart from the official TensorFlow 2.0 [guides](https://www.tensorflow.org/guide) and the [tutorials](https://www.tensorflow.org/tutorials), I highly recommend the following resources if you want to learn more about TensorFlow 2.0 (updated on an ad-hoc basis): 7 | 8 | - [tf.keras for Researchers: Crash Course](https://colab.research.google.com/drive/17u-pRZJnKN0gO5XZmq8n5A2bKGrfKEUg) by Francois Chollet 9 | - [TensorFlow 2.0 + Keras Crash Course](https://colab.research.google.com/drive/1UCJt8EYjlzCs1H1d1X0iDGYJsHKwu-NO) by Francois Chollet 10 | - [Introduction to Keras for Researchers](https://colab.research.google.com/drive/169PfzM0kvtA5UP4k6Sl1yCG9tsE2MLia) by Francois Chollet 11 | - [Introduction to Keras for Engineers](https://colab.research.google.com/drive/1lWUGZarlbORaHYUZlF9muCgpPl8pEvve) by Francois Chollet 12 | - [Inside TensorFlow: tf.Keras (part 1)](https://youtu.be/UYRBHFAvLSs) by Francois Chollet 13 | - [Inside TensorFlow: tf.Keras (part 2)](https://www.youtube.com/watch?v=uhzGTijaw8A) by Francois Chollet 14 | - [keras.io exmples](https://keras.io/examples) 15 | - [keras.io guides](https://keras.io/guides) 16 | - [Deep Learning with Python (second edition)](https://www.manning.com/books/deep-learning-with-python) by Francois Chollet 17 | - [Introduction to TensorFlow in Python](https://www.datacamp.com/courses/introduction-to-tensorflow-in-python?tap_a=5644-dce66f&tap_s=357540-5b28dd) by DataCamp (Instructor: Isaiah Hull) 18 | - [Ten Important Updates from TensorFlow 2.0](https://www.datacamp.com/community/tutorials/ten-important-updates-tensorflow) (article by me) 19 | - [Hands-On Machine Learning with Scikit-Learn, Keras and TensorFlow](https://www.amazon.in/Hands-Machine-Learning-Scikit-Learn-Tensor/dp/9352139054) (best-selling book by Aurélien Géron) 20 | - [Deep Learning for Computer Vision with Python](https://www.pyimagesearch.com/deep-learning-computer-vision-python-book/) by Adrian Rosebrock 21 | - [Intro to TensorFlow for Deep Learning](https://www.udacity.com/course/intro-to-tensorflow-for-deep-learning--ud187) by Udacity (Instructors: Magnus Hyttsten, Juan Delgado, Paige Bailey) 22 | - [Intro to Machine Learning with TensorFlow Nanodegree](https://www.udacity.com/course/intro-to-machine-learning-with-tensorflow-nanodegree--nd230) 23 | - [Practical Machine Learning with Tensorflow](https://www.youtube.com/playlist?list=PLOzRYVm0a65cTV_t0BYj-nV8VX_Me6Es3) by NPTEL (Instructor: Ashish Tendulkar) 24 | - [Hands-On Neural Networks with TensorFlow 2.0](https://www.amazon.com/Hands-Neural-Networks-Tensorflow-2-0-dp-1789615550/dp/1789615550/) (book by Paolo Galeone) 25 | - [DeepLearning.AI TensorFlow Developer Professional Certificate](https://www.coursera.org/professional-certificates/tensorflow-in-practice) (previously known as _TensorFlow in Practice_) 26 | - [TensorFlow: Data and Deployment Specialization](https://www.coursera.org/specializations/tensorflow-data-and-deployment) by deeplearning.ai 27 | - [TensorFlow 2 for Deep Learning Specialization](https://www.coursera.org/specializations/tensorflow2-deeplearning) by Imperial College of London 28 | - [Keras vs. tf.keras: What’s the difference in TensorFlow 2.0?](https://www.pyimagesearch.com/2019/10/21/keras-vs-tf-keras-whats-the-difference-in-tensorflow-2-0/) by PyImageSearch 29 | - [Introduction to TensorFlow Lite](https://www.udacity.com/course/intro-to-tensorflow-lite--ud190) 30 | - [Intro to Machine Learning with TensorFlow Nanodegree](https://www.udacity.com/course/intro-to-machine-learning-with-tensorflow-nanodegree--nd230) 31 | - [freeCodeCamp's Python Machine Learning course with TensorFlow 2.0](https://www.freecodecamp.org/news/massive-tensorflow-2-0-free-course/) 32 | - [TensorFlow: Advanced Techniques Specialization](https://www.coursera.org/specializations/tensorflow-advanced-techniques) by deeplearning.ai 33 | -------------------------------------------------------------------------------- /Simple_CBOW_in_TF_2.0.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "The **Continuous Bag-of-Words model (CBOW)** is frequently used in NLP deep\n", 8 | "learning. It is a model that tries to predict words given the context of\n", 9 | "a few words before and a few words after the target word. This is\n", 10 | "distinct from language modeling, since CBOW is not sequential and does\n", 11 | "not have to be probabilistic. Typcially, CBOW is used to quickly train\n", 12 | "word embeddings, and these embeddings are used to initialize the\n", 13 | "embeddings of some more complicated model. Usually, this is referred to\n", 14 | "as *pretraining embeddings*. It almost always helps performance a couple\n", 15 | "of percent.\n", 16 | "\n", 17 | "The CBOW model is as follows. Given a target word $w_i$ and an\n", 18 | "$N$ context window on each side, $w_{i-1}, \\dots, w_{i-N}$\n", 19 | "and $w_{i+1}, \\dots, w_{i+N}$, referring to all context words\n", 20 | "collectively as $C$, CBOW tries to minimize\n", 21 | "\n", 22 | "\\begin{align}-\\log p(w_i | C) = -\\log \\text{Softmax}(A(\\sum_{w \\in C} q_w) + b)\\end{align}\n", 23 | "\n", 24 | "where $q_w$ is the embedding for word $w$." 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "In this notebook, I am going to use `tensorflow 2.0.0-beta0` and take the advantage of its flexible custom model definitions to create a COBW model and will train it to get the embeddings of a toy dataset. " 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 122, 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "name": "stdout", 41 | "output_type": "stream", 42 | "text": [ 43 | "2.0.0-beta0\n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "import tensorflow as tf\n", 49 | "from tensorflow.keras.models import Model\n", 50 | "from tensorflow.keras.layers import *\n", 51 | "import numpy as np\n", 52 | "from tqdm import tqdm\n", 53 | "\n", 54 | "print(tf.__version__)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 3, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "# 2 words to the left, 2 to the right\n", 64 | "CONTEXT_SIZE = 2 \n", 65 | "# Embedding dimension on left\n", 66 | "EMBED_LEFT = 10\n", 67 | "# Embedding dimension on right\n", 68 | "EMBED_RIGHT = 10" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 2, 74 | "metadata": {}, 75 | "outputs": [ 76 | { 77 | "name": "stdout", 78 | "output_type": "stream", 79 | "text": [ 80 | "[(['When', 'forty', 'shall', 'besiege'], 'winters'), (['forty', 'winters', 'besiege', 'thy'], 'shall'), (['winters', 'shall', 'thy', 'brow,'], 'besiege'), (['shall', 'besiege', 'brow,', 'And'], 'thy'), (['besiege', 'thy', 'And', 'dig'], 'brow,')]\n" 81 | ] 82 | } 83 | ], 84 | "source": [ 85 | "# Row test from Shakespeare Sonnet 2\n", 86 | "raw_text = \"\"\"When forty winters shall besiege thy brow,\n", 87 | "And dig deep trenches in thy beauty's field,\n", 88 | "Thy youth's proud livery so gazed on now,\n", 89 | "Will be a totter'd weed of small worth held:\n", 90 | "Then being asked, where all thy beauty lies,\n", 91 | "Where all the treasure of thy lusty days;\n", 92 | "To say, within thine own deep sunken eyes,\n", 93 | "Were an all-eating shame, and thriftless praise.\n", 94 | "How much more praise deserv'd thy beauty's use,\n", 95 | "If thou couldst answer 'This fair child of mine\n", 96 | "Shall sum my count, and make my old excuse,'\n", 97 | "Proving his beauty by succession thine!\n", 98 | "This were to be new made when thou art old,\n", 99 | "And see thy blood warm when thou feel'st it cold.\"\"\".split()\n", 100 | "\n", 101 | "# By deriving a set from `raw_text`, we deduplicate the array\n", 102 | "vocab = set(raw_text)\n", 103 | "vocab_size = len(vocab)\n", 104 | "\n", 105 | "word_to_ix = {word: i for i, word in enumerate(vocab)}\n", 106 | "data = []\n", 107 | "for i in range(2, len(raw_text) - 2):\n", 108 | " context = [raw_text[i - 2], raw_text[i - 1],\n", 109 | " raw_text[i + 1], raw_text[i + 2]]\n", 110 | " target = raw_text[i]\n", 111 | " data.append((context, target))\n", 112 | "print(data[:5])" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "The toy dataset is ready. Let's now specify the model definition. " 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 158, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "class CBOW(Model):\n", 129 | "\n", 130 | " def __init__(self, vocab_size, embedding_dim_left, embedding_dim_right, context_size):\n", 131 | " super(CBOW, self).__init__()\n", 132 | " self.embeddings_left = Embedding(vocab_size, embedding_dim_left)\n", 133 | " self.embeddings_right = Embedding(vocab_size, embedding_dim_right)\n", 134 | " self.linear1 = Dense(128, activation='linear', input_dim=(context_size * embedding_dim_left))\n", 135 | " self.linear2 = Dense(vocab_size, activation='linear', input_dim=128)\n", 136 | "\n", 137 | " def call(self, inputs):\n", 138 | " left_embeds = tf.reshape(self.embeddings_left(inputs[:2]), [1,20])\n", 139 | " right_embeds = tf.reshape(self.embeddings_left(inputs[:2]), [1,20])\n", 140 | " left_out = tf.keras.activations.relu(self.linear1(left_embeds))\n", 141 | " right_out = tf.keras.activations.relu(self.linear1(right_embeds))\n", 142 | " out = self.linear2(left_out + right_out)\n", 143 | " return out" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "Let's instantiate the model. " 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 175, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "model = CBOW(vocab_size, EMBED_LEFT, EMBED_RIGHT, CONTEXT_SIZE)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "Define our loss function and optimizer. " 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 176, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "loss_func = tf.keras.losses.SparseCategoricalCrossentropy()\n", 176 | "optimizer = tf.keras.optimizers.SGD(learning_rate=0.001)" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "The function which will train our model. We use `GradientTape` to make use of automatic differentiation. Makes the process of Backpropagation much more flexible. Makes it also easy to chart dependencies inside the training loop. " 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": 177, 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [ 192 | "@tf.function\n", 193 | "def model_train(features, labels):\n", 194 | " # Define the GradientTape context\n", 195 | " with tf.GradientTape() as tape:\n", 196 | " # Get the probabilities\n", 197 | " predictions = model(features)\n", 198 | " # Calculate the loss\n", 199 | " loss = loss_func(labels, predictions)\n", 200 | " # Get the gradients\n", 201 | " gradients = tape.gradient(loss, model.trainable_variables)\n", 202 | " # Update the weights\n", 203 | " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", 204 | " return loss" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "Finally we train the model for 10 epochs. " 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 178, 217 | "metadata": {}, 218 | "outputs": [ 219 | { 220 | "name": "stderr", 221 | "output_type": "stream", 222 | "text": [ 223 | "100%|██████████| 10/10 [00:01<00:00, 8.50it/s]" 224 | ] 225 | }, 226 | { 227 | "name": "stdout", 228 | "output_type": "stream", 229 | "text": [ 230 | "[, , , , , , , , , ]\n" 231 | ] 232 | }, 233 | { 234 | "name": "stderr", 235 | "output_type": "stream", 236 | "text": [ 237 | "\n" 238 | ] 239 | } 240 | ], 241 | "source": [ 242 | "running_loss = []\n", 243 | "for epoch in tqdm(range(10)):\n", 244 | " total_loss = 0\n", 245 | " for context, target in data:\n", 246 | " context_idxs = np.array([word_to_ix[w] for w in context])\n", 247 | " target_idxs = np.array([word_to_ix[target]])\n", 248 | " loss = model_train(context_idxs, target_idxs)\n", 249 | " total_loss += loss\n", 250 | " running_loss.append(total_loss)\n", 251 | "print(running_loss)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": {}, 257 | "source": [ 258 | "The loss is pretty high but it does decrease anyway. The purpose of this notebook is to show how we can combine the low level TF ops with the high-level APIs to aid utmost flexibility in model definition. The example is inspired from [this tutorial](https://pytorch.org/tutorials/beginner/nlp/word_embeddings_tutorial.html). " 259 | ] 260 | } 261 | ], 262 | "metadata": { 263 | "kernelspec": { 264 | "display_name": "Python 3", 265 | "language": "python", 266 | "name": "python3" 267 | }, 268 | "language_info": { 269 | "codemirror_mode": { 270 | "name": "ipython", 271 | "version": 3 272 | }, 273 | "file_extension": ".py", 274 | "mimetype": "text/x-python", 275 | "name": "python", 276 | "nbconvert_exporter": "python", 277 | "pygments_lexer": "ipython3", 278 | "version": "3.7.1" 279 | } 280 | }, 281 | "nbformat": 4, 282 | "nbformat_minor": 2 283 | } 284 | -------------------------------------------------------------------------------- /Speed comparison between TF 1.x and TF 2.0/FashionMNIST_TF1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "FashionMNIST-TF1.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [] 9 | }, 10 | "kernelspec": { 11 | "name": "python3", 12 | "display_name": "Python 3" 13 | }, 14 | "accelerator": "GPU" 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "_OtBEJ_6JF_8", 21 | "colab_type": "text" 22 | }, 23 | "source": [ 24 | "## Install TensorFlow 1.14 (GPU variant) W&B" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "id": "TtMPhycuIf_N", 31 | "colab_type": "code", 32 | "colab": {} 33 | }, 34 | "source": [ 35 | "!pip install -q tensorflow-gpu==1.14.0\n", 36 | "!pip install wandb" 37 | ], 38 | "execution_count": 0, 39 | "outputs": [] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": { 44 | "id": "7kLomn0fJNFw", 45 | "colab_type": "text" 46 | }, 47 | "source": [ 48 | "## Imports and installation verification" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "metadata": { 54 | "id": "NHP9XxRMG4Sk", 55 | "colab_type": "code", 56 | "colab": {} 57 | }, 58 | "source": [ 59 | "from tensorflow.keras.layers import *\n", 60 | "from tensorflow.keras.models import *\n", 61 | "from wandb.keras import WandbCallback\n", 62 | "import tensorflow as tf\n", 63 | "import numpy as np\n", 64 | "import wandb\n", 65 | "import time" 66 | ], 67 | "execution_count": 0, 68 | "outputs": [] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "metadata": { 73 | "id": "DQ5n_n3oHNJ7", 74 | "colab_type": "code", 75 | "colab": {} 76 | }, 77 | "source": [ 78 | "tf.__version__" 79 | ], 80 | "execution_count": 0, 81 | "outputs": [] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": { 86 | "id": "wj4MoKGQMXhJ", 87 | "colab_type": "text" 88 | }, 89 | "source": [ 90 | "## W&B setup" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "metadata": { 96 | "id": "At8WmcQuMZK2", 97 | "colab_type": "code", 98 | "colab": {} 99 | }, 100 | "source": [ 101 | "!wandb login" 102 | ], 103 | "execution_count": 0, 104 | "outputs": [] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "metadata": { 109 | "id": "RpfaHrzUMz2m", 110 | "colab_type": "code", 111 | "colab": { 112 | "base_uri": "https://localhost:8080/", 113 | "height": 34 114 | }, 115 | "outputId": "6afb3def-7b0a-472a-d0d7-8905fe45ff0c" 116 | }, 117 | "source": [ 118 | "wandb.init()\n", 119 | "config = wandb.config\n", 120 | "\n", 121 | "config.batch_size = 128\n", 122 | "config.epochs=5" 123 | ], 124 | "execution_count": 4, 125 | "outputs": [ 126 | { 127 | "output_type": "display_data", 128 | "data": { 129 | "text/html": [ 130 | "\n", 131 | " Notebook configured with W&B. You can open the run page, or call %%wandb\n", 132 | " in a cell containing your training loop to display live results. Learn more in our docs.\n", 133 | " " 134 | ], 135 | "text/plain": [ 136 | "" 137 | ] 138 | }, 139 | "metadata": { 140 | "tags": [] 141 | } 142 | } 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "metadata": { 148 | "id": "6IkBUSIiJQD1", 149 | "colab_type": "text" 150 | }, 151 | "source": [ 152 | "## Data collection and preprocessing" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "metadata": { 158 | "id": "KeZuHzCYHX_H", 159 | "colab_type": "code", 160 | "colab": {} 161 | }, 162 | "source": [ 163 | "fashion_mnist = mnist = tf.keras.datasets.fashion_mnist\n", 164 | "\n", 165 | "(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\n", 166 | "x_train, x_test = x_train / 255.0, x_test / 255.0" 167 | ], 168 | "execution_count": 0, 169 | "outputs": [] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "metadata": { 174 | "id": "iCXnO-3VHv7X", 175 | "colab_type": "code", 176 | "colab": {} 177 | }, 178 | "source": [ 179 | "x_train = x_train.reshape((60000, 28, 28, 1))\n", 180 | "x_test = x_test.reshape((10000, 28, 28, 1))" 181 | ], 182 | "execution_count": 0, 183 | "outputs": [] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "metadata": { 188 | "id": "MG0aufQIJhwU", 189 | "colab_type": "code", 190 | "colab": { 191 | "base_uri": "https://localhost:8080/", 192 | "height": 34 193 | }, 194 | "outputId": "891c2984-b47e-48e0-bc05-171a32f2c272" 195 | }, 196 | "source": [ 197 | "x_train.shape, y_train.shape, x_test.shape, y_test.shape" 198 | ], 199 | "execution_count": 7, 200 | "outputs": [ 201 | { 202 | "output_type": "execute_result", 203 | "data": { 204 | "text/plain": [ 205 | "((60000, 28, 28, 1), (60000,), (10000, 28, 28, 1), (10000,))" 206 | ] 207 | }, 208 | "metadata": { 209 | "tags": [] 210 | }, 211 | "execution_count": 7 212 | } 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "metadata": { 218 | "id": "teQR1m3OJUAt", 219 | "colab_type": "text" 220 | }, 221 | "source": [ 222 | "## Model building and summarization" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "metadata": { 228 | "id": "t1XpwpThHdZd", 229 | "colab_type": "code", 230 | "colab": {} 231 | }, 232 | "source": [ 233 | "def create_model():\n", 234 | " model = Sequential([\n", 235 | " Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n", 236 | " MaxPooling2D((2,2)),\n", 237 | " Conv2D(64, (3, 3), activation='relu'),\n", 238 | " MaxPooling2D((2,2)),\n", 239 | " Conv2D(64, (3, 3), activation='relu')\n", 240 | " ])\n", 241 | " \n", 242 | " model.add(Flatten())\n", 243 | " model.add(Dense(64, activation='relu'))\n", 244 | " model.add(Dense(10, activation='softmax'))\n", 245 | "\n", 246 | " model.compile(optimizer='adam',\n", 247 | " loss='sparse_categorical_crossentropy',\n", 248 | " metrics=['accuracy'])\n", 249 | " \n", 250 | " return model" 251 | ], 252 | "execution_count": 0, 253 | "outputs": [] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "metadata": { 258 | "id": "CnsLABD3H61y", 259 | "colab_type": "code", 260 | "colab": { 261 | "base_uri": "https://localhost:8080/", 262 | "height": 496 263 | }, 264 | "outputId": "425c6564-8093-455d-9dfe-65bd75527bbd" 265 | }, 266 | "source": [ 267 | "model = create_model()\n", 268 | "model.summary()" 269 | ], 270 | "execution_count": 9, 271 | "outputs": [ 272 | { 273 | "output_type": "stream", 274 | "text": [ 275 | "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", 276 | "Instructions for updating:\n", 277 | "Call initializer instance with the dtype argument instead of passing it to the constructor\n", 278 | "Model: \"sequential\"\n", 279 | "_________________________________________________________________\n", 280 | "Layer (type) Output Shape Param # \n", 281 | "=================================================================\n", 282 | "conv2d (Conv2D) (None, 26, 26, 32) 320 \n", 283 | "_________________________________________________________________\n", 284 | "max_pooling2d (MaxPooling2D) (None, 13, 13, 32) 0 \n", 285 | "_________________________________________________________________\n", 286 | "conv2d_1 (Conv2D) (None, 11, 11, 64) 18496 \n", 287 | "_________________________________________________________________\n", 288 | "max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64) 0 \n", 289 | "_________________________________________________________________\n", 290 | "conv2d_2 (Conv2D) (None, 3, 3, 64) 36928 \n", 291 | "_________________________________________________________________\n", 292 | "flatten (Flatten) (None, 576) 0 \n", 293 | "_________________________________________________________________\n", 294 | "dense (Dense) (None, 64) 36928 \n", 295 | "_________________________________________________________________\n", 296 | "dense_1 (Dense) (None, 10) 650 \n", 297 | "=================================================================\n", 298 | "Total params: 93,322\n", 299 | "Trainable params: 93,322\n", 300 | "Non-trainable params: 0\n", 301 | "_________________________________________________________________\n" 302 | ], 303 | "name": "stdout" 304 | } 305 | ] 306 | }, 307 | { 308 | "cell_type": "markdown", 309 | "metadata": { 310 | "id": "fwyZ1nO0JYqu", 311 | "colab_type": "text" 312 | }, 313 | "source": [ 314 | "## Model training profiling" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "metadata": { 320 | "id": "10wVDoZtH_BR", 321 | "colab_type": "code", 322 | "colab": { 323 | "base_uri": "https://localhost:8080/", 324 | "height": 221 325 | }, 326 | "outputId": "77fb38a3-ecc6-4d8c-b7c3-3673f273cb68" 327 | }, 328 | "source": [ 329 | "%%prun\n", 330 | "model.fit(x_train, y_train, validation_data=(x_test, y_test),\n", 331 | " epochs=config.epochs, batch_size=config.batch_size, \n", 332 | " callbacks=[WandbCallback()], verbose=1)" 333 | ], 334 | "execution_count": 10, 335 | "outputs": [ 336 | { 337 | "output_type": "stream", 338 | "text": [ 339 | "Train on 60000 samples, validate on 10000 samples\n", 340 | "Epoch 1/5\n", 341 | "60000/60000 [==============================] - 54s 901us/sample - loss: 0.6179 - acc: 0.7748 - val_loss: 0.4529 - val_acc: 0.8347\n", 342 | "Epoch 2/5\n", 343 | "60000/60000 [==============================] - 54s 908us/sample - loss: 0.3854 - acc: 0.8613 - val_loss: 0.3699 - val_acc: 0.8687\n", 344 | "Epoch 3/5\n", 345 | "60000/60000 [==============================] - 56s 930us/sample - loss: 0.3329 - acc: 0.8796 - val_loss: 0.3479 - val_acc: 0.8723\n", 346 | "Epoch 4/5\n", 347 | "60000/60000 [==============================] - 54s 898us/sample - loss: 0.3000 - acc: 0.8910 - val_loss: 0.3210 - val_acc: 0.8848\n", 348 | "Epoch 5/5\n", 349 | "60000/60000 [==============================] - 54s 896us/sample - loss: 0.2775 - acc: 0.8982 - val_loss: 0.3028 - val_acc: 0.8934\n", 350 | " " 351 | ], 352 | "name": "stdout" 353 | } 354 | ] 355 | }, 356 | { 357 | "cell_type": "markdown", 358 | "metadata": { 359 | "id": "VSE063gqOqHX", 360 | "colab_type": "text" 361 | }, 362 | "source": [ 363 | "Here are the major logs from the profiling:\n", 364 | "\n", 365 | "```\n", 366 | "3326560 function calls (3256171 primitive calls) in 272.912 seconds\n", 367 | "\n", 368 | " Ordered by: internal time\n", 369 | " ncalls tottime percall cumtime percall filename:lineno(function)\n", 370 | " 2740 259.207 0.095 259.207 0.095 {built-in method _pywrap_tensorflow_internal.TF_SessionRunCallable}\n", 371 | " 31609 2.641 0.000 2.641 0.000 {method 'acquire' of '_thread.lock' objects}\n", 372 | " 2740 1.945 0.001 1.945 0.001 generic_utils.py:531()\n", 373 | " 34989 0.777 0.000 0.778 0.000 {built-in method numpy.array}\n", 374 | " 16429 0.558 0.000 0.558 0.000 socket.py:333(send)\n", 375 | " 6/1 0.451 0.075 272.567 272.567 training_arrays.py:45(model_iteration)\n", 376 | " 2740 0.313 0.000 262.203 0.096 backend.py:3250(__call__)\n", 377 | " 2345 0.273 0.000 4.382 0.002 generic_utils.py:353(update)\n", 378 | " ```" 379 | ] 380 | }, 381 | { 382 | "cell_type": "markdown", 383 | "metadata": { 384 | "id": "e_TyH93aSLmg", 385 | "colab_type": "text" 386 | }, 387 | "source": [ 388 | "CPU and Memory profilings can be found [here](https://app.wandb.ai/sayakpaul/uncategorized/runs/5d54sfb1). Let's now do the *exact* same thing with TensorFlow 2.0. " 389 | ] 390 | } 391 | ] 392 | } -------------------------------------------------------------------------------- /Speed comparison between TF 1.x and TF 2.0/FashionMNIST_TF2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "FashionMNIST_TF2.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [] 9 | }, 10 | "kernelspec": { 11 | "name": "python3", 12 | "display_name": "Python 3" 13 | }, 14 | "accelerator": "GPU" 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "_OtBEJ_6JF_8", 21 | "colab_type": "text" 22 | }, 23 | "source": [ 24 | "## Install TensorFlow 2.0 (GPU variant) W&B" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "id": "TtMPhycuIf_N", 31 | "colab_type": "code", 32 | "colab": {} 33 | }, 34 | "source": [ 35 | "!pip install -q tensorflow-gpu\n", 36 | "!pip install wandb" 37 | ], 38 | "execution_count": 0, 39 | "outputs": [] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": { 44 | "id": "7kLomn0fJNFw", 45 | "colab_type": "text" 46 | }, 47 | "source": [ 48 | "## Imports and installation verification" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "metadata": { 54 | "id": "NHP9XxRMG4Sk", 55 | "colab_type": "code", 56 | "colab": {} 57 | }, 58 | "source": [ 59 | "from tensorflow.keras.layers import *\n", 60 | "from tensorflow.keras.models import *\n", 61 | "from wandb.keras import WandbCallback\n", 62 | "import tensorflow as tf\n", 63 | "import numpy as np\n", 64 | "import wandb\n", 65 | "import time" 66 | ], 67 | "execution_count": 0, 68 | "outputs": [] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "metadata": { 73 | "id": "DQ5n_n3oHNJ7", 74 | "colab_type": "code", 75 | "colab": { 76 | "base_uri": "https://localhost:8080/", 77 | "height": 34 78 | }, 79 | "outputId": "cb704dcc-cbb6-4f43-f405-97b7ea1dcfc0" 80 | }, 81 | "source": [ 82 | "tf.__version__" 83 | ], 84 | "execution_count": 2, 85 | "outputs": [ 86 | { 87 | "output_type": "execute_result", 88 | "data": { 89 | "text/plain": [ 90 | "'2.0.0'" 91 | ] 92 | }, 93 | "metadata": { 94 | "tags": [] 95 | }, 96 | "execution_count": 2 97 | } 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": { 103 | "id": "wj4MoKGQMXhJ", 104 | "colab_type": "text" 105 | }, 106 | "source": [ 107 | "## W&B setup" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "metadata": { 113 | "id": "At8WmcQuMZK2", 114 | "colab_type": "code", 115 | "colab": {} 116 | }, 117 | "source": [ 118 | "!wandb login" 119 | ], 120 | "execution_count": 0, 121 | "outputs": [] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "metadata": { 126 | "id": "RpfaHrzUMz2m", 127 | "colab_type": "code", 128 | "colab": {} 129 | }, 130 | "source": [ 131 | "wandb.init()\n", 132 | "config = wandb.config\n", 133 | "\n", 134 | "config.batch_size = 128\n", 135 | "config.epochs=5" 136 | ], 137 | "execution_count": 0, 138 | "outputs": [] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": { 143 | "id": "6IkBUSIiJQD1", 144 | "colab_type": "text" 145 | }, 146 | "source": [ 147 | "## Data collection and preprocessing" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "metadata": { 153 | "id": "KeZuHzCYHX_H", 154 | "colab_type": "code", 155 | "colab": {} 156 | }, 157 | "source": [ 158 | "fashion_mnist = mnist = tf.keras.datasets.fashion_mnist\n", 159 | "\n", 160 | "(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\n", 161 | "x_train, x_test = x_train / 255.0, x_test / 255.0" 162 | ], 163 | "execution_count": 0, 164 | "outputs": [] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "metadata": { 169 | "id": "iCXnO-3VHv7X", 170 | "colab_type": "code", 171 | "colab": {} 172 | }, 173 | "source": [ 174 | "x_train = x_train.reshape((60000, 28, 28, 1))\n", 175 | "x_test = x_test.reshape((10000, 28, 28, 1))" 176 | ], 177 | "execution_count": 0, 178 | "outputs": [] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "metadata": { 183 | "id": "MG0aufQIJhwU", 184 | "colab_type": "code", 185 | "outputId": "b3d6bc46-3c96-48e1-f344-b5fca9c59c49", 186 | "colab": { 187 | "base_uri": "https://localhost:8080/", 188 | "height": 34 189 | } 190 | }, 191 | "source": [ 192 | "x_train.shape, y_train.shape, x_test.shape, y_test.shape" 193 | ], 194 | "execution_count": 7, 195 | "outputs": [ 196 | { 197 | "output_type": "execute_result", 198 | "data": { 199 | "text/plain": [ 200 | "((60000, 28, 28, 1), (60000,), (10000, 28, 28, 1), (10000,))" 201 | ] 202 | }, 203 | "metadata": { 204 | "tags": [] 205 | }, 206 | "execution_count": 7 207 | } 208 | ] 209 | }, 210 | { 211 | "cell_type": "markdown", 212 | "metadata": { 213 | "id": "teQR1m3OJUAt", 214 | "colab_type": "text" 215 | }, 216 | "source": [ 217 | "## Model building and summarization" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "metadata": { 223 | "id": "t1XpwpThHdZd", 224 | "colab_type": "code", 225 | "colab": {} 226 | }, 227 | "source": [ 228 | "def create_model():\n", 229 | " model = Sequential([\n", 230 | " Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n", 231 | " MaxPooling2D((2,2)),\n", 232 | " Conv2D(64, (3, 3), activation='relu'),\n", 233 | " MaxPooling2D((2,2)),\n", 234 | " Conv2D(64, (3, 3), activation='relu')\n", 235 | " ])\n", 236 | " \n", 237 | " model.add(Flatten())\n", 238 | " model.add(Dense(64, activation='relu'))\n", 239 | " model.add(Dense(10, activation='softmax'))\n", 240 | "\n", 241 | " model.compile(optimizer='adam',\n", 242 | " loss='sparse_categorical_crossentropy',\n", 243 | " metrics=['accuracy'])\n", 244 | " \n", 245 | " return model" 246 | ], 247 | "execution_count": 0, 248 | "outputs": [] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "metadata": { 253 | "id": "CnsLABD3H61y", 254 | "colab_type": "code", 255 | "outputId": "f9138472-3452-463b-f948-c0ebaafdc93f", 256 | "colab": { 257 | "base_uri": "https://localhost:8080/", 258 | "height": 425 259 | } 260 | }, 261 | "source": [ 262 | "model = create_model()\n", 263 | "model.summary()" 264 | ], 265 | "execution_count": 9, 266 | "outputs": [ 267 | { 268 | "output_type": "stream", 269 | "text": [ 270 | "Model: \"sequential\"\n", 271 | "_________________________________________________________________\n", 272 | "Layer (type) Output Shape Param # \n", 273 | "=================================================================\n", 274 | "conv2d (Conv2D) (None, 26, 26, 32) 320 \n", 275 | "_________________________________________________________________\n", 276 | "max_pooling2d (MaxPooling2D) (None, 13, 13, 32) 0 \n", 277 | "_________________________________________________________________\n", 278 | "conv2d_1 (Conv2D) (None, 11, 11, 64) 18496 \n", 279 | "_________________________________________________________________\n", 280 | "max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64) 0 \n", 281 | "_________________________________________________________________\n", 282 | "conv2d_2 (Conv2D) (None, 3, 3, 64) 36928 \n", 283 | "_________________________________________________________________\n", 284 | "flatten (Flatten) (None, 576) 0 \n", 285 | "_________________________________________________________________\n", 286 | "dense (Dense) (None, 64) 36928 \n", 287 | "_________________________________________________________________\n", 288 | "dense_1 (Dense) (None, 10) 650 \n", 289 | "=================================================================\n", 290 | "Total params: 93,322\n", 291 | "Trainable params: 93,322\n", 292 | "Non-trainable params: 0\n", 293 | "_________________________________________________________________\n" 294 | ], 295 | "name": "stdout" 296 | } 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": { 302 | "id": "fwyZ1nO0JYqu", 303 | "colab_type": "text" 304 | }, 305 | "source": [ 306 | "## Model training profiling" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "metadata": { 312 | "id": "10wVDoZtH_BR", 313 | "colab_type": "code", 314 | "outputId": "5ced439c-f940-4f66-c5c8-4501953b1a87", 315 | "colab": { 316 | "base_uri": "https://localhost:8080/", 317 | "height": 221 318 | } 319 | }, 320 | "source": [ 321 | "%%prun\n", 322 | "model.fit(x_train, y_train, validation_data=(x_test, y_test),\n", 323 | " epochs=config.epochs, batch_size=config.batch_size, \n", 324 | " callbacks=[WandbCallback()], verbose=1)" 325 | ], 326 | "execution_count": 10, 327 | "outputs": [ 328 | { 329 | "output_type": "stream", 330 | "text": [ 331 | "Train on 60000 samples, validate on 10000 samples\n", 332 | "Epoch 1/5\n", 333 | "60000/60000 [==============================] - 10s 163us/sample - loss: 0.6141 - accuracy: 0.7766 - val_loss: 0.4247 - val_accuracy: 0.8486\n", 334 | "Epoch 2/5\n", 335 | "60000/60000 [==============================] - 5s 89us/sample - loss: 0.3814 - accuracy: 0.8624 - val_loss: 0.3493 - val_accuracy: 0.8777\n", 336 | "Epoch 3/5\n", 337 | "60000/60000 [==============================] - 5s 89us/sample - loss: 0.3232 - accuracy: 0.8842 - val_loss: 0.3192 - val_accuracy: 0.8857\n", 338 | "Epoch 4/5\n", 339 | "60000/60000 [==============================] - 5s 88us/sample - loss: 0.2914 - accuracy: 0.8941 - val_loss: 0.3027 - val_accuracy: 0.8943\n", 340 | "Epoch 5/5\n", 341 | "60000/60000 [==============================] - 5s 88us/sample - loss: 0.2666 - accuracy: 0.9037 - val_loss: 0.2870 - val_accuracy: 0.8965\n", 342 | " " 343 | ], 344 | "name": "stdout" 345 | } 346 | ] 347 | }, 348 | { 349 | "cell_type": "markdown", 350 | "metadata": { 351 | "id": "VSE063gqOqHX", 352 | "colab_type": "text" 353 | }, 354 | "source": [ 355 | "Here are the major logs from the profiling:\n", 356 | "\n", 357 | "```\n", 358 | "\n", 359 | " 4766351 function calls (4456966 primitive calls) in 31.667 seconds\n", 360 | "\n", 361 | " Ordered by: internal time\n", 362 | "\n", 363 | " ncalls tottime percall cumtime percall filename:lineno(function)\n", 364 | " 2755 16.461 0.006 16.461 0.006 {built-in method _pywrap_tensorflow_internal.TFE_Py_Execute}\n", 365 | " 8396 8.319 0.001 8.319 0.001 {method '_numpy' of 'tensorflow.python.framework.ops.EagerTensor' objects}\n", 366 | " 10452 0.724 0.000 0.724 0.000 {method 'acquire' of '_thread.lock' objects}\n", 367 | " 3268 0.345 0.000 0.345 0.000 socket.py:333(send)\n", 368 | "598631/598365 0.209 0.000 0.425 0.000 {built-in method builtins.isinstance}\n", 369 | " 721 0.193 0.000 0.204 0.000 tf_stack.py:141(extract_stack)\n", 370 | " 116188 0.126 0.000 0.233 0.000 abc.py:180(__instancecheck__)\n", 371 | " 22054 0.122 0.000 0.221 0.000 nest.py:117(_sequence_like)\n", 372 | "87712/21928 0.117 0.000 0.363 0.000 type_spec.py:311(__make_cmp_key)\n", 373 | " 23516 0.099 0.000 0.099 0.000 {built-in method numpy.array}\n", 374 | " 14829 0.099 0.000 0.113 0.000 {built-in method _pywrap_tensorflow_internal.Flatten}\n", 375 | "22054/5593 0.093 0.000 0.589 0.000 nest.py:384(_packed_nest_with_indices)\n", 376 | " 369 0.089 0.000 0.089 0.000 {built-in method _pywrap_tensorflow_internal.TFE_Py_FastPathExecute}\n", 377 | " 187793 0.086 0.000 0.086 0.000 _weakrefset.py:70(__contains__)\n", 378 | " 2741 0.085 0.000 17.175 0.006 function.py:1143(_call_flat)\n", 379 | " 2345 0.082 0.000 0.930 0.000 generic_utils.py:351(update)\n", 380 | " ```" 381 | ] 382 | }, 383 | { 384 | "cell_type": "markdown", 385 | "metadata": { 386 | "id": "e_TyH93aSLmg", 387 | "colab_type": "text" 388 | }, 389 | "source": [ 390 | "CPU and Memory profilings can be found [here](https://app.wandb.ai/sayakpaul/uncategorized/runs/xukoozma). TensorFlow 2.0 is indeed faster because of the fact that Eager Execution happens here by default. " 391 | ] 392 | } 393 | ] 394 | } -------------------------------------------------------------------------------- /Speed comparison between TF 1.x and TF 2.0/README.md: -------------------------------------------------------------------------------- 1 | In this mini-project, I present how eager execution can really be helpful in speeding up the model training process. 2 | 3 | Steps I followed to conduct the experiments: 4 | - I maintained the exact same environment, model configuration, dataset (FashionMNIST) for the experiments. I only changed the TensorFlow versions. 5 | - I ran thorough profiling to check what really causes execution in TensorFlow 1.14 to be slow and I found out it was _Sessions_. 6 | 7 | Apart from these, I used [Weights and Biases](https://wandb.com) to log the CPU usage and memory footprints of the experiments. I was amazed to find out that TensorFlow 2.0 was much more performant in terms of CPU usage as well. Here are some snaps: 8 | 9 | ![](https://i.ibb.co/D9VcssK/Screen-Shot-2019-10-03-at-12-12-03-PM.png) 10 | 11 | ![](https://i.ibb.co/5vPTfwj/Screen-Shot-2019-10-03-at-12-13-21-PM.png) 12 | 13 | -------------------------------------------------------------------------------- /TF_2_0_and_cloud_functions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "TF 2.0 and cloud functions.ipynb", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "id": "view-in-github", 23 | "colab_type": "text" 24 | }, 25 | "source": [ 26 | "\"Open" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": { 32 | "id": "LDc_vepL86LA", 33 | "colab_type": "text" 34 | }, 35 | "source": [ 36 | "The purpose of this notebook is to show how easy it is to serve a machine learning model via [Cloud Functions](https://console.cloud.google.com/functions/) on the Google Cloud Platform. It is absolutely possible to do this via Colab. In this notebook, we will be\n", 37 | "- building a simple neural network model to classify the apparels as listed in the FashionMNIST dataset\n", 38 | "- serializing the model weights in a way that is compatible with the Cloud Functions' ecosystem\n", 39 | "- using the `gcloud` CLI to deploy our model on GCP via Cloud Functions\n", 40 | "\n", 41 | "So, let's get started. " 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "metadata": { 47 | "id": "xW-MxeH4kTld", 48 | "colab_type": "code", 49 | "colab": {} 50 | }, 51 | "source": [ 52 | "# install `tensorflow` 2.0 latest\n", 53 | "pip install tensorflow==2.0.0b1" 54 | ], 55 | "execution_count": 0, 56 | "outputs": [] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "metadata": { 61 | "id": "U5-crVLxkiPN", 62 | "colab_type": "code", 63 | "colab": { 64 | "base_uri": "https://localhost:8080/", 65 | "height": 36 66 | }, 67 | "outputId": "d2f8eefc-834c-4e3f-f041-a0e329ba3cc8" 68 | }, 69 | "source": [ 70 | "# import tensorflow and verify the version\n", 71 | "import tensorflow as tf\n", 72 | "\n", 73 | "print(tf.__version__)" 74 | ], 75 | "execution_count": 2, 76 | "outputs": [ 77 | { 78 | "output_type": "stream", 79 | "text": [ 80 | "2.0.0-beta1\n" 81 | ], 82 | "name": "stdout" 83 | } 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "metadata": { 89 | "id": "3qPW76p9k5nq", 90 | "colab_type": "code", 91 | "colab": {} 92 | }, 93 | "source": [ 94 | "# all the imports we care about in this notebook\n", 95 | "import matplotlib.pyplot as plt\n", 96 | "from tensorflow.keras.layers import *\n", 97 | "from tensorflow.keras.models import *" 98 | ], 99 | "execution_count": 0, 100 | "outputs": [] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "metadata": { 105 | "id": "ZbcZvU-PlLef", 106 | "colab_type": "code", 107 | "colab": {} 108 | }, 109 | "source": [ 110 | "# load and prepare our data\n", 111 | "fashion_mnist = mnist = tf.keras.datasets.fashion_mnist\n", 112 | "\n", 113 | "(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\n", 114 | "x_train, x_test = x_train / 255.0, x_test / 255.0" 115 | ], 116 | "execution_count": 0, 117 | "outputs": [] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "metadata": { 122 | "id": "e4qwoUwcmbMG", 123 | "colab_type": "code", 124 | "colab": {} 125 | }, 126 | "source": [ 127 | "# the humble model\n", 128 | "model = tf.keras.models.Sequential([\n", 129 | " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", 130 | " tf.keras.layers.Dense(128, activation='relu'),\n", 131 | " tf.keras.layers.Dropout(0.2),\n", 132 | " tf.keras.layers.Dense(10, activation='softmax')\n", 133 | "])\n", 134 | "\n", 135 | "model.compile(optimizer='adam',\n", 136 | " loss='sparse_categorical_crossentropy',\n", 137 | " metrics=['accuracy'])" 138 | ], 139 | "execution_count": 0, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "metadata": { 145 | "id": "ShG3bxScmfdn", 146 | "colab_type": "code", 147 | "colab": { 148 | "base_uri": "https://localhost:8080/", 149 | "height": 341 150 | }, 151 | "outputId": "3fe5fd1c-bc12-4a3e-d43d-97a26e26e55c" 152 | }, 153 | "source": [ 154 | "# kickstart the training\n", 155 | "model.fit(x_train, y_train, validation_data=(x_test, y_test), \n", 156 | " epochs=5, batch_size=128,\n", 157 | " verbose=1)\n" 158 | ], 159 | "execution_count": 6, 160 | "outputs": [ 161 | { 162 | "output_type": "stream", 163 | "text": [ 164 | "WARNING: Logging before flag parsing goes to stderr.\n", 165 | "W0711 15:16:04.927937 140034585638784 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support..wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n", 166 | "Instructions for updating:\n", 167 | "Use tf.where in 2.0, which has the same broadcast rule as np.where\n" 168 | ], 169 | "name": "stderr" 170 | }, 171 | { 172 | "output_type": "stream", 173 | "text": [ 174 | "Train on 60000 samples, validate on 10000 samples\n", 175 | "Epoch 1/5\n", 176 | "60000/60000 [==============================] - 3s 42us/sample - loss: 0.5935 - accuracy: 0.7950 - val_loss: 0.4510 - val_accuracy: 0.8406\n", 177 | "Epoch 2/5\n", 178 | "60000/60000 [==============================] - 2s 37us/sample - loss: 0.4178 - accuracy: 0.8515 - val_loss: 0.4079 - val_accuracy: 0.8559\n", 179 | "Epoch 3/5\n", 180 | "60000/60000 [==============================] - 2s 37us/sample - loss: 0.3810 - accuracy: 0.8625 - val_loss: 0.3864 - val_accuracy: 0.8592\n", 181 | "Epoch 4/5\n", 182 | "60000/60000 [==============================] - 2s 36us/sample - loss: 0.3578 - accuracy: 0.8703 - val_loss: 0.3728 - val_accuracy: 0.8656\n", 183 | "Epoch 5/5\n", 184 | "60000/60000 [==============================] - 2s 37us/sample - loss: 0.3388 - accuracy: 0.8773 - val_loss: 0.3660 - val_accuracy: 0.8660\n" 185 | ], 186 | "name": "stdout" 187 | }, 188 | { 189 | "output_type": "execute_result", 190 | "data": { 191 | "text/plain": [ 192 | "" 193 | ] 194 | }, 195 | "metadata": { 196 | "tags": [] 197 | }, 198 | "execution_count": 6 199 | } 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "metadata": { 205 | "id": "4UAxuKOzmxCM", 206 | "colab_type": "code", 207 | "colab": {} 208 | }, 209 | "source": [ 210 | "# save the weights\n", 211 | "model.save_weights('fashion_mnist_weights')" 212 | ], 213 | "execution_count": 0, 214 | "outputs": [] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": { 219 | "id": "e1JbsSZZAPXy", 220 | "colab_type": "text" 221 | }, 222 | "source": [ 223 | "This will give birth to two files:\n", 224 | "- fashion_mnist_weights.data-00000-of-00001\n", 225 | "- fashion_mnist_weights.index" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "metadata": { 231 | "id": "egqd7uhvx5Wd", 232 | "colab_type": "code", 233 | "colab": { 234 | "base_uri": "https://localhost:8080/", 235 | "height": 36 236 | }, 237 | "outputId": "d8b551d1-a7d3-48f6-827a-65180fb1ee55" 238 | }, 239 | "source": [ 240 | "# sample prediction\n", 241 | "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", 242 | " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']\n", 243 | "test_img = plt.imread('test.png')\n", 244 | "prob = model.predict(test_img.reshape(1, 28, 28))\n", 245 | "print(class_names[prob.argmax()])" 246 | ], 247 | "execution_count": 17, 248 | "outputs": [ 249 | { 250 | "output_type": "stream", 251 | "text": [ 252 | "Trouser\n" 253 | ], 254 | "name": "stdout" 255 | } 256 | ] 257 | }, 258 | { 259 | "cell_type": "markdown", 260 | "metadata": { 261 | "id": "CVtyUV5u-BSw", 262 | "colab_type": "text" 263 | }, 264 | "source": [ 265 | "The test image looks like the following, by the way:\n", 266 | "\n", 267 | "![](https://storage.googleapis.com/gweb-cloudblog-publish/images/Cloud_Storage_test_image.max-100x100.png)" 268 | ] 269 | }, 270 | { 271 | "cell_type": "markdown", 272 | "metadata": { 273 | "id": "wInfsoDw-NJP", 274 | "colab_type": "text" 275 | }, 276 | "source": [ 277 | "Once the model weights are saved we need to create a `.py` file named `main.py` as required by Cloud Functions. The `main.py` file should look like so:\n", 278 | "\n", 279 | "```python\n", 280 | "import numpy\n", 281 | "import tensorflow as tf\n", 282 | "\n", 283 | "from google.cloud import storage\n", 284 | "from tensorflow.keras.layers import *\n", 285 | "from tensorflow.keras.models import *\n", 286 | "from PIL import Image\n", 287 | "\n", 288 | "\n", 289 | "# we keep model as global variable so we don't have to reload it \n", 290 | "# in case of warm invocations\n", 291 | "model = None\n", 292 | "\n", 293 | "def get_me_the_model():\n", 294 | " model = tf.keras.models.Sequential([\n", 295 | " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", 296 | " tf.keras.layers.Dense(128, activation='relu'),\n", 297 | " tf.keras.layers.Dropout(0.2),\n", 298 | " tf.keras.layers.Dense(10, activation='softmax')\n", 299 | "])\n", 300 | "\n", 301 | " model.compile(optimizer='adam',\n", 302 | " loss='sparse_categorical_crossentropy',\n", 303 | " metrics=['accuracy'])\n", 304 | "\n", 305 | " return model\n", 306 | "\n", 307 | "def download_blob(bucket_name, source_blob_name, destination_file_name):\n", 308 | " \"\"\"downloads a blob from the bucket.\"\"\"\n", 309 | " storage_client = storage.Client()\n", 310 | " bucket = storage_client.get_bucket(bucket_name)\n", 311 | " blob = bucket.blob(source_blob_name)\n", 312 | "\n", 313 | " blob.download_to_filename(destination_file_name)\n", 314 | "\n", 315 | " print('Blob {} downloaded to {}.'.format(\n", 316 | " source_blob_name,\n", 317 | " destination_file_name))\n", 318 | "\n", 319 | "def handler(request):\n", 320 | " global model\n", 321 | " class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", 322 | " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']\n", 323 | "\n", 324 | " # model load which only happens during cold invocations\n", 325 | " if model is None:\n", 326 | " download_blob('', 'tensorflow/fashion_mnist_weights.index', '/tmp/fashion_mnist_weights.index')\n", 327 | " download_blob('', 'tensorflow/fashion_mnist_weights.data-00000-of-00001', '/tmp/fashion_mnist_weights.data-00000-of-00001')\n", 328 | " model = get_me_the_model()\n", 329 | " model.load_weights('/tmp/fashion_mnist_weights')\n", 330 | " \n", 331 | " download_blob('', 'tensorflow/test.png', '/tmp/test.png')\n", 332 | " image = numpy.array(Image.open('/tmp/test.png'))\n", 333 | " input_np = (numpy.array(Image.open('/tmp/test.png'))/255)\n", 334 | " input_np = input_np.reshape(1, 28, 28)\n", 335 | " predictions = model.predict(input_np)\n", 336 | " print(predictions)\n", 337 | " print(\"Image is \"+class_names[numpy.argmax(predictions)])\n", 338 | " \n", 339 | " return class_names[numpy.argmax(predictions)]\n", 340 | " ```" 341 | ] 342 | }, 343 | { 344 | "cell_type": "markdown", 345 | "metadata": { 346 | "id": "kxzhkJAK_w3A", 347 | "colab_type": "text" 348 | }, 349 | "source": [ 350 | "**Note** that in place of `` enter the bucket's name (without `gs://`) in which you have stored the model weights. Also note that, I have stored them in a folder named **tensorflow**. When the model is deployed as a cloud function, `main.py`will download the model from the storage bucket and will store it into **tmp** folder. \n", 351 | "\n", 352 | "Now to get started with the deployment process, first authenticate yourself. " 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "metadata": { 358 | "id": "LRTGladR4GV0", 359 | "colab_type": "code", 360 | "colab": {} 361 | }, 362 | "source": [ 363 | "!gcloud auth login" 364 | ], 365 | "execution_count": 0, 366 | "outputs": [] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "metadata": { 371 | "id": "qKPeeELHBLj7", 372 | "colab_type": "text" 373 | }, 374 | "source": [ 375 | "Set the GCP project (preferably billing enabled)." 376 | ] 377 | }, 378 | { 379 | "cell_type": "code", 380 | "metadata": { 381 | "id": "bkdGl7wf4eJz", 382 | "colab_type": "code", 383 | "colab": {} 384 | }, 385 | "source": [ 386 | "!gcloud config set project fast-ai-exploration" 387 | ], 388 | "execution_count": 0, 389 | "outputs": [] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": { 394 | "id": "WJ65TFI5BTlr", 395 | "colab_type": "text" 396 | }, 397 | "source": [ 398 | "And deploy!" 399 | ] 400 | }, 401 | { 402 | "cell_type": "code", 403 | "metadata": { 404 | "id": "ilzIcfKu4BRC", 405 | "colab_type": "code", 406 | "colab": { 407 | "base_uri": "https://localhost:8080/", 408 | "height": 341 409 | }, 410 | "outputId": "0db88533-9776-4512-b607-196b53858bdb" 411 | }, 412 | "source": [ 413 | "!gcloud functions deploy handler --runtime python37 --trigger-http --memory 2048\n", 414 | "!gcloud functions call handler" 415 | ], 416 | "execution_count": 23, 417 | "outputs": [ 418 | { 419 | "output_type": "stream", 420 | "text": [ 421 | "availableMemoryMb: 2048\n", 422 | "entryPoint: handler\n", 423 | "httpsTrigger:\n", 424 | " url: https://us-central1-fast-ai-exploration.cloudfunctions.net/handler\n", 425 | "labels:\n", 426 | " deployment-tool: cli-gcloud\n", 427 | "name: projects/fast-ai-exploration/locations/us-central1/functions/handler\n", 428 | "runtime: python37\n", 429 | "serviceAccountEmail: fast-ai-exploration@appspot.gserviceaccount.com\n", 430 | "sourceUploadUrl: https://storage.googleapis.com/gcf-upload-us-central1-9df9b855-cdbd-46fb-8ddd-787a7047f525/e641be16-f3ca-4d53-b188-8108a000fa07.zip?GoogleAccessId=service-29880397572@gcf-admin-robot.iam.gserviceaccount.com&Expires=1562864691&Signature=n%2F%2F4sB55%2FYRNY%2FpmXQ7PH1wrABZKwpz6XcOlDSBcsauPdciHebBywXv%2FKTvvV9O04tUT%2BrvjXvdfyweMSMcKtHyPYfyaBWIxxlPW89NHEZtzvgQ1l5dGHQv7BNdOqMkbnykFoNAyCbxr8oK5COJaOAXTRbrWR%2FRfVuo9FA6aJnNNqvQAwGbzmrlesyGqcZ0bYNMOOZf2MySjJ1cypDr8UNwuxls2HntAo0yLjyCpDvTynfHKyVlMwn1SegvxOixYZ14lpH%2B8rHfnjWCY9eyZ2cfjcVsx%2FBHZQNgXVT5tST0zA28V0b13Fh1Zkr%2B9EmwCzzVrLeugqmC%2BCZWuLIK%2B%2Fg%3D%3D\n", 431 | "status: ACTIVE\n", 432 | "timeout: 60s\n", 433 | "updateTime: '2019-07-11T16:37:58Z'\n", 434 | "versionId: '1'\n", 435 | "executionId: gpn5hqhg1rab\n", 436 | "result: Trouser\n" 437 | ], 438 | "name": "stdout" 439 | } 440 | ] 441 | }, 442 | { 443 | "cell_type": "markdown", 444 | "metadata": { 445 | "id": "AyB6f1h3BW52", 446 | "colab_type": "text" 447 | }, 448 | "source": [ 449 | "**Notice** that the function `handler()` in `main.py` internally calls the test image, so you don't need to worry about it. " 450 | ] 451 | }, 452 | { 453 | "cell_type": "markdown", 454 | "metadata": { 455 | "id": "Gc9tgcm8BmJj", 456 | "colab_type": "text" 457 | }, 458 | "source": [ 459 | "**Reference**:\n", 460 | "- https://cloud.google.com/blog/products/ai-machine-learning/how-to-serve-deep-learning-models-using-tensorflow-2-0-with-cloud-functions" 461 | ] 462 | } 463 | ] 464 | } -------------------------------------------------------------------------------- /data2vec_vision_image_classification.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "data2vec-vision-image-classification.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyPEmUxVC6HNz8s8TjfpN6cJ", 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | }, 16 | "language_info": { 17 | "name": "python" 18 | }, 19 | "accelerator": "GPU" 20 | }, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "id": "view-in-github", 26 | "colab_type": "text" 27 | }, 28 | "source": [ 29 | "\"Open" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "source": [ 35 | "In this notebook, we'll fine-tune a model pre-trained using data2vec. We'll leverage the `transformers` library for loading the pre-trained model and we'll then append a custom classification head for fine-tuning. For data handling we'll use `tf.data`.\n", 36 | "\n", 37 | "Checkout the data2vec paper [here](https://arxiv.org/abs/2202.03555) and the model documentation page [here](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec)." 38 | ], 39 | "metadata": { 40 | "id": "uP9zYDHBxN7V" 41 | } 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "source": [ 46 | "## Setup" 47 | ], 48 | "metadata": { 49 | "id": "RJOiIwQix0y2" 50 | } 51 | }, 52 | { 53 | "cell_type": "code", 54 | "source": [ 55 | "# Otherwise TF's data2vec vision model won't be available.\n", 56 | "# Date: May 06, 2022\n", 57 | "!pip install git+https://github.com/huggingface/transformers -q" 58 | ], 59 | "metadata": { 60 | "id": "SDfJH5HmfpWF" 61 | }, 62 | "execution_count": null, 63 | "outputs": [] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "source": [ 68 | "## Imports" 69 | ], 70 | "metadata": { 71 | "id": "rNoyeQxdx2SK" 72 | } 73 | }, 74 | { 75 | "cell_type": "code", 76 | "source": [ 77 | "import tensorflow as tf\n", 78 | "from tensorflow import keras \n", 79 | "from tensorflow.keras import layers\n", 80 | "\n", 81 | "from transformers import TFAutoModelForImageClassification, TFData2VecVisionModel\n", 82 | "from transformers import create_optimizer\n", 83 | "\n", 84 | "import tensorflow_datasets as tfds\n", 85 | "\n", 86 | "tfds.disable_progress_bar()\n", 87 | "\n", 88 | "import pandas as pd\n", 89 | "import numpy as np\n", 90 | "import matplotlib.pyplot as plt" 91 | ], 92 | "metadata": { 93 | "id": "3BGFcoWPf3ht" 94 | }, 95 | "execution_count": null, 96 | "outputs": [] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "source": [ 101 | "## Primary constants" 102 | ], 103 | "metadata": { 104 | "id": "cM4gh8Ubx3nV" 105 | } 106 | }, 107 | { 108 | "cell_type": "code", 109 | "source": [ 110 | "# Model\n", 111 | "IMAGE_SIZE = [224, 224]\n", 112 | "MODEL_ID = \"facebook/data2vec-vision-base\" # pre-trained model from which to fine-tune\n", 113 | "\n", 114 | "# Data\n", 115 | "BATCH_SIZE = 16 # on Colab/GPU, a higher batch size may throw(OOM)\n", 116 | "\n", 117 | "# Dataset\n", 118 | "CLASSES = [\n", 119 | " \"dandelion\",\n", 120 | " \"daisy\",\n", 121 | " \"tulips\",\n", 122 | " \"sunflowers\",\n", 123 | " \"roses\",\n", 124 | "] # don't change the order\n", 125 | "\n", 126 | "# Other constants\n", 127 | "MEAN = tf.constant([0.485 * 255, 0.456 * 255, 0.406 * 255]) # imagenet mean\n", 128 | "STD = tf.constant([0.229 * 255, 0.224 * 255, 0.225 * 255]) # imagenet std\n", 129 | "AUTO = tf.data.AUTOTUNE" 130 | ], 131 | "metadata": { 132 | "id": "cJ9uU4iMgJFF" 133 | }, 134 | "execution_count": null, 135 | "outputs": [] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "source": [ 140 | "## Data related utilities" 141 | ], 142 | "metadata": { 143 | "id": "nXHBYes2x5Nf" 144 | } 145 | }, 146 | { 147 | "cell_type": "code", 148 | "source": [ 149 | "# Closely aligns with https://github.com/huggingface/transformers/blob/main/src/transformers/models/beit/feature_extraction_beit.py\n", 150 | "# in regards to resizing and normalization.\n", 151 | "\n", 152 | "\n", 153 | "def make_dataset(dataset: tf.data.Dataset, train: bool, image_size: int = IMAGE_SIZE):\n", 154 | " def preprocess(image, label):\n", 155 | " # for training, do augmentation\n", 156 | " if train:\n", 157 | " if tf.random.uniform(shape=[]) > 0.5:\n", 158 | " image = tf.image.flip_left_right(image)\n", 159 | " image = tf.image.resize(image, size=image_size, method=\"bicubic\")\n", 160 | " image = (image - MEAN) / STD # normalization\n", 161 | " return image, label\n", 162 | "\n", 163 | " if train:\n", 164 | " dataset = dataset.shuffle(BATCH_SIZE * 10)\n", 165 | "\n", 166 | " dataset = dataset.map(preprocess, AUTO).batch(BATCH_SIZE)\n", 167 | " # Transpose because the `transformers` model has a leading channel dimension.\n", 168 | " dataset = dataset.map(lambda x, y: (tf.transpose(x, [0, 3, 1, 2]), y), AUTO)\n", 169 | " return dataset.prefetch(AUTO)" 170 | ], 171 | "metadata": { 172 | "id": "Vz_kqZWrgykI" 173 | }, 174 | "execution_count": null, 175 | "outputs": [] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "source": [ 180 | "## Load and visualize the `tf_flowers` dataset" 181 | ], 182 | "metadata": { 183 | "id": "ifd4d8GPx_nu" 184 | } 185 | }, 186 | { 187 | "cell_type": "code", 188 | "source": [ 189 | "train_dataset, val_dataset = tfds.load(\n", 190 | " \"tf_flowers\",\n", 191 | " split=[\"train[:90%]\", \"train[90%:]\"],\n", 192 | " as_supervised=True,\n", 193 | " try_gcs=False, # gcs_path is necessary for tpu,\n", 194 | ")\n", 195 | "\n", 196 | "num_train = tf.data.experimental.cardinality(train_dataset)\n", 197 | "num_val = tf.data.experimental.cardinality(val_dataset)\n", 198 | "print(f\"Number of training examples: {num_train}\")\n", 199 | "print(f\"Number of validation examples: {num_val}\")" 200 | ], 201 | "metadata": { 202 | "id": "7zVtKpWIokJd" 203 | }, 204 | "execution_count": null, 205 | "outputs": [] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "source": [ 210 | "train_dataset = make_dataset(train_dataset, True)\n", 211 | "val_dataset = make_dataset(val_dataset, False)" 212 | ], 213 | "metadata": { 214 | "id": "M-KhjVX9okxd" 215 | }, 216 | "execution_count": null, 217 | "outputs": [] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "source": [ 222 | "sample_images, sample_labels = next(iter(train_dataset))\n", 223 | "\n", 224 | "plt.figure(figsize=(5 * 3, 3 * 3))\n", 225 | "for n in range(15):\n", 226 | " ax = plt.subplot(3, 5, n + 1)\n", 227 | " image = (sample_images[n].numpy().transpose(1, 2, 0) * STD + MEAN).numpy()\n", 228 | " image = (image - image.min()) / (\n", 229 | " image.max() - image.min()\n", 230 | " ) # convert to [0, 1] for avoiding matplotlib warning\n", 231 | " plt.imshow(image)\n", 232 | " plt.title(CLASSES[sample_labels[n]])\n", 233 | " plt.axis(\"off\")\n", 234 | "plt.tight_layout()\n", 235 | "plt.show()" 236 | ], 237 | "metadata": { 238 | "id": "TS9Taf-UonJc" 239 | }, 240 | "execution_count": null, 241 | "outputs": [] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "source": [ 246 | "## Model building utilities" 247 | ], 248 | "metadata": { 249 | "id": "vjV7yqO1yCz_" 250 | } 251 | }, 252 | { 253 | "cell_type": "code", 254 | "source": [ 255 | "def get_model():\n", 256 | " base_model = TFData2VecVisionModel.from_pretrained(MODEL_ID)\n", 257 | " base_model.trainable = True\n", 258 | "\n", 259 | " inputs = layers.Input((3, IMAGE_SIZE[0], IMAGE_SIZE[1]))\n", 260 | " x = base_model(inputs, training=False)\n", 261 | " outputs = layers.Dense(5)(x.last_hidden_state[:, 0])\n", 262 | " model = keras.Model(inputs, outputs)\n", 263 | "\n", 264 | " return model" 265 | ], 266 | "metadata": { 267 | "id": "Sw4h4jFVg-Up" 268 | }, 269 | "execution_count": null, 270 | "outputs": [] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "source": [ 275 | "model = get_model()\n", 276 | "model.summary()" 277 | ], 278 | "metadata": { 279 | "id": "s1mU4RHgmgN8" 280 | }, 281 | "execution_count": null, 282 | "outputs": [] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "source": [ 287 | "## Compile model and train it" 288 | ], 289 | "metadata": { 290 | "id": "fx7-5d-SyUQ8" 291 | } 292 | }, 293 | { 294 | "cell_type": "code", 295 | "source": [ 296 | "num_train_epochs = 3\n", 297 | "learning_rate = 3e-5\n", 298 | "weight_decay_rate = 0.01\n", 299 | "num_warmup_steps = 0\n", 300 | "\n", 301 | "\n", 302 | "num_train_steps = (num_train // BATCH_SIZE) * num_train_epochs\n", 303 | "optimizer, lr_schedule = create_optimizer(\n", 304 | " init_lr=learning_rate,\n", 305 | " num_train_steps=num_train_steps,\n", 306 | " weight_decay_rate=weight_decay_rate,\n", 307 | " num_warmup_steps=num_warmup_steps,\n", 308 | ")" 309 | ], 310 | "metadata": { 311 | "id": "J0c-VsDEoXN1" 312 | }, 313 | "execution_count": null, 314 | "outputs": [] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "source": [ 319 | "model.compile(\n", 320 | " optimizer=optimizer,\n", 321 | " loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", 322 | " metrics=[\"accuracy\"],\n", 323 | ")" 324 | ], 325 | "metadata": { 326 | "id": "rIenATY_o7-Y" 327 | }, 328 | "execution_count": null, 329 | "outputs": [] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "source": [ 334 | "history = model.fit(train_dataset, validation_data=val_dataset, epochs=num_train_epochs)" 335 | ], 336 | "metadata": { 337 | "id": "fYZd0jEio_X-" 338 | }, 339 | "execution_count": null, 340 | "outputs": [] 341 | }, 342 | { 343 | "cell_type": "markdown", 344 | "source": [ 345 | "## Visualize training progress" 346 | ], 347 | "metadata": { 348 | "id": "KwWFR6vYyavE" 349 | } 350 | }, 351 | { 352 | "cell_type": "code", 353 | "source": [ 354 | "result = pd.DataFrame(history.history)\n", 355 | "fig, ax = plt.subplots(2, 1, figsize=(10, 10))\n", 356 | "result[[\"accuracy\", \"val_accuracy\"]].plot(xlabel=\"epoch\", ylabel=\"score\", ax=ax[0])\n", 357 | "result[[\"loss\", \"val_loss\"]].plot(xlabel=\"epoch\", ylabel=\"score\", ax=ax[1])" 358 | ], 359 | "metadata": { 360 | "id": "R10teItxqSFs" 361 | }, 362 | "execution_count": null, 363 | "outputs": [] 364 | }, 365 | { 366 | "cell_type": "markdown", 367 | "source": [ 368 | "## References\n", 369 | "\n", 370 | "* https://github.com/sayakpaul/cait-tf/blob/main/notebooks/finetune.ipynb\n", 371 | "* https://colab.research.google.com/drive/1M4CEl6Jgf4KgqLHb4_bi3acJvNNLZqZd" 372 | ], 373 | "metadata": { 374 | "id": "b88z7qyQtYvH" 375 | } 376 | } 377 | ] 378 | } -------------------------------------------------------------------------------- /tf.data exploration/tf.data - Initial Experiments.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Imports" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": { 14 | "scrolled": true 15 | }, 16 | "outputs": [ 17 | { 18 | "data": { 19 | "text/plain": [ 20 | "'2.0.0'" 21 | ] 22 | }, 23 | "execution_count": 1, 24 | "metadata": {}, 25 | "output_type": "execute_result" 26 | } 27 | ], 28 | "source": [ 29 | "import tensorflow as tf\n", 30 | "import time\n", 31 | "\n", 32 | "tf.__version__" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "## Data gathering" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/plain": [ 50 | "((60000, 28, 28), (60000,), (10000, 28, 28), (10000,))" 51 | ] 52 | }, 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "output_type": "execute_result" 56 | } 57 | ], 58 | "source": [ 59 | "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n", 60 | "\n", 61 | "X_train.shape, y_train.shape, X_test.shape, y_test.shape" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "## Prepare `tf.data` datasets" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "# https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices\n", 78 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))\n", 79 | "test_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test))" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "## Common data streaming steps\n", 87 | "\n", 88 | "- Shuffle\n", 89 | "- Batching\n", 90 | "- Batches to be available as soon as possible\n", 91 | "\n", 92 | "Reference: https://www.tensorflow.org/tutorials/load_data/images" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 4, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "data": { 102 | "text/plain": [ 103 | "tensorflow.python.data.ops.dataset_ops.PrefetchDataset" 104 | ] 105 | }, 106 | "execution_count": 4, 107 | "metadata": {}, 108 | "output_type": "execute_result" 109 | } 110 | ], 111 | "source": [ 112 | "train_dataset = train_dataset.\\\n", 113 | " shuffle(buffer_size=1000).\\\n", 114 | " repeat().\\\n", 115 | " batch(256).\\\n", 116 | " prefetch(buffer_size=1000)\n", 117 | "type(train_dataset)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "## Inspection of the batches" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 5, 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "data": { 134 | "text/plain": [ 135 | "TensorShape([256, 28, 28])" 136 | ] 137 | }, 138 | "execution_count": 5, 139 | "metadata": {}, 140 | "output_type": "execute_result" 141 | } 142 | ], 143 | "source": [ 144 | "for (images, labels) in train_dataset.take(1):\n", 145 | " pass\n", 146 | "\n", 147 | "images.shape" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "## Speed comparison" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": 6, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "gen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1/255.)\n", 164 | "train_flow = gen.flow(X_train.reshape(60000, 28, 28, 1), \n", 165 | " y_train, batch_size=256, shuffle=True)\n", 166 | "test_flow = gen.flow(X_test.reshape(10000, 28, 28, 1), \n", 167 | " y_test, batch_size=256)" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 7, 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [ 176 | "# Function courtesy: https://www.tensorflow.org/tutorials/load_data/images#performance\n", 177 | "default_timeit_steps = 1000\n", 178 | "\n", 179 | "def timeit(ds, steps=default_timeit_steps):\n", 180 | " start = time.time()\n", 181 | " it = iter(ds)\n", 182 | " for i in range(steps):\n", 183 | " batch = next(it)\n", 184 | " if i%10 == 0:\n", 185 | " print('.',end='')\n", 186 | " print()\n", 187 | " end = time.time()\n", 188 | "\n", 189 | " duration = end-start\n", 190 | " print(\"{} batches: {} s\".format(steps, duration))\n", 191 | " print(\"{:0.5f} Images/s\".format(256*steps/duration))" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 8, 197 | "metadata": {}, 198 | "outputs": [ 199 | { 200 | "name": "stdout", 201 | "output_type": "stream", 202 | "text": [ 203 | "....................................................................................................\n", 204 | "1000 batches: 4.16782808303833 s\n", 205 | "61422.87899 Images/s\n" 206 | ] 207 | } 208 | ], 209 | "source": [ 210 | "# Keras ImageDataGenerator\n", 211 | "timeit(train_flow)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 9, 217 | "metadata": {}, 218 | "outputs": [ 219 | { 220 | "name": "stdout", 221 | "output_type": "stream", 222 | "text": [ 223 | "....................................................................................................\n", 224 | "1000 batches: 0.9977211952209473 s\n", 225 | "256584.70646 Images/s\n" 226 | ] 227 | } 228 | ], 229 | "source": [ 230 | "# `tf.data`\n", 231 | "timeit(train_dataset)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "markdown", 236 | "metadata": {}, 237 | "source": [ 238 | "## With `AUTOTUNE` prefetching" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": 10, 244 | "metadata": {}, 245 | "outputs": [ 246 | { 247 | "name": "stdout", 248 | "output_type": "stream", 249 | "text": [ 250 | "....................................................................................................\n", 251 | "1000 batches: 0.9096441268920898 s\n", 252 | "281428.73947 Images/s\n" 253 | ] 254 | } 255 | ], 256 | "source": [ 257 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))\n", 258 | "\n", 259 | "train_dataset = train_dataset.\\\n", 260 | " shuffle(buffer_size=1000).\\\n", 261 | " repeat().\\\n", 262 | " batch(256).\\\n", 263 | " prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n", 264 | "\n", 265 | "timeit(train_dataset)" 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": {}, 271 | "source": [ 272 | "## With caching" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": 11, 278 | "metadata": {}, 279 | "outputs": [ 280 | { 281 | "name": "stdout", 282 | "output_type": "stream", 283 | "text": [ 284 | "....................................................................................................\n", 285 | "1000 batches: 0.6213550567626953 s\n", 286 | "412002.76269 Images/s\n" 287 | ] 288 | } 289 | ], 290 | "source": [ 291 | "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))\n", 292 | "\n", 293 | "train_dataset = train_dataset.cache().\\\n", 294 | " shuffle(buffer_size=1000).\\\n", 295 | " repeat().\\\n", 296 | " batch(256).\\\n", 297 | " prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n", 298 | "\n", 299 | "timeit(train_dataset)" 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "metadata": {}, 305 | "source": [ 306 | "## Modeling with `tf.data` datasets" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 12, 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [ 315 | "# Define and compile a model\n", 316 | "\n", 317 | "model = tf.keras.models.Sequential([\n", 318 | " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", 319 | " tf.keras.layers.Dense(128, activation='relu'),\n", 320 | " tf.keras.layers.Dropout(0.2),\n", 321 | " tf.keras.layers.Dense(10, activation='softmax')\n", 322 | "])\n", 323 | "\n", 324 | "model.compile(optimizer='adam',\n", 325 | " loss='sparse_categorical_crossentropy',\n", 326 | " metrics=['accuracy'])" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 13, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [ 335 | "test_dataset = test_dataset.batch(256)" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": 14, 341 | "metadata": {}, 342 | "outputs": [ 343 | { 344 | "name": "stdout", 345 | "output_type": "stream", 346 | "text": [ 347 | "Train for 234 steps, validate for 40 steps\n", 348 | "Epoch 1/5\n", 349 | "234/234 [==============================] - 2s 8ms/step - loss: 6.7519 - accuracy: 0.5885 - val_loss: 0.9638 - val_accuracy: 0.6781\n", 350 | "Epoch 2/5\n", 351 | "234/234 [==============================] - 1s 5ms/step - loss: 1.0317 - accuracy: 0.6524 - val_loss: 0.8071 - val_accuracy: 0.7143\n", 352 | "Epoch 3/5\n", 353 | "234/234 [==============================] - 1s 5ms/step - loss: 0.8962 - accuracy: 0.6831 - val_loss: 0.7533 - val_accuracy: 0.7413\n", 354 | "Epoch 4/5\n", 355 | "234/234 [==============================] - 1s 5ms/step - loss: 0.7967 - accuracy: 0.7080 - val_loss: 0.6861 - val_accuracy: 0.7614\n", 356 | "Epoch 5/5\n", 357 | "234/234 [==============================] - 1s 5ms/step - loss: 0.7276 - accuracy: 0.7306 - val_loss: 0.6658 - val_accuracy: 0.7783\n" 358 | ] 359 | }, 360 | { 361 | "data": { 362 | "text/plain": [ 363 | "" 364 | ] 365 | }, 366 | "execution_count": 14, 367 | "metadata": {}, 368 | "output_type": "execute_result" 369 | } 370 | ], 371 | "source": [ 372 | "model.fit(train_dataset,\n", 373 | " steps_per_epoch=len(X_train)//256,\n", 374 | " epochs=5,\n", 375 | " validation_data=test_dataset)" 376 | ] 377 | } 378 | ], 379 | "metadata": { 380 | "kernelspec": { 381 | "display_name": "Python 3", 382 | "language": "python", 383 | "name": "python3" 384 | }, 385 | "language_info": { 386 | "codemirror_mode": { 387 | "name": "ipython", 388 | "version": 3 389 | }, 390 | "file_extension": ".py", 391 | "mimetype": "text/x-python", 392 | "name": "python", 393 | "nbconvert_exporter": "python", 394 | "pygments_lexer": "ipython3", 395 | "version": "3.7.1" 396 | } 397 | }, 398 | "nbformat": 4, 399 | "nbformat_minor": 2 400 | } 401 | -------------------------------------------------------------------------------- /tf.data exploration/tf_data_with_tf_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "tf.data with tf.keras.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [] 9 | }, 10 | "kernelspec": { 11 | "name": "python3", 12 | "display_name": "Python 3" 13 | }, 14 | "accelerator": "GPU" 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "07mAPLQ8yqc9", 21 | "colab_type": "text" 22 | }, 23 | "source": [ 24 | "The purpose of this notebook is two-fold:\n", 25 | "\n", 26 | "- Show the usage of `tf.data.Dataset.from_generator` along with Keras `ImageDataGenerator` for Keras models.\n", 27 | "- Compare the performance between `tf.data.Dataset.from_generator` and `ImageDtaGenerator`. \n", 28 | "\n", 29 | "A huge thanks to **Picsou Balthazar** for helping me out in this. " 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "metadata": { 35 | "id": "PHp2tBzeAZT3", 36 | "colab_type": "code", 37 | "colab": {} 38 | }, 39 | "source": [ 40 | "# Install TensorFlow 2.0 (GPU)\n", 41 | "!pip install tensorflow-gpu" 42 | ], 43 | "execution_count": 0, 44 | "outputs": [] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "metadata": { 49 | "id": "UQ53B3_fA6CY", 50 | "colab_type": "code", 51 | "colab": {} 52 | }, 53 | "source": [ 54 | "# Import the packages\n", 55 | "# Import the packages for DL stuff\n", 56 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 57 | "from tensorflow.keras.applications import VGG16\n", 58 | "from tensorflow.keras.layers import Dropout\n", 59 | "from tensorflow.keras.layers import Flatten\n", 60 | "from tensorflow.keras.layers import Dense\n", 61 | "from tensorflow.keras.layers import Input\n", 62 | "from tensorflow.keras.models import Model\n", 63 | "from tensorflow.keras.optimizers import SGD\n", 64 | "from imutils import paths\n", 65 | "import matplotlib.pyplot as plt\n", 66 | "import tensorflow as tf\n", 67 | "import pandas as pd\n", 68 | "import numpy as np\n", 69 | "import time" 70 | ], 71 | "execution_count": 0, 72 | "outputs": [] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "metadata": { 77 | "id": "ArwTIOe7uBay", 78 | "colab_type": "code", 79 | "outputId": "36539805-9f39-48e4-c6f6-62b6c2bc0805", 80 | "colab": { 81 | "base_uri": "https://localhost:8080/", 82 | "height": 34 83 | } 84 | }, 85 | "source": [ 86 | "# verify if the right version was installed\n", 87 | "tf.__version__" 88 | ], 89 | "execution_count": 2, 90 | "outputs": [ 91 | { 92 | "output_type": "execute_result", 93 | "data": { 94 | "text/plain": [ 95 | "'2.0.0'" 96 | ] 97 | }, 98 | "metadata": { 99 | "tags": [] 100 | }, 101 | "execution_count": 2 102 | } 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "metadata": { 108 | "id": "A6Qij-Z5BELC", 109 | "colab_type": "code", 110 | "colab": {} 111 | }, 112 | "source": [ 113 | "# Get the flowers' dataset\n", 114 | "flowers = tf.keras.utils.get_file(\n", 115 | " 'flower_photos',\n", 116 | " 'https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", 117 | " untar=True)" 118 | ], 119 | "execution_count": 0, 120 | "outputs": [] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "metadata": { 125 | "id": "d_6Lc1Yaze9d", 126 | "colab_type": "code", 127 | "outputId": "c27cc451-953c-484b-a86e-c6aa33ca1cd8", 128 | "colab": { 129 | "base_uri": "https://localhost:8080/", 130 | "height": 34 131 | } 132 | }, 133 | "source": [ 134 | "!ls {flowers}" 135 | ], 136 | "execution_count": 4, 137 | "outputs": [ 138 | { 139 | "output_type": "stream", 140 | "text": [ 141 | "daisy dandelion LICENSE.txt roses sunflowers tulips\n" 142 | ], 143 | "name": "stdout" 144 | } 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": { 150 | "id": "hJVAZAqxxzmP", 151 | "colab_type": "text" 152 | }, 153 | "source": [ 154 | "Know more about the dataset here: https://www.tensorflow.org/tutorials/load_data/images#load_using_tfdata. Following is the glimpse of the dataset.\n", 155 | "\n", 156 | "![](https://www.tensorflow.org/tutorials/load_data/images_files/output_suh6Sjv68rY3_0.png)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "metadata": { 162 | "id": "RvcluHUqBJ26", 163 | "colab_type": "code", 164 | "colab": {} 165 | }, 166 | "source": [ 167 | "# Initialize the data augmentation object and set its mean to the\n", 168 | "# mean of the ImageNet dataset\n", 169 | "img_gen = tf.keras.preprocessing.image.ImageDataGenerator(rotation_range=20)\n", 170 | "mean = np.array([123.68, 116.779, 103.939], dtype=\"float32\")\n", 171 | "img_gen.mean = mean" 172 | ], 173 | "execution_count": 0, 174 | "outputs": [] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "metadata": { 179 | "id": "qb8tvXmvBSjL", 180 | "colab_type": "code", 181 | "colab": { 182 | "base_uri": "https://localhost:8080/", 183 | "height": 34 184 | }, 185 | "outputId": "5165ce96-cbae-483f-be17-3c346d6e5f7b" 186 | }, 187 | "source": [ 188 | "# Wrap the generator with tf.data\n", 189 | "ds = tf.data.Dataset.from_generator(\n", 190 | " lambda: img_gen.flow_from_directory(flowers,\n", 191 | " class_mode=\"categorical\",\n", 192 | " target_size=(224, 224),\n", 193 | " color_mode=\"rgb\",\n", 194 | " shuffle=True),\n", 195 | " output_types=(tf.float32, tf.float32),\n", 196 | " output_shapes = ([None,224,224,3],[None,5])\n", 197 | ")\n", 198 | "\n", 199 | "ds" 200 | ], 201 | "execution_count": 11, 202 | "outputs": [ 203 | { 204 | "output_type": "execute_result", 205 | "data": { 206 | "text/plain": [ 207 | "" 208 | ] 209 | }, 210 | "metadata": { 211 | "tags": [] 212 | }, 213 | "execution_count": 11 214 | } 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "metadata": { 220 | "id": "3gsOo6vSBWNd", 221 | "colab_type": "code", 222 | "colab": { 223 | "base_uri": "https://localhost:8080/", 224 | "height": 51 225 | }, 226 | "outputId": "9359cfa1-afa3-4d2b-c2f4-2635f5039bf7" 227 | }, 228 | "source": [ 229 | "# Verify the shapes yielded by the data generator\n", 230 | "train_gen = img_gen.flow_from_directory(flowers,\n", 231 | " class_mode=\"categorical\",\n", 232 | " target_size=(224, 224),\n", 233 | " color_mode=\"rgb\",\n", 234 | " shuffle=True)\n", 235 | "images, labels = next(train_gen)\n", 236 | "images.shape, labels.shape" 237 | ], 238 | "execution_count": 7, 239 | "outputs": [ 240 | { 241 | "output_type": "stream", 242 | "text": [ 243 | "Found 3670 images belonging to 5 classes.\n" 244 | ], 245 | "name": "stdout" 246 | }, 247 | { 248 | "output_type": "execute_result", 249 | "data": { 250 | "text/plain": [ 251 | "((32, 224, 224, 3), (32, 5))" 252 | ] 253 | }, 254 | "metadata": { 255 | "tags": [] 256 | }, 257 | "execution_count": 7 258 | } 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "metadata": { 264 | "id": "PHlIxEt_BZE_", 265 | "colab_type": "code", 266 | "colab": {} 267 | }, 268 | "source": [ 269 | "# A helper script for which would initialize and compile \n", 270 | "# our model\n", 271 | "def get_training_model():\n", 272 | " baseModel = VGG16(weights=\"imagenet\", include_top=False,\n", 273 | " input_tensor=Input(shape=(224, 224, 3)))\n", 274 | "\n", 275 | " headModel = baseModel.output\n", 276 | " headModel = Flatten(name=\"flatten\")(headModel)\n", 277 | " headModel = Dense(512, activation=\"relu\")(headModel)\n", 278 | " headModel = Dropout(0.5)(headModel)\n", 279 | " headModel = Dense(5, activation=\"softmax\")(headModel)\n", 280 | "\n", 281 | " model = Model(inputs=baseModel.input, outputs=headModel)\n", 282 | "\n", 283 | " for layer in baseModel.layers:\n", 284 | " layer.trainable = False\n", 285 | "\n", 286 | " opt = SGD(lr=1e-4, momentum=0.9)\n", 287 | " model.compile(loss=\"categorical_crossentropy\", optimizer=opt,\n", 288 | " metrics=[\"accuracy\"])\n", 289 | " return model" 290 | ], 291 | "execution_count": 0, 292 | "outputs": [] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "metadata": { 297 | "id": "ERLsOthaBj76", 298 | "colab_type": "code", 299 | "colab": {} 300 | }, 301 | "source": [ 302 | "# Get the total number of images present in the\n", 303 | "# root dataset directory\n", 304 | "total_data = len(list(paths.list_images(flowers)))" 305 | ], 306 | "execution_count": 0, 307 | "outputs": [] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "metadata": { 312 | "id": "M32wuUHvBwlU", 313 | "colab_type": "code", 314 | "colab": { 315 | "base_uri": "https://localhost:8080/", 316 | "height": 238 317 | }, 318 | "outputId": "c733a950-7a59-4c32-8cdf-57b54f13ec42" 319 | }, 320 | "source": [ 321 | "# Kickstart model training with tf.data\n", 322 | "model = get_training_model()\n", 323 | "start = time.time()\n", 324 | "model.fit(ds, \n", 325 | " steps_per_epoch=total_data//32,\n", 326 | " epochs=5)\n", 327 | "print(\"It took {} seconds\".format(time.time() - start))" 328 | ], 329 | "execution_count": 12, 330 | "outputs": [ 331 | { 332 | "output_type": "stream", 333 | "text": [ 334 | "Train for 114 steps\n", 335 | "Epoch 1/5\n", 336 | "Found 3670 images belonging to 5 classes.\n", 337 | "114/114 [==============================] - 80s 704ms/step - loss: 5.2687 - accuracy: 0.6757\n", 338 | "Epoch 2/5\n", 339 | "114/114 [==============================] - 82s 715ms/step - loss: 0.9821 - accuracy: 0.7883\n", 340 | "Epoch 3/5\n", 341 | "114/114 [==============================] - 79s 695ms/step - loss: 0.6616 - accuracy: 0.8219\n", 342 | "Epoch 4/5\n", 343 | "114/114 [==============================] - 79s 695ms/step - loss: 0.4871 - accuracy: 0.8535\n", 344 | "Epoch 5/5\n", 345 | "114/114 [==============================] - 80s 699ms/step - loss: 0.3722 - accuracy: 0.8821\n", 346 | "It took 399.8591330051422 seconds\n" 347 | ], 348 | "name": "stdout" 349 | } 350 | ] 351 | }, 352 | { 353 | "cell_type": "markdown", 354 | "metadata": { 355 | "id": "yfDEGi7Zwvkr", 356 | "colab_type": "text" 357 | }, 358 | "source": [ 359 | "It took **399.85 seconds**. Let's now see how `ImageDataGenerator` performs. " 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "metadata": { 365 | "id": "9dMLGo9Vw0LM", 366 | "colab_type": "code", 367 | "outputId": "87ed4320-5d07-41ea-b6d5-42c4cb22dde8", 368 | "colab": { 369 | "base_uri": "https://localhost:8080/", 370 | "height": 204 371 | } 372 | }, 373 | "source": [ 374 | "# Kickstart model training with ImageDataGenerator\n", 375 | "model = get_training_model()\n", 376 | "start = time.time()\n", 377 | "model.fit_generator(train_gen, \n", 378 | " steps_per_epoch=total_data//32,\n", 379 | " epochs=5)\n", 380 | "print(\"It took {} seconds\".format(time.time() - start))" 381 | ], 382 | "execution_count": 13, 383 | "outputs": [ 384 | { 385 | "output_type": "stream", 386 | "text": [ 387 | "Epoch 1/5\n", 388 | "114/114 [==============================] - 144s 1s/step - loss: 5.5306 - accuracy: 0.5962\n", 389 | "Epoch 2/5\n", 390 | "114/114 [==============================] - 135s 1s/step - loss: 3.3431 - accuracy: 0.7526\n", 391 | "Epoch 3/5\n", 392 | "114/114 [==============================] - 135s 1s/step - loss: 2.7142 - accuracy: 0.7974\n", 393 | "Epoch 4/5\n", 394 | "114/114 [==============================] - 135s 1s/step - loss: 2.4073 - accuracy: 0.8120\n", 395 | "Epoch 5/5\n", 396 | "114/114 [==============================] - 135s 1s/step - loss: 2.0779 - accuracy: 0.8378\n", 397 | "It took 685.5545630455017 seconds\n" 398 | ], 399 | "name": "stdout" 400 | } 401 | ] 402 | }, 403 | { 404 | "cell_type": "markdown", 405 | "metadata": { 406 | "id": "IlvOBeUOyQZl", 407 | "colab_type": "text" 408 | }, 409 | "source": [ 410 | "Takes **685.55 seconds**. " 411 | ] 412 | }, 413 | { 414 | "cell_type": "markdown", 415 | "metadata": { 416 | "id": "pv6aB4K23Wnv", 417 | "colab_type": "text" 418 | }, 419 | "source": [ 420 | "Links to `tf.data` guides: \n", 421 | "- https://www.tensorflow.org/guide/data\n", 422 | "- https://www.tensorflow.org/guide/data_performance" 423 | ] 424 | } 425 | ] 426 | } --------------------------------------------------------------------------------