├── .gitignore ├── 1.Neural Networks and Deep Learning ├── 1.Intro to Deep Learning │ └── Introduction to Deep Learning.ipynb ├── 2.Neural Networks Basics │ ├── Assignment1.ipynb │ ├── Neural Networks Basics.ipynb │ ├── __pycache__ │ │ └── lr_utils.cpython-36.pyc │ ├── datasets │ │ ├── test_catvnoncat.h5 │ │ └── train_catvnoncat.h5 │ ├── images │ │ ├── LogReg_kiank.png │ │ ├── cat_in_iran.jpg │ │ ├── gargouille.jpg │ │ ├── image1.png │ │ ├── image2.png │ │ ├── la_defense.jpg │ │ ├── my_image.jpg │ │ └── my_image2.jpg │ └── lr_utils.py ├── 3.Shallow Neural Networks │ ├── Planar Data Classification With One Hidden Layer.ipynb │ ├── Shallow Neural Network.ipynb │ ├── __pycache__ │ │ ├── planar_utils.cpython-36.pyc │ │ └── testCases.cpython-36.pyc │ ├── planar_utils.py │ └── testCases.py └── 4.Deep Neural Networks │ ├── Building a Deep Neural Network.ipynb │ ├── Deep Neural Network - Application.ipynb │ ├── __pycache__ │ ├── dnn_app_utils_v2.cpython-36.pyc │ ├── dnn_utils_v2.cpython-36.pyc │ └── testCases_v2.cpython-36.pyc │ ├── datasets │ ├── test_catvnoncat.h5 │ └── train_catvnoncat.h5 │ ├── dnn_app_utils_v2.py │ ├── dnn_utils_v2.py │ ├── my_image.jpg │ └── testCases_v2.py ├── 2.Improving Deep Neural Networks ├── 1.Practical Aspects of Deep Learning │ ├── Gradient Checking.ipynb │ ├── Initialization.ipynb │ ├── Regularization.ipynb │ ├── __pycache__ │ │ ├── gc_utils.cpython-36.pyc │ │ ├── init_utils.cpython-36.pyc │ │ ├── reg_utils.cpython-36.pyc │ │ └── testCases.cpython-36.pyc │ ├── datasets │ │ ├── data.mat │ │ ├── test_catvnoncat.h5 │ │ └── train_catvnoncat.h5 │ ├── gc_utils.py │ ├── init_utils.py │ ├── reg_utils.py │ └── testCases.py ├── 2.Algorithm Optimization │ ├── Optimization Methods.ipynb │ ├── __pycache__ │ │ ├── opt_utils.cpython-36.pyc │ │ └── testCases.cpython-36.pyc │ ├── datasets │ │ └── data.mat │ ├── opt_utils.py │ └── testCases.py └── 3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks │ ├── Tensorflow Application.ipynb │ ├── Tensorflow Tutorial.ipynb │ ├── __pycache__ │ └── tf_utils.cpython-36.pyc │ ├── datasets │ ├── test_signs.h5 │ └── train_signs.h5 │ ├── improv_utils.py │ ├── tf_utils.py │ └── thumbs_up.jpg ├── 3.Structuring Machine Learning Projects └── Structuring Machine Learning Project.ipynb ├── 4.Convolutional Neural Networks ├── 1.Foundations of Convolutional Neural Networks │ ├── ConvNet in TensorFlow.ipynb │ ├── Convolutional Neural Network - Step by Step.ipynb │ ├── __pycache__ │ │ └── cnn_utils.cpython-36.pyc │ ├── cnn_utils.py │ └── datasets │ │ ├── test_signs.h5 │ │ └── train_signs.h5 ├── 2.Deep Convolutional Models │ ├── HappyModel.png │ ├── Keras Tutorial.ipynb │ ├── ResNet.png │ ├── Residual Networks.ipynb │ ├── __pycache__ │ │ ├── kt_utils.cpython-36.pyc │ │ └── resnets_utils.cpython-36.pyc │ ├── datasets │ │ ├── test_happy.h5 │ │ ├── test_signs.h5 │ │ ├── train_happy.h5 │ │ └── train_signs.h5 │ ├── kt_utils.py │ └── resnets_utils.py ├── 4.Special Applications - Facial Recognition & Neural Style Transfer │ ├── Neural Style Transfer.ipynb │ ├── datasets │ │ ├── test_happy.h5 │ │ ├── train_face.h5 │ │ └── train_happy.h5 │ ├── fr_utils.py │ ├── inception_blocks.py │ ├── nn4.small2.v7.h5 │ └── nst_utils.py └── Object Detection │ ├── YOLO Implementation.ipynb │ └── sample.txt └── 5.Sequence Models ├── 1.Recurrent Neural Networks ├── Building a Recurrent Neural Network From Scratch.ipynb ├── Dinosoraus Land.ipynb ├── Improvise a Jazz Solo with LSTM.ipynb ├── data_utils.py ├── dinos.txt ├── grammar.py ├── music_utils.py ├── my_music.midi ├── preprocess.py ├── qa.py ├── rnn_utils.py ├── sample.txt ├── shakespeare.txt ├── shakespeare_utils.py └── utils.py ├── 2.Natural Language Processing & Word Embeddings ├── Emojify.ipynb ├── Remove Bias.ipynb ├── emo_utils.py ├── sample.txt └── w2v_utils.py └── 3.Sequence Models & Attention Mechanism ├── Neural Machine Translation.ipynb ├── Trigger+word+detection+-+v1.ipynb ├── nmt_utils.py └── sample.txt /.gitignore: -------------------------------------------------------------------------------- 1 | /1.Neural Networks and Deep Learning/1.Intro to Deep Learning/.ipynb_checkpoints/ 2 | /1.Neural Networks and Deep Learning/2.Neural Networks Basics/.ipynb_checkpoints/ 3 | /1.Neural Networks and Deep Learning/3.Shallow Neural Networks/.ipynb_checkpoints/ 4 | /1.Neural Networks and Deep Learning/4.Deep Neural Networks/.ipynb_checkpoints/ 5 | 6 | /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/.ipynb_checkpoints/ 7 | /2.Improving Deep Neural Networks/2.Algorithm Optimization/.ipynb_checkpoints/ 8 | /2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/.ipynb_checkpoints/ 9 | 10 | /3.Structuring Machine Learning Projects/.ipynb_checkpoints/ 11 | 12 | /4.Convolutional Neural Networks/1.Foundations of Convolutional Neural Networks/.ipynb_checkpoints/ 13 | /4.Convolutional Neural Networks/2.Deep Convolutional Models/.ipynb_checkpoints/ 14 | /4.Convolutional Neural Networks/3.Object Detection/.ipynb_checkpoints/ 15 | /4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/.ipynb_checkpoints/ -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/1.Intro to Deep Learning/Introduction to Deep Learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Introduction to Deep Learning " 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## What is a neural network?" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "Deep learning refers to training neural network. But what is a neural network?\n", 22 | "\n", 23 | "A neural network is simply a function that fits some data. The simplest form a neural network is a single neuron. It can take multiple inputs, but it will only fit one function, and generate one output.\n", 24 | "\n", 25 | "![Image](https://www.researchgate.net/profile/Ravi_Singh100/publication/307173547/figure/fig1/AS:400319998644225@1472455133744/Information-processing-by-a-single-neuron-in-Artificial-Neural-Network-ANN.png)\n", 26 | "\n", 27 | "Now, we can think of a neuron as a building block of a neural network. By combining multiple neurons, we can now build a neural network.\n", 28 | "\n", 29 | "![Image](https://cdn-images-1.medium.com/max/1200/1*RGV6Bb3ChmVWsA8Q6Qth6Q.png)\n", 30 | "\n", 31 | "Notice how each input is fed to each neuron. The neural network will figure out by itself which function is the best to fit the inputs to some data. All we need to provide are the inputs. We say this network is *density connected* because each feature is fed to each node." 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "## Supervised learning with neural networks" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "Neural networks have been succefully applied in many supervised learning scenarios such as:\n", 46 | "- online advertising\n", 47 | "- photo tagging\n", 48 | "- speech recognition\n", 49 | "- machine translation\n", 50 | "- autonomous driving" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "metadata": {}, 56 | "source": [ 57 | "## Why deep learning?" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "The performance of traditional machine learning algorithms will stall, no matter how much data is available. On the other hand, very large neural networks will show an increasingly better performance as the available amount of data increases. Nowadays, data storage is cheaper, and the popularity of websites and smartphones allow to generate a large amount of data! That's why deep learning is so exciting right now." 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [] 73 | } 74 | ], 75 | "metadata": { 76 | "kernelspec": { 77 | "display_name": "Python 3", 78 | "language": "python", 79 | "name": "python3" 80 | }, 81 | "language_info": { 82 | "codemirror_mode": { 83 | "name": "ipython", 84 | "version": 3 85 | }, 86 | "file_extension": ".py", 87 | "mimetype": "text/x-python", 88 | "name": "python", 89 | "nbconvert_exporter": "python", 90 | "pygments_lexer": "ipython3", 91 | "version": "3.6.6" 92 | } 93 | }, 94 | "nbformat": 4, 95 | "nbformat_minor": 2 96 | } 97 | -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/Neural Networks Basics.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Neural Networks Basics" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Vectorization" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 15, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import numpy as np\n", 24 | "import time" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "name": "stdout", 34 | "output_type": "stream", 35 | "text": [ 36 | "[1 2 3 4]\n" 37 | ] 38 | } 39 | ], 40 | "source": [ 41 | "#Example of an array in numpy\n", 42 | "\n", 43 | "a = np.array([1,2,3,4])\n", 44 | "\n", 45 | "print(a)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 16, 51 | "metadata": {}, 52 | "outputs": [ 53 | { 54 | "name": "stdout", 55 | "output_type": "stream", 56 | "text": [ 57 | "Vectorized version: 1.0001659393310547 ms\n" 58 | ] 59 | } 60 | ], 61 | "source": [ 62 | "#Vectorized operation\n", 63 | "\n", 64 | "a = np.random.rand(1000000)\n", 65 | "b = np.random.rand(1000000)\n", 66 | "\n", 67 | "tic = time.time()\n", 68 | "c = np.dot(a, b)\n", 69 | "toc = time.time()\n", 70 | "\n", 71 | "print('Vectorized version: ' + str(1000*(toc-tic)) + ' ms')" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 17, 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "name": "stdout", 81 | "output_type": "stream", 82 | "text": [ 83 | "For loop: 380.7802200317383 ms\n" 84 | ] 85 | } 86 | ], 87 | "source": [ 88 | "#Same operation but with a for loop\n", 89 | "\n", 90 | "c = 0\n", 91 | "\n", 92 | "tic = time.time()\n", 93 | "for i in range(1000000):\n", 94 | " c += a[i] * b[i]\n", 95 | "\n", 96 | "toc = time.time()\n", 97 | "\n", 98 | "print('For loop: ' + str(1000*(toc-tic)) + ' ms')" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "## Broadcasting in Python" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 18, 111 | "metadata": {}, 112 | "outputs": [ 113 | { 114 | "name": "stdout", 115 | "output_type": "stream", 116 | "text": [ 117 | "[[ 56. 0. 4.4 68. ]\n", 118 | " [ 1.2 104. 52. 8. ]\n", 119 | " [ 1.8 135. 99. 0.9]]\n" 120 | ] 121 | } 122 | ], 123 | "source": [ 124 | "A = np.array([[56.0, 0.0, 4.4, 68.0],\n", 125 | " [1.2, 104.0, 52.0, 8.0],\n", 126 | " [1.8, 135.0, 99.0, 0.9]])\n", 127 | "\n", 128 | "print(A)" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 21, 134 | "metadata": {}, 135 | "outputs": [ 136 | { 137 | "name": "stdout", 138 | "output_type": "stream", 139 | "text": [ 140 | "[ 59. 239. 155.4 76.9]\n" 141 | ] 142 | } 143 | ], 144 | "source": [ 145 | "calories = A.sum(axis=0)\n", 146 | "print(calories)" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 22, 152 | "metadata": {}, 153 | "outputs": [ 154 | { 155 | "name": "stdout", 156 | "output_type": "stream", 157 | "text": [ 158 | "[[94.91525424 0. 2.83140283 88.42652796]\n", 159 | " [ 2.03389831 43.51464435 33.46203346 10.40312094]\n", 160 | " [ 3.05084746 56.48535565 63.70656371 1.17035111]]\n" 161 | ] 162 | } 163 | ], 164 | "source": [ 165 | "percentage = 100 * A / calories.reshape(1,4)\n", 166 | "print(percentage)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [] 175 | } 176 | ], 177 | "metadata": { 178 | "kernelspec": { 179 | "display_name": "Python 3", 180 | "language": "python", 181 | "name": "python3" 182 | }, 183 | "language_info": { 184 | "codemirror_mode": { 185 | "name": "ipython", 186 | "version": 3 187 | }, 188 | "file_extension": ".py", 189 | "mimetype": "text/x-python", 190 | "name": "python", 191 | "nbconvert_exporter": "python", 192 | "pygments_lexer": "ipython3", 193 | "version": "3.6.6" 194 | } 195 | }, 196 | "nbformat": 4, 197 | "nbformat_minor": 2 198 | } 199 | -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/__pycache__/lr_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/__pycache__/lr_utils.cpython-36.pyc -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/datasets/test_catvnoncat.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/datasets/test_catvnoncat.h5 -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/datasets/train_catvnoncat.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/datasets/train_catvnoncat.h5 -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/LogReg_kiank.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/LogReg_kiank.png -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/cat_in_iran.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/cat_in_iran.jpg -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/gargouille.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/gargouille.jpg -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/image1.png -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/image2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/image2.png -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/la_defense.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/la_defense.jpg -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/my_image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/my_image.jpg -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/my_image2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/2.Neural Networks Basics/images/my_image2.jpg -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/2.Neural Networks Basics/lr_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | 4 | 5 | def load_dataset(): 6 | train_dataset = h5py.File('datasets/train_catvnoncat.h5', "r") 7 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 8 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 9 | 10 | test_dataset = h5py.File('datasets/test_catvnoncat.h5', "r") 11 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 12 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 13 | 14 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 15 | 16 | train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 17 | test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 18 | 19 | return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/3.Shallow Neural Networks/Shallow Neural Network.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Shallow Neural Network" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Neural Networks Overview" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "Simply a stack of single neurons." 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": {}, 27 | "source": [ 28 | "## Neural Network Representation" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "There is an input layer, a hidden layer, and an output layer. The values in the hidden layer are not observed.\n", 36 | "\n", 37 | "The input layer does not count in the number of layers. We only count the number of hidden layers and add the output layer." 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Activation function" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "`tanh` is usually a better activation function than he sigmoid function. However, if the ouput is binary, the sigmoid function makes more sense to use." 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "However, both functions have the downside of having very small derivatives for large values of `x`. Therefore, the rectified linear unit (ReLU) can also be used.\n", 59 | "\n", 60 | "ReLU = max(0, z)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "## Why non-linear activation function?" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "Didn't get it... Need to look somewhere else" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "## Random initialization" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "Cannot initalize weight to 0, because all activations will be the same. Likely, the derivatives will be the same. Therefore, the hidden units will be symmetric (completely identic). \n", 89 | "\n", 90 | "Hence, they compute the same activation. Therefore, no matter how we train the network, it will always compute the same output.\n", 91 | "\n", 92 | "That's why you must initialize the weights randomly." 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [] 101 | } 102 | ], 103 | "metadata": { 104 | "kernelspec": { 105 | "display_name": "Python 3", 106 | "language": "python", 107 | "name": "python3" 108 | }, 109 | "language_info": { 110 | "codemirror_mode": { 111 | "name": "ipython", 112 | "version": 3 113 | }, 114 | "file_extension": ".py", 115 | "mimetype": "text/x-python", 116 | "name": "python", 117 | "nbconvert_exporter": "python", 118 | "pygments_lexer": "ipython3", 119 | "version": "3.6.6" 120 | } 121 | }, 122 | "nbformat": 4, 123 | "nbformat_minor": 2 124 | } 125 | -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/3.Shallow Neural Networks/__pycache__/planar_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/3.Shallow Neural Networks/__pycache__/planar_utils.cpython-36.pyc -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/3.Shallow Neural Networks/__pycache__/testCases.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/3.Shallow Neural Networks/__pycache__/testCases.cpython-36.pyc -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/3.Shallow Neural Networks/planar_utils.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import sklearn 4 | import sklearn.datasets 5 | import sklearn.linear_model 6 | 7 | def plot_decision_boundary(model, X, y): 8 | # Set min and max values and give it some padding 9 | x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1 10 | y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1 11 | h = 0.01 12 | # Generate a grid of points with distance h between them 13 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) 14 | # Predict the function value for the whole grid 15 | Z = model(np.c_[xx.ravel(), yy.ravel()]) 16 | Z = Z.reshape(xx.shape) 17 | # Plot the contour and training examples 18 | plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral) 19 | plt.ylabel('x2') 20 | plt.xlabel('x1') 21 | plt.scatter(X[0, :], X[1, :], c=y.ravel(), cmap=plt.cm.Spectral) 22 | 23 | 24 | def sigmoid(x): 25 | """ 26 | Compute the sigmoid of x 27 | 28 | Arguments: 29 | x -- A scalar or numpy array of any size. 30 | 31 | Return: 32 | s -- sigmoid(x) 33 | """ 34 | s = 1/(1+np.exp(-x)) 35 | return s 36 | 37 | def load_planar_dataset(): 38 | np.random.seed(1) 39 | m = 400 # number of examples 40 | N = int(m/2) # number of points per class 41 | D = 2 # dimensionality 42 | X = np.zeros((m,D)) # data matrix where each row is a single example 43 | Y = np.zeros((m,1), dtype='uint8') # labels vector (0 for red, 1 for blue) 44 | a = 4 # maximum ray of the flower 45 | 46 | for j in range(2): 47 | ix = range(N*j,N*(j+1)) 48 | t = np.linspace(j*3.12,(j+1)*3.12,N) + np.random.randn(N)*0.2 # theta 49 | r = a*np.sin(4*t) + np.random.randn(N)*0.2 # radius 50 | X[ix] = np.c_[r*np.sin(t), r*np.cos(t)] 51 | Y[ix] = j 52 | 53 | X = X.T 54 | Y = Y.T 55 | 56 | return X, Y 57 | 58 | def load_extra_datasets(): 59 | N = 200 60 | noisy_circles = sklearn.datasets.make_circles(n_samples=N, factor=.5, noise=.3) 61 | noisy_moons = sklearn.datasets.make_moons(n_samples=N, noise=.2) 62 | blobs = sklearn.datasets.make_blobs(n_samples=N, random_state=5, n_features=2, centers=6) 63 | gaussian_quantiles = sklearn.datasets.make_gaussian_quantiles(mean=None, cov=0.5, n_samples=N, n_features=2, n_classes=2, shuffle=True, random_state=None) 64 | no_structure = np.random.rand(N, 2), np.random.rand(N, 2) 65 | 66 | return noisy_circles, noisy_moons, blobs, gaussian_quantiles, no_structure -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/3.Shallow Neural Networks/testCases.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def layer_sizes_test_case(): 4 | np.random.seed(1) 5 | X_assess = np.random.randn(5, 3) 6 | Y_assess = np.random.randn(2, 3) 7 | return X_assess, Y_assess 8 | 9 | def initialize_parameters_test_case(): 10 | n_x, n_h, n_y = 2, 4, 1 11 | return n_x, n_h, n_y 12 | 13 | def forward_propagation_test_case(): 14 | np.random.seed(1) 15 | X_assess = np.random.randn(2, 3) 16 | 17 | parameters = {'W1': np.array([[-0.00416758, -0.00056267], 18 | [-0.02136196, 0.01640271], 19 | [-0.01793436, -0.00841747], 20 | [ 0.00502881, -0.01245288]]), 21 | 'W2': np.array([[-0.01057952, -0.00909008, 0.00551454, 0.02292208]]), 22 | 'b1': np.array([[ 0.], 23 | [ 0.], 24 | [ 0.], 25 | [ 0.]]), 26 | 'b2': np.array([[ 0.]])} 27 | 28 | return X_assess, parameters 29 | 30 | def compute_cost_test_case(): 31 | np.random.seed(1) 32 | Y_assess = np.random.randn(1, 3) 33 | parameters = {'W1': np.array([[-0.00416758, -0.00056267], 34 | [-0.02136196, 0.01640271], 35 | [-0.01793436, -0.00841747], 36 | [ 0.00502881, -0.01245288]]), 37 | 'W2': np.array([[-0.01057952, -0.00909008, 0.00551454, 0.02292208]]), 38 | 'b1': np.array([[ 0.], 39 | [ 0.], 40 | [ 0.], 41 | [ 0.]]), 42 | 'b2': np.array([[ 0.]])} 43 | 44 | a2 = (np.array([[ 0.5002307 , 0.49985831, 0.50023963]])) 45 | 46 | return a2, Y_assess, parameters 47 | 48 | def backward_propagation_test_case(): 49 | np.random.seed(1) 50 | X_assess = np.random.randn(2, 3) 51 | Y_assess = np.random.randn(1, 3) 52 | parameters = {'W1': np.array([[-0.00416758, -0.00056267], 53 | [-0.02136196, 0.01640271], 54 | [-0.01793436, -0.00841747], 55 | [ 0.00502881, -0.01245288]]), 56 | 'W2': np.array([[-0.01057952, -0.00909008, 0.00551454, 0.02292208]]), 57 | 'b1': np.array([[ 0.], 58 | [ 0.], 59 | [ 0.], 60 | [ 0.]]), 61 | 'b2': np.array([[ 0.]])} 62 | 63 | cache = {'A1': np.array([[-0.00616578, 0.0020626 , 0.00349619], 64 | [-0.05225116, 0.02725659, -0.02646251], 65 | [-0.02009721, 0.0036869 , 0.02883756], 66 | [ 0.02152675, -0.01385234, 0.02599885]]), 67 | 'A2': np.array([[ 0.5002307 , 0.49985831, 0.50023963]]), 68 | 'Z1': np.array([[-0.00616586, 0.0020626 , 0.0034962 ], 69 | [-0.05229879, 0.02726335, -0.02646869], 70 | [-0.02009991, 0.00368692, 0.02884556], 71 | [ 0.02153007, -0.01385322, 0.02600471]]), 72 | 'Z2': np.array([[ 0.00092281, -0.00056678, 0.00095853]])} 73 | return parameters, cache, X_assess, Y_assess 74 | 75 | def update_parameters_test_case(): 76 | parameters = {'W1': np.array([[-0.00615039, 0.0169021 ], 77 | [-0.02311792, 0.03137121], 78 | [-0.0169217 , -0.01752545], 79 | [ 0.00935436, -0.05018221]]), 80 | 'W2': np.array([[-0.0104319 , -0.04019007, 0.01607211, 0.04440255]]), 81 | 'b1': np.array([[ -8.97523455e-07], 82 | [ 8.15562092e-06], 83 | [ 6.04810633e-07], 84 | [ -2.54560700e-06]]), 85 | 'b2': np.array([[ 9.14954378e-05]])} 86 | 87 | grads = {'dW1': np.array([[ 0.00023322, -0.00205423], 88 | [ 0.00082222, -0.00700776], 89 | [-0.00031831, 0.0028636 ], 90 | [-0.00092857, 0.00809933]]), 91 | 'dW2': np.array([[ -1.75740039e-05, 3.70231337e-03, -1.25683095e-03, 92 | -2.55715317e-03]]), 93 | 'db1': np.array([[ 1.05570087e-07], 94 | [ -3.81814487e-06], 95 | [ -1.90155145e-07], 96 | [ 5.46467802e-07]]), 97 | 'db2': np.array([[ -1.08923140e-05]])} 98 | return parameters, grads 99 | 100 | def nn_model_test_case(): 101 | np.random.seed(1) 102 | X_assess = np.random.randn(2, 3) 103 | Y_assess = np.random.randn(1, 3) 104 | return X_assess, Y_assess 105 | 106 | def predict_test_case(): 107 | np.random.seed(1) 108 | X_assess = np.random.randn(2, 3) 109 | parameters = {'W1': np.array([[-0.00615039, 0.0169021 ], 110 | [-0.02311792, 0.03137121], 111 | [-0.0169217 , -0.01752545], 112 | [ 0.00935436, -0.05018221]]), 113 | 'W2': np.array([[-0.0104319 , -0.04019007, 0.01607211, 0.04440255]]), 114 | 'b1': np.array([[ -8.97523455e-07], 115 | [ 8.15562092e-06], 116 | [ 6.04810633e-07], 117 | [ -2.54560700e-06]]), 118 | 'b2': np.array([[ 9.14954378e-05]])} 119 | return parameters, X_assess 120 | -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/4.Deep Neural Networks/__pycache__/dnn_app_utils_v2.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/4.Deep Neural Networks/__pycache__/dnn_app_utils_v2.cpython-36.pyc -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/4.Deep Neural Networks/__pycache__/dnn_utils_v2.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/4.Deep Neural Networks/__pycache__/dnn_utils_v2.cpython-36.pyc -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/4.Deep Neural Networks/__pycache__/testCases_v2.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/4.Deep Neural Networks/__pycache__/testCases_v2.cpython-36.pyc -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/4.Deep Neural Networks/datasets/test_catvnoncat.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/4.Deep Neural Networks/datasets/test_catvnoncat.h5 -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/4.Deep Neural Networks/datasets/train_catvnoncat.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/4.Deep Neural Networks/datasets/train_catvnoncat.h5 -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/4.Deep Neural Networks/dnn_utils_v2.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def sigmoid(Z): 4 | """ 5 | Implements the sigmoid activation in numpy 6 | 7 | Arguments: 8 | Z -- numpy array of any shape 9 | 10 | Returns: 11 | A -- output of sigmoid(z), same shape as Z 12 | cache -- returns Z as well, useful during backpropagation 13 | """ 14 | 15 | A = 1/(1+np.exp(-Z)) 16 | cache = Z 17 | 18 | return A, cache 19 | 20 | def relu(Z): 21 | """ 22 | Implement the RELU function. 23 | 24 | Arguments: 25 | Z -- Output of the linear layer, of any shape 26 | 27 | Returns: 28 | A -- Post-activation parameter, of the same shape as Z 29 | cache -- a python dictionary containing "A" ; stored for computing the backward pass efficiently 30 | """ 31 | 32 | A = np.maximum(0,Z) 33 | 34 | assert(A.shape == Z.shape) 35 | 36 | cache = Z 37 | return A, cache 38 | 39 | 40 | def relu_backward(dA, cache): 41 | """ 42 | Implement the backward propagation for a single RELU unit. 43 | 44 | Arguments: 45 | dA -- post-activation gradient, of any shape 46 | cache -- 'Z' where we store for computing backward propagation efficiently 47 | 48 | Returns: 49 | dZ -- Gradient of the cost with respect to Z 50 | """ 51 | 52 | Z = cache 53 | dZ = np.array(dA, copy=True) # just converting dz to a correct object. 54 | 55 | # When z <= 0, you should set dz to 0 as well. 56 | dZ[Z <= 0] = 0 57 | 58 | assert (dZ.shape == Z.shape) 59 | 60 | return dZ 61 | 62 | def sigmoid_backward(dA, cache): 63 | """ 64 | Implement the backward propagation for a single SIGMOID unit. 65 | 66 | Arguments: 67 | dA -- post-activation gradient, of any shape 68 | cache -- 'Z' where we store for computing backward propagation efficiently 69 | 70 | Returns: 71 | dZ -- Gradient of the cost with respect to Z 72 | """ 73 | 74 | Z = cache 75 | 76 | s = 1/(1+np.exp(-Z)) 77 | dZ = dA * s * (1-s) 78 | 79 | assert (dZ.shape == Z.shape) 80 | 81 | return dZ 82 | 83 | -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/4.Deep Neural Networks/my_image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/1.Neural Networks and Deep Learning/4.Deep Neural Networks/my_image.jpg -------------------------------------------------------------------------------- /1.Neural Networks and Deep Learning/4.Deep Neural Networks/testCases_v2.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def linear_forward_test_case(): 4 | np.random.seed(1) 5 | """ 6 | X = np.array([[-1.02387576, 1.12397796], 7 | [-1.62328545, 0.64667545], 8 | [-1.74314104, -0.59664964]]) 9 | W = np.array([[ 0.74505627, 1.97611078, -1.24412333]]) 10 | b = np.array([[1]]) 11 | """ 12 | A = np.random.randn(3,2) 13 | W = np.random.randn(1,3) 14 | b = np.random.randn(1,1) 15 | 16 | return A, W, b 17 | 18 | def linear_activation_forward_test_case(): 19 | """ 20 | X = np.array([[-1.02387576, 1.12397796], 21 | [-1.62328545, 0.64667545], 22 | [-1.74314104, -0.59664964]]) 23 | W = np.array([[ 0.74505627, 1.97611078, -1.24412333]]) 24 | b = 5 25 | """ 26 | np.random.seed(2) 27 | A_prev = np.random.randn(3,2) 28 | W = np.random.randn(1,3) 29 | b = np.random.randn(1,1) 30 | return A_prev, W, b 31 | 32 | def L_model_forward_test_case(): 33 | """ 34 | X = np.array([[-1.02387576, 1.12397796], 35 | [-1.62328545, 0.64667545], 36 | [-1.74314104, -0.59664964]]) 37 | parameters = {'W1': np.array([[ 1.62434536, -0.61175641, -0.52817175], 38 | [-1.07296862, 0.86540763, -2.3015387 ]]), 39 | 'W2': np.array([[ 1.74481176, -0.7612069 ]]), 40 | 'b1': np.array([[ 0.], 41 | [ 0.]]), 42 | 'b2': np.array([[ 0.]])} 43 | """ 44 | np.random.seed(1) 45 | X = np.random.randn(4,2) 46 | W1 = np.random.randn(3,4) 47 | b1 = np.random.randn(3,1) 48 | W2 = np.random.randn(1,3) 49 | b2 = np.random.randn(1,1) 50 | parameters = {"W1": W1, 51 | "b1": b1, 52 | "W2": W2, 53 | "b2": b2} 54 | 55 | return X, parameters 56 | 57 | def compute_cost_test_case(): 58 | Y = np.asarray([[1, 1, 1]]) 59 | aL = np.array([[.8,.9,0.4]]) 60 | 61 | return Y, aL 62 | 63 | def linear_backward_test_case(): 64 | """ 65 | z, linear_cache = (np.array([[-0.8019545 , 3.85763489]]), (np.array([[-1.02387576, 1.12397796], 66 | [-1.62328545, 0.64667545], 67 | [-1.74314104, -0.59664964]]), np.array([[ 0.74505627, 1.97611078, -1.24412333]]), np.array([[1]])) 68 | """ 69 | np.random.seed(1) 70 | dZ = np.random.randn(1,2) 71 | A = np.random.randn(3,2) 72 | W = np.random.randn(1,3) 73 | b = np.random.randn(1,1) 74 | linear_cache = (A, W, b) 75 | return dZ, linear_cache 76 | 77 | def linear_activation_backward_test_case(): 78 | """ 79 | aL, linear_activation_cache = (np.array([[ 3.1980455 , 7.85763489]]), ((np.array([[-1.02387576, 1.12397796], [-1.62328545, 0.64667545], [-1.74314104, -0.59664964]]), np.array([[ 0.74505627, 1.97611078, -1.24412333]]), 5), np.array([[ 3.1980455 , 7.85763489]]))) 80 | """ 81 | np.random.seed(2) 82 | dA = np.random.randn(1,2) 83 | A = np.random.randn(3,2) 84 | W = np.random.randn(1,3) 85 | b = np.random.randn(1,1) 86 | Z = np.random.randn(1,2) 87 | linear_cache = (A, W, b) 88 | activation_cache = Z 89 | linear_activation_cache = (linear_cache, activation_cache) 90 | 91 | return dA, linear_activation_cache 92 | 93 | def L_model_backward_test_case(): 94 | """ 95 | X = np.random.rand(3,2) 96 | Y = np.array([[1, 1]]) 97 | parameters = {'W1': np.array([[ 1.78862847, 0.43650985, 0.09649747]]), 'b1': np.array([[ 0.]])} 98 | 99 | aL, caches = (np.array([[ 0.60298372, 0.87182628]]), [((np.array([[ 0.20445225, 0.87811744], 100 | [ 0.02738759, 0.67046751], 101 | [ 0.4173048 , 0.55868983]]), 102 | np.array([[ 1.78862847, 0.43650985, 0.09649747]]), 103 | np.array([[ 0.]])), 104 | np.array([[ 0.41791293, 1.91720367]]))]) 105 | """ 106 | np.random.seed(3) 107 | AL = np.random.randn(1, 2) 108 | Y = np.array([[1, 0]]) 109 | 110 | A1 = np.random.randn(4,2) 111 | W1 = np.random.randn(3,4) 112 | b1 = np.random.randn(3,1) 113 | Z1 = np.random.randn(3,2) 114 | linear_cache_activation_1 = ((A1, W1, b1), Z1) 115 | 116 | A2 = np.random.randn(3,2) 117 | W2 = np.random.randn(1,3) 118 | b2 = np.random.randn(1,1) 119 | Z2 = np.random.randn(1,2) 120 | linear_cache_activation_2 = ( (A2, W2, b2), Z2) 121 | 122 | caches = (linear_cache_activation_1, linear_cache_activation_2) 123 | 124 | return AL, Y, caches 125 | 126 | def update_parameters_test_case(): 127 | """ 128 | parameters = {'W1': np.array([[ 1.78862847, 0.43650985, 0.09649747], 129 | [-1.8634927 , -0.2773882 , -0.35475898], 130 | [-0.08274148, -0.62700068, -0.04381817], 131 | [-0.47721803, -1.31386475, 0.88462238]]), 132 | 'W2': np.array([[ 0.88131804, 1.70957306, 0.05003364, -0.40467741], 133 | [-0.54535995, -1.54647732, 0.98236743, -1.10106763], 134 | [-1.18504653, -0.2056499 , 1.48614836, 0.23671627]]), 135 | 'W3': np.array([[-1.02378514, -0.7129932 , 0.62524497], 136 | [-0.16051336, -0.76883635, -0.23003072]]), 137 | 'b1': np.array([[ 0.], 138 | [ 0.], 139 | [ 0.], 140 | [ 0.]]), 141 | 'b2': np.array([[ 0.], 142 | [ 0.], 143 | [ 0.]]), 144 | 'b3': np.array([[ 0.], 145 | [ 0.]])} 146 | grads = {'dW1': np.array([[ 0.63070583, 0.66482653, 0.18308507], 147 | [ 0. , 0. , 0. ], 148 | [ 0. , 0. , 0. ], 149 | [ 0. , 0. , 0. ]]), 150 | 'dW2': np.array([[ 1.62934255, 0. , 0. , 0. ], 151 | [ 0. , 0. , 0. , 0. ], 152 | [ 0. , 0. , 0. , 0. ]]), 153 | 'dW3': np.array([[-1.40260776, 0. , 0. ]]), 154 | 'da1': np.array([[ 0.70760786, 0.65063504], 155 | [ 0.17268975, 0.15878569], 156 | [ 0.03817582, 0.03510211]]), 157 | 'da2': np.array([[ 0.39561478, 0.36376198], 158 | [ 0.7674101 , 0.70562233], 159 | [ 0.0224596 , 0.02065127], 160 | [-0.18165561, -0.16702967]]), 161 | 'da3': np.array([[ 0.44888991, 0.41274769], 162 | [ 0.31261975, 0.28744927], 163 | [-0.27414557, -0.25207283]]), 164 | 'db1': 0.75937676204411464, 165 | 'db2': 0.86163759922811056, 166 | 'db3': -0.84161956022334572} 167 | """ 168 | np.random.seed(2) 169 | W1 = np.random.randn(3,4) 170 | b1 = np.random.randn(3,1) 171 | W2 = np.random.randn(1,3) 172 | b2 = np.random.randn(1,1) 173 | parameters = {"W1": W1, 174 | "b1": b1, 175 | "W2": W2, 176 | "b2": b2} 177 | np.random.seed(3) 178 | dW1 = np.random.randn(3,4) 179 | db1 = np.random.randn(3,1) 180 | dW2 = np.random.randn(1,3) 181 | db2 = np.random.randn(1,1) 182 | grads = {"dW1": dW1, 183 | "db1": db1, 184 | "dW2": dW2, 185 | "db2": db2} 186 | 187 | return parameters, grads 188 | -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/Gradient Checking.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Gradient Checking" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Packages" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import numpy as np\n", 24 | "from testCases import *\n", 25 | "from gc_utils import *" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "metadata": {}, 31 | "source": [ 32 | "## Gradient checking" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "### Forward propagation" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "def forward_propagation(x, theta):\n", 49 | " J = np.dot(theta, x)\n", 50 | " \n", 51 | " return J" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 3, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "name": "stdout", 61 | "output_type": "stream", 62 | "text": [ 63 | "J = 8\n" 64 | ] 65 | } 66 | ], 67 | "source": [ 68 | "x, theta = 2, 4\n", 69 | "J = forward_propagation(x, theta)\n", 70 | "print (\"J = \" + str(J))" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "### Backward propagation" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 4, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "def backward_propagation(x, theta):\n", 87 | " dtheta = x\n", 88 | " \n", 89 | " return dtheta" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 5, 95 | "metadata": {}, 96 | "outputs": [ 97 | { 98 | "name": "stdout", 99 | "output_type": "stream", 100 | "text": [ 101 | "dtheta = 2\n" 102 | ] 103 | } 104 | ], 105 | "source": [ 106 | "x, theta = 2, 4\n", 107 | "dtheta = backward_propagation(x, theta)\n", 108 | "print (\"dtheta = \" + str(dtheta))" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "### Gradient checking" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 6, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "def gradient_check(x, theta, epsilon=1e-7):\n", 125 | " thetaplus = theta + epsilon\n", 126 | " thetaminus = theta - epsilon\n", 127 | " \n", 128 | " J_plus = forward_propagation(x, thetaplus)\n", 129 | " J_minus = forward_propagation(x, thetaminus)\n", 130 | " \n", 131 | " gradapprox = (J_plus - J_minus) / (2 * epsilon)\n", 132 | " \n", 133 | " # Check if gradapprox is close enough to backward propagation\n", 134 | " grad = backward_propagation(x, theta)\n", 135 | " \n", 136 | " numerator = np.linalg.norm(grad - gradapprox)\n", 137 | " denominator = np.linalg.norm(grad) + np.linalg.norm(gradapprox)\n", 138 | " difference = numerator / denominator\n", 139 | " \n", 140 | " if difference < 1e-7:\n", 141 | " print('The gradient is correct')\n", 142 | " else:\n", 143 | " print('The gradient is wrong')\n", 144 | " \n", 145 | " return difference" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 7, 151 | "metadata": {}, 152 | "outputs": [ 153 | { 154 | "name": "stdout", 155 | "output_type": "stream", 156 | "text": [ 157 | "The gradient is correct\n", 158 | "difference = 2.919335883291695e-10\n" 159 | ] 160 | } 161 | ], 162 | "source": [ 163 | "x, theta = 2, 4\n", 164 | "difference = gradient_check(x, theta)\n", 165 | "print(\"difference = \" + str(difference))" 166 | ] 167 | } 168 | ], 169 | "metadata": { 170 | "kernelspec": { 171 | "display_name": "Python 3", 172 | "language": "python", 173 | "name": "python3" 174 | }, 175 | "language_info": { 176 | "codemirror_mode": { 177 | "name": "ipython", 178 | "version": 3 179 | }, 180 | "file_extension": ".py", 181 | "mimetype": "text/x-python", 182 | "name": "python", 183 | "nbconvert_exporter": "python", 184 | "pygments_lexer": "ipython3", 185 | "version": "3.6.6" 186 | } 187 | }, 188 | "nbformat": 4, 189 | "nbformat_minor": 2 190 | } 191 | -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/__pycache__/gc_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/__pycache__/gc_utils.cpython-36.pyc -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/__pycache__/init_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/__pycache__/init_utils.cpython-36.pyc -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/__pycache__/reg_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/__pycache__/reg_utils.cpython-36.pyc -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/__pycache__/testCases.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/__pycache__/testCases.cpython-36.pyc -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/datasets/data.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/datasets/data.mat -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/datasets/test_catvnoncat.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/datasets/test_catvnoncat.h5 -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/datasets/train_catvnoncat.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/datasets/train_catvnoncat.h5 -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/gc_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def sigmoid(x): 4 | """ 5 | Compute the sigmoid of x 6 | 7 | Arguments: 8 | x -- A scalar or numpy array of any size. 9 | 10 | Return: 11 | s -- sigmoid(x) 12 | """ 13 | s = 1/(1+np.exp(-x)) 14 | return s 15 | 16 | def relu(x): 17 | """ 18 | Compute the relu of x 19 | 20 | Arguments: 21 | x -- A scalar or numpy array of any size. 22 | 23 | Return: 24 | s -- relu(x) 25 | """ 26 | s = np.maximum(0,x) 27 | 28 | return s 29 | 30 | def dictionary_to_vector(parameters): 31 | """ 32 | Roll all our parameters dictionary into a single vector satisfying our specific required shape. 33 | """ 34 | keys = [] 35 | count = 0 36 | for key in ["W1", "b1", "W2", "b2", "W3", "b3"]: 37 | 38 | # flatten parameter 39 | new_vector = np.reshape(parameters[key], (-1,1)) 40 | keys = keys + [key]*new_vector.shape[0] 41 | 42 | if count == 0: 43 | theta = new_vector 44 | else: 45 | theta = np.concatenate((theta, new_vector), axis=0) 46 | count = count + 1 47 | 48 | return theta, keys 49 | 50 | def vector_to_dictionary(theta): 51 | """ 52 | Unroll all our parameters dictionary from a single vector satisfying our specific required shape. 53 | """ 54 | parameters = {} 55 | parameters["W1"] = theta[:20].reshape((5,4)) 56 | parameters["b1"] = theta[20:25].reshape((5,1)) 57 | parameters["W2"] = theta[25:40].reshape((3,5)) 58 | parameters["b2"] = theta[40:43].reshape((3,1)) 59 | parameters["W3"] = theta[43:46].reshape((1,3)) 60 | parameters["b3"] = theta[46:47].reshape((1,1)) 61 | 62 | return parameters 63 | 64 | def gradients_to_vector(gradients): 65 | """ 66 | Roll all our gradients dictionary into a single vector satisfying our specific required shape. 67 | """ 68 | 69 | count = 0 70 | for key in ["dW1", "db1", "dW2", "db2", "dW3", "db3"]: 71 | # flatten parameter 72 | new_vector = np.reshape(gradients[key], (-1,1)) 73 | 74 | if count == 0: 75 | theta = new_vector 76 | else: 77 | theta = np.concatenate((theta, new_vector), axis=0) 78 | count = count + 1 79 | 80 | return theta -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/init_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import h5py 4 | import sklearn 5 | import sklearn.datasets 6 | 7 | def sigmoid(x): 8 | """ 9 | Compute the sigmoid of x 10 | 11 | Arguments: 12 | x -- A scalar or numpy array of any size. 13 | 14 | Return: 15 | s -- sigmoid(x) 16 | """ 17 | s = 1/(1+np.exp(-x)) 18 | return s 19 | 20 | def relu(x): 21 | """ 22 | Compute the relu of x 23 | 24 | Arguments: 25 | x -- A scalar or numpy array of any size. 26 | 27 | Return: 28 | s -- relu(x) 29 | """ 30 | s = np.maximum(0,x) 31 | 32 | return s 33 | 34 | def forward_propagation(X, parameters): 35 | """ 36 | Implements the forward propagation (and computes the loss) presented in Figure 2. 37 | 38 | Arguments: 39 | X -- input dataset, of shape (input size, number of examples) 40 | Y -- true "label" vector (containing 0 if cat, 1 if non-cat) 41 | parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3": 42 | W1 -- weight matrix of shape () 43 | b1 -- bias vector of shape () 44 | W2 -- weight matrix of shape () 45 | b2 -- bias vector of shape () 46 | W3 -- weight matrix of shape () 47 | b3 -- bias vector of shape () 48 | 49 | Returns: 50 | loss -- the loss function (vanilla logistic loss) 51 | """ 52 | 53 | # retrieve parameters 54 | W1 = parameters["W1"] 55 | b1 = parameters["b1"] 56 | W2 = parameters["W2"] 57 | b2 = parameters["b2"] 58 | W3 = parameters["W3"] 59 | b3 = parameters["b3"] 60 | 61 | # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID 62 | z1 = np.dot(W1, X) + b1 63 | a1 = relu(z1) 64 | z2 = np.dot(W2, a1) + b2 65 | a2 = relu(z2) 66 | z3 = np.dot(W3, a2) + b3 67 | a3 = sigmoid(z3) 68 | 69 | cache = (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3) 70 | 71 | return a3, cache 72 | 73 | def backward_propagation(X, Y, cache): 74 | """ 75 | Implement the backward propagation presented in figure 2. 76 | 77 | Arguments: 78 | X -- input dataset, of shape (input size, number of examples) 79 | Y -- true "label" vector (containing 0 if cat, 1 if non-cat) 80 | cache -- cache output from forward_propagation() 81 | 82 | Returns: 83 | gradients -- A dictionary with the gradients with respect to each parameter, activation and pre-activation variables 84 | """ 85 | m = X.shape[1] 86 | (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3) = cache 87 | 88 | dz3 = 1./m * (a3 - Y) 89 | dW3 = np.dot(dz3, a2.T) 90 | db3 = np.sum(dz3, axis=1, keepdims = True) 91 | 92 | da2 = np.dot(W3.T, dz3) 93 | dz2 = np.multiply(da2, np.int64(a2 > 0)) 94 | dW2 = np.dot(dz2, a1.T) 95 | db2 = np.sum(dz2, axis=1, keepdims = True) 96 | 97 | da1 = np.dot(W2.T, dz2) 98 | dz1 = np.multiply(da1, np.int64(a1 > 0)) 99 | dW1 = np.dot(dz1, X.T) 100 | db1 = np.sum(dz1, axis=1, keepdims = True) 101 | 102 | gradients = {"dz3": dz3, "dW3": dW3, "db3": db3, 103 | "da2": da2, "dz2": dz2, "dW2": dW2, "db2": db2, 104 | "da1": da1, "dz1": dz1, "dW1": dW1, "db1": db1} 105 | 106 | return gradients 107 | 108 | def update_parameters(parameters, grads, learning_rate): 109 | """ 110 | Update parameters using gradient descent 111 | 112 | Arguments: 113 | parameters -- python dictionary containing your parameters 114 | grads -- python dictionary containing your gradients, output of n_model_backward 115 | 116 | Returns: 117 | parameters -- python dictionary containing your updated parameters 118 | parameters['W' + str(i)] = ... 119 | parameters['b' + str(i)] = ... 120 | """ 121 | 122 | L = len(parameters) // 2 # number of layers in the neural networks 123 | 124 | # Update rule for each parameter 125 | for k in range(L): 126 | parameters["W" + str(k+1)] = parameters["W" + str(k+1)] - learning_rate * grads["dW" + str(k+1)] 127 | parameters["b" + str(k+1)] = parameters["b" + str(k+1)] - learning_rate * grads["db" + str(k+1)] 128 | 129 | return parameters 130 | 131 | def compute_loss(a3, Y): 132 | 133 | """ 134 | Implement the loss function 135 | 136 | Arguments: 137 | a3 -- post-activation, output of forward propagation 138 | Y -- "true" labels vector, same shape as a3 139 | 140 | Returns: 141 | loss - value of the loss function 142 | """ 143 | 144 | m = Y.shape[1] 145 | logprobs = np.multiply(-np.log(a3),Y) + np.multiply(-np.log(1 - a3), 1 - Y) 146 | loss = 1./m * np.nansum(logprobs) 147 | 148 | return loss 149 | 150 | def load_cat_dataset(): 151 | train_dataset = h5py.File('datasets/train_catvnoncat.h5', "r") 152 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 153 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 154 | 155 | test_dataset = h5py.File('datasets/test_catvnoncat.h5', "r") 156 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 157 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 158 | 159 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 160 | 161 | train_set_y = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 162 | test_set_y = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 163 | 164 | train_set_x_orig = train_set_x_orig.reshape(train_set_x_orig.shape[0], -1).T 165 | test_set_x_orig = test_set_x_orig.reshape(test_set_x_orig.shape[0], -1).T 166 | 167 | train_set_x = train_set_x_orig/255 168 | test_set_x = test_set_x_orig/255 169 | 170 | return train_set_x, train_set_y, test_set_x, test_set_y, classes 171 | 172 | 173 | def predict(X, y, parameters): 174 | """ 175 | This function is used to predict the results of a n-layer neural network. 176 | 177 | Arguments: 178 | X -- data set of examples you would like to label 179 | parameters -- parameters of the trained model 180 | 181 | Returns: 182 | p -- predictions for the given dataset X 183 | """ 184 | 185 | m = X.shape[1] 186 | p = np.zeros((1,m), dtype = np.int) 187 | 188 | # Forward propagation 189 | a3, caches = forward_propagation(X, parameters) 190 | 191 | # convert probas to 0/1 predictions 192 | for i in range(0, a3.shape[1]): 193 | if a3[0,i] > 0.5: 194 | p[0,i] = 1 195 | else: 196 | p[0,i] = 0 197 | 198 | # print results 199 | print("Accuracy: " + str(np.mean((p[0,:] == y[0,:])))) 200 | 201 | return p 202 | 203 | def plot_decision_boundary(model, X, y): 204 | # Set min and max values and give it some padding 205 | x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1 206 | y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1 207 | h = 0.01 208 | # Generate a grid of points with distance h between them 209 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) 210 | # Predict the function value for the whole grid 211 | Z = model(np.c_[xx.ravel(), yy.ravel()]) 212 | Z = Z.reshape(xx.shape) 213 | # Plot the contour and training examples 214 | plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral) 215 | plt.ylabel('x2') 216 | plt.xlabel('x1') 217 | plt.scatter(X[0, :], X[1, :], c=y.ravel(), cmap=plt.cm.Spectral) 218 | plt.show() 219 | 220 | def predict_dec(parameters, X): 221 | """ 222 | Used for plotting decision boundary. 223 | 224 | Arguments: 225 | parameters -- python dictionary containing your parameters 226 | X -- input data of size (m, K) 227 | 228 | Returns 229 | predictions -- vector of predictions of our model (red: 0 / blue: 1) 230 | """ 231 | 232 | # Predict using forward propagation and a classification threshold of 0.5 233 | a3, cache = forward_propagation(X, parameters) 234 | predictions = (a3>0.5) 235 | return predictions 236 | 237 | def load_dataset(): 238 | np.random.seed(1) 239 | train_X, train_Y = sklearn.datasets.make_circles(n_samples=300, noise=.05) 240 | np.random.seed(2) 241 | test_X, test_Y = sklearn.datasets.make_circles(n_samples=100, noise=.05) 242 | # Visualize the data 243 | plt.scatter(train_X[:, 0], train_X[:, 1], c=train_Y, s=40, cmap=plt.cm.Spectral); 244 | train_X = train_X.T 245 | train_Y = train_Y.reshape((1, train_Y.shape[0])) 246 | test_X = test_X.T 247 | test_Y = test_Y.reshape((1, test_Y.shape[0])) 248 | return train_X, train_Y, test_X, test_Y -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/reg_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import h5py 4 | import sklearn 5 | import sklearn.datasets 6 | import sklearn.linear_model 7 | import scipy.io 8 | 9 | def sigmoid(x): 10 | """ 11 | Compute the sigmoid of x 12 | 13 | Arguments: 14 | x -- A scalar or numpy array of any size. 15 | 16 | Return: 17 | s -- sigmoid(x) 18 | """ 19 | s = 1/(1+np.exp(-x)) 20 | return s 21 | 22 | def relu(x): 23 | """ 24 | Compute the relu of x 25 | 26 | Arguments: 27 | x -- A scalar or numpy array of any size. 28 | 29 | Return: 30 | s -- relu(x) 31 | """ 32 | s = np.maximum(0,x) 33 | 34 | return s 35 | 36 | def load_planar_dataset(seed): 37 | 38 | np.random.seed(seed) 39 | 40 | m = 400 # number of examples 41 | N = int(m/2) # number of points per class 42 | D = 2 # dimensionality 43 | X = np.zeros((m,D)) # data matrix where each row is a single example 44 | Y = np.zeros((m,1), dtype='uint8') # labels vector (0 for red, 1 for blue) 45 | a = 4 # maximum ray of the flower 46 | 47 | for j in range(2): 48 | ix = range(N*j,N*(j+1)) 49 | t = np.linspace(j*3.12,(j+1)*3.12,N) + np.random.randn(N)*0.2 # theta 50 | r = a*np.sin(4*t) + np.random.randn(N)*0.2 # radius 51 | X[ix] = np.c_[r*np.sin(t), r*np.cos(t)] 52 | Y[ix] = j 53 | 54 | X = X.T 55 | Y = Y.T 56 | 57 | return X, Y 58 | 59 | def initialize_parameters(layer_dims): 60 | """ 61 | Arguments: 62 | layer_dims -- python array (list) containing the dimensions of each layer in our network 63 | 64 | Returns: 65 | parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL": 66 | W1 -- weight matrix of shape (layer_dims[l], layer_dims[l-1]) 67 | b1 -- bias vector of shape (layer_dims[l], 1) 68 | Wl -- weight matrix of shape (layer_dims[l-1], layer_dims[l]) 69 | bl -- bias vector of shape (1, layer_dims[l]) 70 | 71 | Tips: 72 | - For example: the layer_dims for the "Planar Data classification model" would have been [2,2,1]. 73 | This means W1's shape was (2,2), b1 was (1,2), W2 was (2,1) and b2 was (1,1). Now you have to generalize it! 74 | - In the for loop, use parameters['W' + str(l)] to access Wl, where l is the iterative integer. 75 | """ 76 | 77 | np.random.seed(3) 78 | parameters = {} 79 | L = len(layer_dims) # number of layers in the network 80 | 81 | for l in range(1, L): 82 | parameters['W' + str(l)] = np.random.randn(layer_dims[l], layer_dims[l-1]) / np.sqrt(layer_dims[l-1]) 83 | parameters['b' + str(l)] = np.zeros((layer_dims[l], 1)) 84 | 85 | assert(parameters['W' + str(l)].shape == (layer_dims[l], layer_dims[l-1])) 86 | assert(parameters['W' + str(l)].shape == layer_dims[l], 1) 87 | 88 | 89 | return parameters 90 | 91 | def forward_propagation(X, parameters): 92 | """ 93 | Implements the forward propagation (and computes the loss) presented in Figure 2. 94 | 95 | Arguments: 96 | X -- input dataset, of shape (input size, number of examples) 97 | parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3": 98 | W1 -- weight matrix of shape () 99 | b1 -- bias vector of shape () 100 | W2 -- weight matrix of shape () 101 | b2 -- bias vector of shape () 102 | W3 -- weight matrix of shape () 103 | b3 -- bias vector of shape () 104 | 105 | Returns: 106 | loss -- the loss function (vanilla logistic loss) 107 | """ 108 | 109 | # retrieve parameters 110 | W1 = parameters["W1"] 111 | b1 = parameters["b1"] 112 | W2 = parameters["W2"] 113 | b2 = parameters["b2"] 114 | W3 = parameters["W3"] 115 | b3 = parameters["b3"] 116 | 117 | # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID 118 | Z1 = np.dot(W1, X) + b1 119 | A1 = relu(Z1) 120 | Z2 = np.dot(W2, A1) + b2 121 | A2 = relu(Z2) 122 | Z3 = np.dot(W3, A2) + b3 123 | A3 = sigmoid(Z3) 124 | 125 | cache = (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) 126 | 127 | return A3, cache 128 | 129 | def backward_propagation(X, Y, cache): 130 | """ 131 | Implement the backward propagation presented in figure 2. 132 | 133 | Arguments: 134 | X -- input dataset, of shape (input size, number of examples) 135 | Y -- true "label" vector (containing 0 if cat, 1 if non-cat) 136 | cache -- cache output from forward_propagation() 137 | 138 | Returns: 139 | gradients -- A dictionary with the gradients with respect to each parameter, activation and pre-activation variables 140 | """ 141 | m = X.shape[1] 142 | (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache 143 | 144 | dZ3 = A3 - Y 145 | dW3 = 1./m * np.dot(dZ3, A2.T) 146 | db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True) 147 | 148 | dA2 = np.dot(W3.T, dZ3) 149 | dZ2 = np.multiply(dA2, np.int64(A2 > 0)) 150 | dW2 = 1./m * np.dot(dZ2, A1.T) 151 | db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True) 152 | 153 | dA1 = np.dot(W2.T, dZ2) 154 | dZ1 = np.multiply(dA1, np.int64(A1 > 0)) 155 | dW1 = 1./m * np.dot(dZ1, X.T) 156 | db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True) 157 | 158 | gradients = {"dZ3": dZ3, "dW3": dW3, "db3": db3, 159 | "dA2": dA2, "dZ2": dZ2, "dW2": dW2, "db2": db2, 160 | "dA1": dA1, "dZ1": dZ1, "dW1": dW1, "db1": db1} 161 | 162 | return gradients 163 | 164 | def update_parameters(parameters, grads, learning_rate): 165 | """ 166 | Update parameters using gradient descent 167 | 168 | Arguments: 169 | parameters -- python dictionary containing your parameters: 170 | parameters['W' + str(i)] = Wi 171 | parameters['b' + str(i)] = bi 172 | grads -- python dictionary containing your gradients for each parameters: 173 | grads['dW' + str(i)] = dWi 174 | grads['db' + str(i)] = dbi 175 | learning_rate -- the learning rate, scalar. 176 | 177 | Returns: 178 | parameters -- python dictionary containing your updated parameters 179 | """ 180 | 181 | n = len(parameters) // 2 # number of layers in the neural networks 182 | 183 | # Update rule for each parameter 184 | for k in range(n): 185 | parameters["W" + str(k+1)] = parameters["W" + str(k+1)] - learning_rate * grads["dW" + str(k+1)] 186 | parameters["b" + str(k+1)] = parameters["b" + str(k+1)] - learning_rate * grads["db" + str(k+1)] 187 | 188 | return parameters 189 | 190 | def predict(X, y, parameters): 191 | """ 192 | This function is used to predict the results of a n-layer neural network. 193 | 194 | Arguments: 195 | X -- data set of examples you would like to label 196 | parameters -- parameters of the trained model 197 | 198 | Returns: 199 | p -- predictions for the given dataset X 200 | """ 201 | 202 | m = X.shape[1] 203 | p = np.zeros((1,m), dtype = np.int) 204 | 205 | # Forward propagation 206 | a3, caches = forward_propagation(X, parameters) 207 | 208 | # convert probas to 0/1 predictions 209 | for i in range(0, a3.shape[1]): 210 | if a3[0,i] > 0.5: 211 | p[0,i] = 1 212 | else: 213 | p[0,i] = 0 214 | 215 | # print results 216 | 217 | #print ("predictions: " + str(p[0,:])) 218 | #print ("true labels: " + str(y[0,:])) 219 | print("Accuracy: " + str(np.mean((p[0,:] == y[0,:])))) 220 | 221 | return p 222 | 223 | def compute_cost(a3, Y): 224 | """ 225 | Implement the cost function 226 | 227 | Arguments: 228 | a3 -- post-activation, output of forward propagation 229 | Y -- "true" labels vector, same shape as a3 230 | 231 | Returns: 232 | cost - value of the cost function 233 | """ 234 | m = Y.shape[1] 235 | 236 | logprobs = np.multiply(-np.log(a3),Y) + np.multiply(-np.log(1 - a3), 1 - Y) 237 | cost = 1./m * np.nansum(logprobs) 238 | 239 | return cost 240 | 241 | def load_dataset(): 242 | train_dataset = h5py.File('datasets/train_catvnoncat.h5', "r") 243 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 244 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 245 | 246 | test_dataset = h5py.File('datasets/test_catvnoncat.h5', "r") 247 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 248 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 249 | 250 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 251 | 252 | train_set_y = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 253 | test_set_y = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 254 | 255 | train_set_x_orig = train_set_x_orig.reshape(train_set_x_orig.shape[0], -1).T 256 | test_set_x_orig = test_set_x_orig.reshape(test_set_x_orig.shape[0], -1).T 257 | 258 | train_set_x = train_set_x_orig/255 259 | test_set_x = test_set_x_orig/255 260 | 261 | return train_set_x, train_set_y, test_set_x, test_set_y, classes 262 | 263 | 264 | def predict_dec(parameters, X): 265 | """ 266 | Used for plotting decision boundary. 267 | 268 | Arguments: 269 | parameters -- python dictionary containing your parameters 270 | X -- input data of size (m, K) 271 | 272 | Returns 273 | predictions -- vector of predictions of our model (red: 0 / blue: 1) 274 | """ 275 | 276 | # Predict using forward propagation and a classification threshold of 0.5 277 | a3, cache = forward_propagation(X, parameters) 278 | predictions = (a3>0.5) 279 | return predictions 280 | 281 | def load_planar_dataset(randomness, seed): 282 | 283 | np.random.seed(seed) 284 | 285 | m = 50 286 | N = int(m/2) # number of points per class 287 | D = 2 # dimensionality 288 | X = np.zeros((m,D)) # data matrix where each row is a single example 289 | Y = np.zeros((m,1), dtype='uint8') # labels vector (0 for red, 1 for blue) 290 | a = 2 # maximum ray of the flower 291 | 292 | for j in range(2): 293 | 294 | ix = range(N*j,N*(j+1)) 295 | if j == 0: 296 | t = np.linspace(j, 4*3.1415*(j+1),N) #+ np.random.randn(N)*randomness # theta 297 | r = 0.3*np.square(t) + np.random.randn(N)*randomness # radius 298 | if j == 1: 299 | t = np.linspace(j, 2*3.1415*(j+1),N) #+ np.random.randn(N)*randomness # theta 300 | r = 0.2*np.square(t) + np.random.randn(N)*randomness # radius 301 | 302 | X[ix] = np.c_[r*np.cos(t), r*np.sin(t)] 303 | Y[ix] = j 304 | 305 | X = X.T 306 | Y = Y.T 307 | 308 | return X, Y 309 | 310 | def plot_decision_boundary(model, X, y): 311 | # Set min and max values and give it some padding 312 | x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1 313 | y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1 314 | h = 0.01 315 | # Generate a grid of points with distance h between them 316 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) 317 | # Predict the function value for the whole grid 318 | Z = model(np.c_[xx.ravel(), yy.ravel()]) 319 | Z = Z.reshape(xx.shape) 320 | # Plot the contour and training examples 321 | plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral) 322 | plt.ylabel('x2') 323 | plt.xlabel('x1') 324 | plt.scatter(X[0, :], X[1, :], c=y.ravel(), cmap=plt.cm.Spectral) 325 | plt.show() 326 | 327 | def load_2D_dataset(): 328 | data = scipy.io.loadmat('datasets/data.mat') 329 | train_X = data['X'].T 330 | train_Y = data['y'].T 331 | test_X = data['Xval'].T 332 | test_Y = data['yval'].T 333 | 334 | plt.scatter(train_X[0, :], train_X[1, :], c=train_Y.ravel(), s=40, cmap=plt.cm.Spectral); 335 | 336 | return train_X, train_Y, test_X, test_Y -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/1.Practical Aspects of Deep Learning/testCases.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def compute_cost_with_regularization_test_case(): 4 | np.random.seed(1) 5 | Y_assess = np.array([[1, 1, 0, 1, 0]]) 6 | W1 = np.random.randn(2, 3) 7 | b1 = np.random.randn(2, 1) 8 | W2 = np.random.randn(3, 2) 9 | b2 = np.random.randn(3, 1) 10 | W3 = np.random.randn(1, 3) 11 | b3 = np.random.randn(1, 1) 12 | parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2, "W3": W3, "b3": b3} 13 | a3 = np.array([[ 0.40682402, 0.01629284, 0.16722898, 0.10118111, 0.40682402]]) 14 | return a3, Y_assess, parameters 15 | 16 | def backward_propagation_with_regularization_test_case(): 17 | np.random.seed(1) 18 | X_assess = np.random.randn(3, 5) 19 | Y_assess = np.array([[1, 1, 0, 1, 0]]) 20 | cache = (np.array([[-1.52855314, 3.32524635, 2.13994541, 2.60700654, -0.75942115], 21 | [-1.98043538, 4.1600994 , 0.79051021, 1.46493512, -0.45506242]]), 22 | np.array([[ 0. , 3.32524635, 2.13994541, 2.60700654, 0. ], 23 | [ 0. , 4.1600994 , 0.79051021, 1.46493512, 0. ]]), 24 | np.array([[-1.09989127, -0.17242821, -0.87785842], 25 | [ 0.04221375, 0.58281521, -1.10061918]]), 26 | np.array([[ 1.14472371], 27 | [ 0.90159072]]), 28 | np.array([[ 0.53035547, 5.94892323, 2.31780174, 3.16005701, 0.53035547], 29 | [-0.69166075, -3.47645987, -2.25194702, -2.65416996, -0.69166075], 30 | [-0.39675353, -4.62285846, -2.61101729, -3.22874921, -0.39675353]]), 31 | np.array([[ 0.53035547, 5.94892323, 2.31780174, 3.16005701, 0.53035547], 32 | [ 0. , 0. , 0. , 0. , 0. ], 33 | [ 0. , 0. , 0. , 0. , 0. ]]), 34 | np.array([[ 0.50249434, 0.90085595], 35 | [-0.68372786, -0.12289023], 36 | [-0.93576943, -0.26788808]]), 37 | np.array([[ 0.53035547], 38 | [-0.69166075], 39 | [-0.39675353]]), 40 | np.array([[-0.3771104 , -4.10060224, -1.60539468, -2.18416951, -0.3771104 ]]), 41 | np.array([[ 0.40682402, 0.01629284, 0.16722898, 0.10118111, 0.40682402]]), 42 | np.array([[-0.6871727 , -0.84520564, -0.67124613]]), 43 | np.array([[-0.0126646]])) 44 | return X_assess, Y_assess, cache 45 | 46 | def forward_propagation_with_dropout_test_case(): 47 | np.random.seed(1) 48 | X_assess = np.random.randn(3, 5) 49 | W1 = np.random.randn(2, 3) 50 | b1 = np.random.randn(2, 1) 51 | W2 = np.random.randn(3, 2) 52 | b2 = np.random.randn(3, 1) 53 | W3 = np.random.randn(1, 3) 54 | b3 = np.random.randn(1, 1) 55 | parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2, "W3": W3, "b3": b3} 56 | 57 | return X_assess, parameters 58 | 59 | def backward_propagation_with_dropout_test_case(): 60 | np.random.seed(1) 61 | X_assess = np.random.randn(3, 5) 62 | Y_assess = np.array([[1, 1, 0, 1, 0]]) 63 | cache = (np.array([[-1.52855314, 3.32524635, 2.13994541, 2.60700654, -0.75942115], 64 | [-1.98043538, 4.1600994 , 0.79051021, 1.46493512, -0.45506242]]), np.array([[ True, False, True, True, True], 65 | [ True, True, True, True, False]], dtype=bool), np.array([[ 0. , 0. , 4.27989081, 5.21401307, 0. ], 66 | [ 0. , 8.32019881, 1.58102041, 2.92987024, 0. ]]), np.array([[-1.09989127, -0.17242821, -0.87785842], 67 | [ 0.04221375, 0.58281521, -1.10061918]]), np.array([[ 1.14472371], 68 | [ 0.90159072]]), np.array([[ 0.53035547, 8.02565606, 4.10524802, 5.78975856, 0.53035547], 69 | [-0.69166075, -1.71413186, -3.81223329, -4.61667916, -0.69166075], 70 | [-0.39675353, -2.62563561, -4.82528105, -6.0607449 , -0.39675353]]), np.array([[ True, False, True, False, True], 71 | [False, True, False, True, True], 72 | [False, False, True, False, False]], dtype=bool), np.array([[ 1.06071093, 0. , 8.21049603, 0. , 1.06071093], 73 | [ 0. , 0. , 0. , 0. , 0. ], 74 | [ 0. , 0. , 0. , 0. , 0. ]]), np.array([[ 0.50249434, 0.90085595], 75 | [-0.68372786, -0.12289023], 76 | [-0.93576943, -0.26788808]]), np.array([[ 0.53035547], 77 | [-0.69166075], 78 | [-0.39675353]]), np.array([[-0.7415562 , -0.0126646 , -5.65469333, -0.0126646 , -0.7415562 ]]), np.array([[ 0.32266394, 0.49683389, 0.00348883, 0.49683389, 0.32266394]]), np.array([[-0.6871727 , -0.84520564, -0.67124613]]), np.array([[-0.0126646]])) 79 | 80 | 81 | return X_assess, Y_assess, cache 82 | -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/2.Algorithm Optimization/__pycache__/opt_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/2.Algorithm Optimization/__pycache__/opt_utils.cpython-36.pyc -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/2.Algorithm Optimization/__pycache__/testCases.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/2.Algorithm Optimization/__pycache__/testCases.cpython-36.pyc -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/2.Algorithm Optimization/datasets/data.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/2.Algorithm Optimization/datasets/data.mat -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/2.Algorithm Optimization/opt_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import h5py 4 | import scipy.io 5 | import sklearn 6 | import sklearn.datasets 7 | 8 | def sigmoid(x): 9 | """ 10 | Compute the sigmoid of x 11 | 12 | Arguments: 13 | x -- A scalar or numpy array of any size. 14 | 15 | Return: 16 | s -- sigmoid(x) 17 | """ 18 | s = 1/(1+np.exp(-x)) 19 | return s 20 | 21 | def relu(x): 22 | """ 23 | Compute the relu of x 24 | 25 | Arguments: 26 | x -- A scalar or numpy array of any size. 27 | 28 | Return: 29 | s -- relu(x) 30 | """ 31 | s = np.maximum(0,x) 32 | 33 | return s 34 | 35 | def load_params_and_grads(seed=1): 36 | np.random.seed(seed) 37 | W1 = np.random.randn(2,3) 38 | b1 = np.random.randn(2,1) 39 | W2 = np.random.randn(3,3) 40 | b2 = np.random.randn(3,1) 41 | 42 | dW1 = np.random.randn(2,3) 43 | db1 = np.random.randn(2,1) 44 | dW2 = np.random.randn(3,3) 45 | db2 = np.random.randn(3,1) 46 | 47 | return W1, b1, W2, b2, dW1, db1, dW2, db2 48 | 49 | 50 | def initialize_parameters(layer_dims): 51 | """ 52 | Arguments: 53 | layer_dims -- python array (list) containing the dimensions of each layer in our network 54 | 55 | Returns: 56 | parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL": 57 | W1 -- weight matrix of shape (layer_dims[l], layer_dims[l-1]) 58 | b1 -- bias vector of shape (layer_dims[l], 1) 59 | Wl -- weight matrix of shape (layer_dims[l-1], layer_dims[l]) 60 | bl -- bias vector of shape (1, layer_dims[l]) 61 | 62 | Tips: 63 | - For example: the layer_dims for the "Planar Data classification model" would have been [2,2,1]. 64 | This means W1's shape was (2,2), b1 was (1,2), W2 was (2,1) and b2 was (1,1). Now you have to generalize it! 65 | - In the for loop, use parameters['W' + str(l)] to access Wl, where l is the iterative integer. 66 | """ 67 | 68 | np.random.seed(3) 69 | parameters = {} 70 | L = len(layer_dims) # number of layers in the network 71 | 72 | for l in range(1, L): 73 | parameters['W' + str(l)] = np.random.randn(layer_dims[l], layer_dims[l-1])* np.sqrt(2 / layer_dims[l-1]) 74 | parameters['b' + str(l)] = np.zeros((layer_dims[l], 1)) 75 | 76 | assert(parameters['W' + str(l)].shape == layer_dims[l], layer_dims[l-1]) 77 | assert(parameters['W' + str(l)].shape == layer_dims[l], 1) 78 | 79 | return parameters 80 | 81 | 82 | def compute_cost(a3, Y): 83 | 84 | """ 85 | Implement the cost function 86 | 87 | Arguments: 88 | a3 -- post-activation, output of forward propagation 89 | Y -- "true" labels vector, same shape as a3 90 | 91 | Returns: 92 | cost - value of the cost function 93 | """ 94 | m = Y.shape[1] 95 | 96 | logprobs = np.multiply(-np.log(a3),Y) + np.multiply(-np.log(1 - a3), 1 - Y) 97 | cost = 1./m * np.sum(logprobs) 98 | 99 | return cost 100 | 101 | def forward_propagation(X, parameters): 102 | """ 103 | Implements the forward propagation (and computes the loss) presented in Figure 2. 104 | 105 | Arguments: 106 | X -- input dataset, of shape (input size, number of examples) 107 | parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3": 108 | W1 -- weight matrix of shape () 109 | b1 -- bias vector of shape () 110 | W2 -- weight matrix of shape () 111 | b2 -- bias vector of shape () 112 | W3 -- weight matrix of shape () 113 | b3 -- bias vector of shape () 114 | 115 | Returns: 116 | loss -- the loss function (vanilla logistic loss) 117 | """ 118 | 119 | # retrieve parameters 120 | W1 = parameters["W1"] 121 | b1 = parameters["b1"] 122 | W2 = parameters["W2"] 123 | b2 = parameters["b2"] 124 | W3 = parameters["W3"] 125 | b3 = parameters["b3"] 126 | 127 | # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID 128 | z1 = np.dot(W1, X) + b1 129 | a1 = relu(z1) 130 | z2 = np.dot(W2, a1) + b2 131 | a2 = relu(z2) 132 | z3 = np.dot(W3, a2) + b3 133 | a3 = sigmoid(z3) 134 | 135 | cache = (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3) 136 | 137 | return a3, cache 138 | 139 | def backward_propagation(X, Y, cache): 140 | """ 141 | Implement the backward propagation presented in figure 2. 142 | 143 | Arguments: 144 | X -- input dataset, of shape (input size, number of examples) 145 | Y -- true "label" vector (containing 0 if cat, 1 if non-cat) 146 | cache -- cache output from forward_propagation() 147 | 148 | Returns: 149 | gradients -- A dictionary with the gradients with respect to each parameter, activation and pre-activation variables 150 | """ 151 | m = X.shape[1] 152 | (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3) = cache 153 | 154 | dz3 = 1./m * (a3 - Y) 155 | dW3 = np.dot(dz3, a2.T) 156 | db3 = np.sum(dz3, axis=1, keepdims = True) 157 | 158 | da2 = np.dot(W3.T, dz3) 159 | dz2 = np.multiply(da2, np.int64(a2 > 0)) 160 | dW2 = np.dot(dz2, a1.T) 161 | db2 = np.sum(dz2, axis=1, keepdims = True) 162 | 163 | da1 = np.dot(W2.T, dz2) 164 | dz1 = np.multiply(da1, np.int64(a1 > 0)) 165 | dW1 = np.dot(dz1, X.T) 166 | db1 = np.sum(dz1, axis=1, keepdims = True) 167 | 168 | gradients = {"dz3": dz3, "dW3": dW3, "db3": db3, 169 | "da2": da2, "dz2": dz2, "dW2": dW2, "db2": db2, 170 | "da1": da1, "dz1": dz1, "dW1": dW1, "db1": db1} 171 | 172 | return gradients 173 | 174 | def predict(X, y, parameters): 175 | """ 176 | This function is used to predict the results of a n-layer neural network. 177 | 178 | Arguments: 179 | X -- data set of examples you would like to label 180 | parameters -- parameters of the trained model 181 | 182 | Returns: 183 | p -- predictions for the given dataset X 184 | """ 185 | 186 | m = X.shape[1] 187 | p = np.zeros((1,m), dtype = np.int) 188 | 189 | # Forward propagation 190 | a3, caches = forward_propagation(X, parameters) 191 | 192 | # convert probas to 0/1 predictions 193 | for i in range(0, a3.shape[1]): 194 | if a3[0,i] > 0.5: 195 | p[0,i] = 1 196 | else: 197 | p[0,i] = 0 198 | 199 | # print results 200 | 201 | #print ("predictions: " + str(p[0,:])) 202 | #print ("true labels: " + str(y[0,:])) 203 | print("Accuracy: " + str(np.mean((p[0,:] == y[0,:])))) 204 | 205 | return p 206 | 207 | def load_2D_dataset(): 208 | data = scipy.io.loadmat('datasets/data.mat') 209 | train_X = data['X'].T 210 | train_Y = data['y'].T 211 | test_X = data['Xval'].T 212 | test_Y = data['yval'].T 213 | 214 | plt.scatter(train_X[0, :], train_X[1, :], c=train_Y.ravel(), s=40, cmap=plt.cm.Spectral); 215 | 216 | return train_X, train_Y, test_X, test_Y 217 | 218 | def plot_decision_boundary(model, X, y): 219 | # Set min and max values and give it some padding 220 | x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1 221 | y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1 222 | h = 0.01 223 | # Generate a grid of points with distance h between them 224 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) 225 | # Predict the function value for the whole grid 226 | Z = model(np.c_[xx.ravel(), yy.ravel()]) 227 | Z = Z.reshape(xx.shape) 228 | # Plot the contour and training examples 229 | plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral) 230 | plt.ylabel('x2') 231 | plt.xlabel('x1') 232 | plt.scatter(X[0, :], X[1, :], c=y.ravel(), cmap=plt.cm.Spectral) 233 | plt.show() 234 | 235 | def predict_dec(parameters, X): 236 | """ 237 | Used for plotting decision boundary. 238 | 239 | Arguments: 240 | parameters -- python dictionary containing your parameters 241 | X -- input data of size (m, K) 242 | 243 | Returns 244 | predictions -- vector of predictions of our model (red: 0 / blue: 1) 245 | """ 246 | 247 | # Predict using forward propagation and a classification threshold of 0.5 248 | a3, cache = forward_propagation(X, parameters) 249 | predictions = (a3 > 0.5) 250 | return predictions 251 | 252 | def load_dataset(): 253 | np.random.seed(3) 254 | train_X, train_Y = sklearn.datasets.make_moons(n_samples=300, noise=.2) #300 #0.2 255 | # Visualize the data 256 | plt.scatter(train_X[:, 0], train_X[:, 1], c=train_Y.ravel(), s=40, cmap=plt.cm.Spectral); 257 | train_X = train_X.T 258 | train_Y = train_Y.reshape((1, train_Y.shape[0])) 259 | 260 | return train_X, train_Y -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/2.Algorithm Optimization/testCases.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def update_parameters_with_gd_test_case(): 4 | np.random.seed(1) 5 | learning_rate = 0.01 6 | W1 = np.random.randn(2,3) 7 | b1 = np.random.randn(2,1) 8 | W2 = np.random.randn(3,3) 9 | b2 = np.random.randn(3,1) 10 | 11 | dW1 = np.random.randn(2,3) 12 | db1 = np.random.randn(2,1) 13 | dW2 = np.random.randn(3,3) 14 | db2 = np.random.randn(3,1) 15 | 16 | parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2} 17 | grads = {"dW1": dW1, "db1": db1, "dW2": dW2, "db2": db2} 18 | 19 | return parameters, grads, learning_rate 20 | 21 | """ 22 | def update_parameters_with_sgd_checker(function, inputs, outputs): 23 | if function(inputs) == outputs: 24 | print("Correct") 25 | else: 26 | print("Incorrect") 27 | """ 28 | 29 | def random_mini_batches_test_case(): 30 | np.random.seed(1) 31 | mini_batch_size = 64 32 | X = np.random.randn(12288, 148) 33 | Y = np.random.randn(1, 148) < 0.5 34 | return X, Y, mini_batch_size 35 | 36 | def initialize_velocity_test_case(): 37 | np.random.seed(1) 38 | W1 = np.random.randn(2,3) 39 | b1 = np.random.randn(2,1) 40 | W2 = np.random.randn(3,3) 41 | b2 = np.random.randn(3,1) 42 | parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2} 43 | return parameters 44 | 45 | def update_parameters_with_momentum_test_case(): 46 | np.random.seed(1) 47 | W1 = np.random.randn(2,3) 48 | b1 = np.random.randn(2,1) 49 | W2 = np.random.randn(3,3) 50 | b2 = np.random.randn(3,1) 51 | 52 | dW1 = np.random.randn(2,3) 53 | db1 = np.random.randn(2,1) 54 | dW2 = np.random.randn(3,3) 55 | db2 = np.random.randn(3,1) 56 | parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2} 57 | grads = {"dW1": dW1, "db1": db1, "dW2": dW2, "db2": db2} 58 | v = {'dW1': np.array([[ 0., 0., 0.], 59 | [ 0., 0., 0.]]), 'dW2': np.array([[ 0., 0., 0.], 60 | [ 0., 0., 0.], 61 | [ 0., 0., 0.]]), 'db1': np.array([[ 0.], 62 | [ 0.]]), 'db2': np.array([[ 0.], 63 | [ 0.], 64 | [ 0.]])} 65 | return parameters, grads, v 66 | 67 | def initialize_adam_test_case(): 68 | np.random.seed(1) 69 | W1 = np.random.randn(2,3) 70 | b1 = np.random.randn(2,1) 71 | W2 = np.random.randn(3,3) 72 | b2 = np.random.randn(3,1) 73 | parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2} 74 | return parameters 75 | 76 | def update_parameters_with_adam_test_case(): 77 | np.random.seed(1) 78 | v, s = ({'dW1': np.array([[ 0., 0., 0.], 79 | [ 0., 0., 0.]]), 'dW2': np.array([[ 0., 0., 0.], 80 | [ 0., 0., 0.], 81 | [ 0., 0., 0.]]), 'db1': np.array([[ 0.], 82 | [ 0.]]), 'db2': np.array([[ 0.], 83 | [ 0.], 84 | [ 0.]])}, {'dW1': np.array([[ 0., 0., 0.], 85 | [ 0., 0., 0.]]), 'dW2': np.array([[ 0., 0., 0.], 86 | [ 0., 0., 0.], 87 | [ 0., 0., 0.]]), 'db1': np.array([[ 0.], 88 | [ 0.]]), 'db2': np.array([[ 0.], 89 | [ 0.], 90 | [ 0.]])}) 91 | W1 = np.random.randn(2,3) 92 | b1 = np.random.randn(2,1) 93 | W2 = np.random.randn(3,3) 94 | b2 = np.random.randn(3,1) 95 | 96 | dW1 = np.random.randn(2,3) 97 | db1 = np.random.randn(2,1) 98 | dW2 = np.random.randn(3,3) 99 | db2 = np.random.randn(3,1) 100 | 101 | parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2} 102 | grads = {"dW1": dW1, "db1": db1, "dW2": dW2, "db2": db2} 103 | 104 | return parameters, grads, v, s 105 | -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/Tensorflow Tutorial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Tensorflow Tutorial" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "name": "stderr", 17 | "output_type": "stream", 18 | "text": [ 19 | "C:\\Users\\PEIM001\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:72: UserWarning: h5py is running against HDF5 1.10.2 when it was built against 1.10.3, this may cause problems\n", 20 | " '{0}.{1}.{2}'.format(*version.hdf5_built_version_tuple)\n" 21 | ] 22 | } 23 | ], 24 | "source": [ 25 | "import numpy as np\n", 26 | "import tensorflow as tf" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 6, 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "name": "stdout", 36 | "output_type": "stream", 37 | "text": [ 38 | "0.0\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "w = tf.Variable(0, dtype=tf.float32)\n", 44 | "# cost = tf.add(tf.add(w**2, tf.multiply(-10., w)), 25)\n", 45 | "cost = w**2 - 10*w + 25\n", 46 | "train = tf.train.GradientDescentOptimizer(0.01).minimize(cost)\n", 47 | "\n", 48 | "init = tf.global_variables_initializer()\n", 49 | "session = tf.Session()\n", 50 | "session.run(init)\n", 51 | "print(session.run(w))" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 7, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "name": "stdout", 61 | "output_type": "stream", 62 | "text": [ 63 | "0.099999994\n" 64 | ] 65 | } 66 | ], 67 | "source": [ 68 | "session.run(train)\n", 69 | "print(session.run(w))" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 8, 75 | "metadata": {}, 76 | "outputs": [ 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | "4.9999886\n" 82 | ] 83 | } 84 | ], 85 | "source": [ 86 | "for i in range(1000):\n", 87 | " session.run(train)\n", 88 | "print(session.run(w))" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "## Working with data" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 14, 101 | "metadata": {}, 102 | "outputs": [ 103 | { 104 | "name": "stdout", 105 | "output_type": "stream", 106 | "text": [ 107 | "0.0\n" 108 | ] 109 | } 110 | ], 111 | "source": [ 112 | "coefficients = np.array([[1.], [-20.], [100.]])\n", 113 | "\n", 114 | "w = tf.Variable(0, dtype=tf.float32)\n", 115 | "x = tf.placeholder(tf.float32, [3,1])\n", 116 | "\n", 117 | "cost = x[0][0]*w**2 + x[1][0]*w + x[2][0]\n", 118 | "\n", 119 | "train = tf.train.GradientDescentOptimizer(0.01).minimize(cost)\n", 120 | "\n", 121 | "init = tf.global_variables_initializer()\n", 122 | "session = tf.Session()\n", 123 | "session.run(init)\n", 124 | "print(session.run(w))" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 15, 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "name": "stdout", 134 | "output_type": "stream", 135 | "text": [ 136 | "0.19999999\n" 137 | ] 138 | } 139 | ], 140 | "source": [ 141 | "session.run(train, feed_dict={x:coefficients})\n", 142 | "\n", 143 | "print(session.run(w))" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 16, 149 | "metadata": {}, 150 | "outputs": [ 151 | { 152 | "name": "stdout", 153 | "output_type": "stream", 154 | "text": [ 155 | "9.999977\n" 156 | ] 157 | } 158 | ], 159 | "source": [ 160 | "for i in range(1000):\n", 161 | " session.run(train, feed_dict={x:coefficients})\n", 162 | "print(session.run(w))" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "metadata": {}, 169 | "outputs": [], 170 | "source": [] 171 | } 172 | ], 173 | "metadata": { 174 | "kernelspec": { 175 | "display_name": "Python 3", 176 | "language": "python", 177 | "name": "python3" 178 | }, 179 | "language_info": { 180 | "codemirror_mode": { 181 | "name": "ipython", 182 | "version": 3 183 | }, 184 | "file_extension": ".py", 185 | "mimetype": "text/x-python", 186 | "name": "python", 187 | "nbconvert_exporter": "python", 188 | "pygments_lexer": "ipython3", 189 | "version": "3.6.6" 190 | } 191 | }, 192 | "nbformat": 4, 193 | "nbformat_minor": 2 194 | } 195 | -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/__pycache__/tf_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/__pycache__/tf_utils.cpython-36.pyc -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/datasets/test_signs.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/datasets/test_signs.h5 -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/datasets/train_signs.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/datasets/train_signs.h5 -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/improv_utils.py: -------------------------------------------------------------------------------- 1 | import h5py 2 | import numpy as np 3 | import tensorflow as tf 4 | import math 5 | 6 | def load_dataset(): 7 | train_dataset = h5py.File('datasets/train_signs.h5', "r") 8 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 9 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 10 | 11 | test_dataset = h5py.File('datasets/test_signs.h5', "r") 12 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 13 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 14 | 15 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 16 | 17 | train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 18 | test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 19 | 20 | return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes 21 | 22 | 23 | def random_mini_batches(X, Y, mini_batch_size = 64, seed = 0): 24 | """ 25 | Creates a list of random minibatches from (X, Y) 26 | 27 | Arguments: 28 | X -- input data, of shape (input size, number of examples) 29 | Y -- true "label" vector (containing 0 if cat, 1 if non-cat), of shape (1, number of examples) 30 | mini_batch_size - size of the mini-batches, integer 31 | seed -- this is only for the purpose of grading, so that you're "random minibatches are the same as ours. 32 | 33 | Returns: 34 | mini_batches -- list of synchronous (mini_batch_X, mini_batch_Y) 35 | """ 36 | 37 | m = X.shape[1] # number of training examples 38 | mini_batches = [] 39 | np.random.seed(seed) 40 | 41 | # Step 1: Shuffle (X, Y) 42 | permutation = list(np.random.permutation(m)) 43 | shuffled_X = X[:, permutation] 44 | shuffled_Y = Y[:, permutation].reshape((Y.shape[0],m)) 45 | 46 | # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case. 47 | num_complete_minibatches = math.floor(m/mini_batch_size) # number of mini batches of size mini_batch_size in your partitionning 48 | for k in range(0, num_complete_minibatches): 49 | mini_batch_X = shuffled_X[:, k * mini_batch_size : k * mini_batch_size + mini_batch_size] 50 | mini_batch_Y = shuffled_Y[:, k * mini_batch_size : k * mini_batch_size + mini_batch_size] 51 | mini_batch = (mini_batch_X, mini_batch_Y) 52 | mini_batches.append(mini_batch) 53 | 54 | # Handling the end case (last mini-batch < mini_batch_size) 55 | if m % mini_batch_size != 0: 56 | mini_batch_X = shuffled_X[:, num_complete_minibatches * mini_batch_size : m] 57 | mini_batch_Y = shuffled_Y[:, num_complete_minibatches * mini_batch_size : m] 58 | mini_batch = (mini_batch_X, mini_batch_Y) 59 | mini_batches.append(mini_batch) 60 | 61 | return mini_batches 62 | 63 | def convert_to_one_hot(Y, C): 64 | Y = np.eye(C)[Y.reshape(-1)].T 65 | return Y 66 | 67 | def predict(X, parameters): 68 | 69 | W1 = tf.convert_to_tensor(parameters["W1"]) 70 | b1 = tf.convert_to_tensor(parameters["b1"]) 71 | W2 = tf.convert_to_tensor(parameters["W2"]) 72 | b2 = tf.convert_to_tensor(parameters["b2"]) 73 | W3 = tf.convert_to_tensor(parameters["W3"]) 74 | b3 = tf.convert_to_tensor(parameters["b3"]) 75 | 76 | params = {"W1": W1, 77 | "b1": b1, 78 | "W2": W2, 79 | "b2": b2, 80 | "W3": W3, 81 | "b3": b3} 82 | 83 | x = tf.placeholder("float", [12288, 1]) 84 | 85 | z3 = forward_propagation(x, params) 86 | p = tf.argmax(z3) 87 | 88 | with tf.Session() as sess: 89 | prediction = sess.run(p, feed_dict = {x: X}) 90 | 91 | return prediction 92 | 93 | 94 | def create_placeholders(n_x, n_y): 95 | """ 96 | Creates the placeholders for the tensorflow session. 97 | 98 | Arguments: 99 | n_x -- scalar, size of an image vector (num_px * num_px = 64 * 64 * 3 = 12288) 100 | n_y -- scalar, number of classes (from 0 to 5, so -> 6) 101 | 102 | Returns: 103 | X -- placeholder for the data input, of shape [n_x, None] and dtype "float" 104 | Y -- placeholder for the input labels, of shape [n_y, None] and dtype "float" 105 | 106 | Tips: 107 | - You will use None because it let's us be flexible on the number of examples you will for the placeholders. 108 | In fact, the number of examples during test/train is different. 109 | """ 110 | 111 | ### START CODE HERE ### (approx. 2 lines) 112 | X = tf.placeholder("float", [n_x, None]) 113 | Y = tf.placeholder("float", [n_y, None]) 114 | ### END CODE HERE ### 115 | 116 | return X, Y 117 | 118 | 119 | def initialize_parameters(): 120 | """ 121 | Initializes parameters to build a neural network with tensorflow. The shapes are: 122 | W1 : [25, 12288] 123 | b1 : [25, 1] 124 | W2 : [12, 25] 125 | b2 : [12, 1] 126 | W3 : [6, 12] 127 | b3 : [6, 1] 128 | 129 | Returns: 130 | parameters -- a dictionary of tensors containing W1, b1, W2, b2, W3, b3 131 | """ 132 | 133 | tf.set_random_seed(1) # so that your "random" numbers match ours 134 | 135 | ### START CODE HERE ### (approx. 6 lines of code) 136 | W1 = tf.get_variable("W1", [25,12288], initializer = tf.contrib.layers.xavier_initializer(seed = 1)) 137 | b1 = tf.get_variable("b1", [25,1], initializer = tf.zeros_initializer()) 138 | W2 = tf.get_variable("W2", [12,25], initializer = tf.contrib.layers.xavier_initializer(seed = 1)) 139 | b2 = tf.get_variable("b2", [12,1], initializer = tf.zeros_initializer()) 140 | W3 = tf.get_variable("W3", [6,12], initializer = tf.contrib.layers.xavier_initializer(seed = 1)) 141 | b3 = tf.get_variable("b3", [6,1], initializer = tf.zeros_initializer()) 142 | ### END CODE HERE ### 143 | 144 | parameters = {"W1": W1, 145 | "b1": b1, 146 | "W2": W2, 147 | "b2": b2, 148 | "W3": W3, 149 | "b3": b3} 150 | 151 | return parameters 152 | 153 | 154 | def compute_cost(z3, Y): 155 | """ 156 | Computes the cost 157 | 158 | Arguments: 159 | z3 -- output of forward propagation (output of the last LINEAR unit), of shape (10, number of examples) 160 | Y -- "true" labels vector placeholder, same shape as z3 161 | 162 | Returns: 163 | cost - Tensor of the cost function 164 | """ 165 | 166 | # to fit the tensorflow requirement for tf.nn.softmax_cross_entropy_with_logits() 167 | logits = tf.transpose(z3) 168 | labels = tf.transpose(Y) 169 | 170 | ### START CODE HERE ### (1 line of code) 171 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = labels)) 172 | ### END CODE HERE ### 173 | 174 | return cost 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | def model(X_train, Y_train, X_test, Y_test, learning_rate = 0.0001, 183 | num_epochs = 1500, minibatch_size = 32, print_cost = True): 184 | """ 185 | Implements a three-layer tensorflow neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SOFTMAX. 186 | 187 | Arguments: 188 | X_train -- training set, of shape (input size = 12288, number of training examples = 1080) 189 | Y_train -- test set, of shape (output size = 6, number of training examples = 1080) 190 | X_test -- training set, of shape (input size = 12288, number of training examples = 120) 191 | Y_test -- test set, of shape (output size = 6, number of test examples = 120) 192 | learning_rate -- learning rate of the optimization 193 | num_epochs -- number of epochs of the optimization loop 194 | minibatch_size -- size of a minibatch 195 | print_cost -- True to print the cost every 100 epochs 196 | 197 | Returns: 198 | parameters -- parameters learnt by the model. They can then be used to predict. 199 | """ 200 | 201 | ops.reset_default_graph() # to be able to rerun the model without overwriting tf variables 202 | tf.set_random_seed(1) # to keep consistent results 203 | seed = 3 # to keep consistent results 204 | (n_x, m) = X_train.shape # (n_x: input size, m : number of examples in the train set) 205 | n_y = Y_train.shape[0] # n_y : output size 206 | costs = [] # To keep track of the cost 207 | 208 | # Create Placeholders of shape (n_x, n_y) 209 | ### START CODE HERE ### (1 line) 210 | X, Y = create_placeholders(n_x, n_y) 211 | ### END CODE HERE ### 212 | 213 | # Initialize parameters 214 | ### START CODE HERE ### (1 line) 215 | parameters = initialize_parameters() 216 | ### END CODE HERE ### 217 | 218 | # Forward propagation: Build the forward propagation in the tensorflow graph 219 | ### START CODE HERE ### (1 line) 220 | z3 = forward_propagation(X, parameters) 221 | ### END CODE HERE ### 222 | 223 | # Cost function: Add cost function to tensorflow graph 224 | ### START CODE HERE ### (1 line) 225 | cost = compute_cost(z3, Y) 226 | ### END CODE HERE ### 227 | 228 | # Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer. 229 | ### START CODE HERE ### (1 line) 230 | optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost) 231 | ### END CODE HERE ### 232 | 233 | # Initialize all the variables 234 | init = tf.global_variables_initializer() 235 | 236 | # Start the session to compute the tensorflow graph 237 | with tf.Session() as sess: 238 | 239 | # Run the initialization 240 | sess.run(init) 241 | 242 | # Do the training loop 243 | for epoch in range(num_epochs): 244 | 245 | minibatch_cost = 0. 246 | num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set 247 | seed = seed + 1 248 | minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed) 249 | 250 | for minibatch in minibatches: 251 | 252 | # Select a minibatch 253 | (minibatch_X, minibatch_Y) = minibatch 254 | 255 | # IMPORTANT: The line that runs the graph on a minibatch. 256 | # Run the session to execute the optimizer and the cost, the feedict should contain a minibatch for (X,Y). 257 | ### START CODE HERE ### (1 line) 258 | _ , temp_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y}) 259 | ### END CODE HERE ### 260 | 261 | minibatch_cost += temp_cost / num_minibatches 262 | 263 | # Print the cost every epoch 264 | if print_cost == True and epoch % 100 == 0: 265 | print ("Cost after epoch %i: %f" % (epoch, minibatch_cost)) 266 | if print_cost == True and epoch % 5 == 0: 267 | costs.append(minibatch_cost) 268 | 269 | # plot the cost 270 | plt.plot(np.squeeze(costs)) 271 | plt.ylabel('cost') 272 | plt.xlabel('iterations (per tens)') 273 | plt.title("Learning rate =" + str(learning_rate)) 274 | plt.show() 275 | 276 | # lets save the parameters in a variable 277 | parameters = sess.run(parameters) 278 | print ("Parameters have been trained!") 279 | 280 | # Calculate the correct predictions 281 | correct_prediction = tf.equal(tf.argmax(z3), tf.argmax(Y)) 282 | 283 | # Calculate accuracy on the test set 284 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 285 | 286 | print ("Train Accuracy:", accuracy.eval({X: X_train, Y: Y_train})) 287 | print ("Test Accuracy:", accuracy.eval({X: X_test, Y: Y_test})) 288 | 289 | return parameters -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/tf_utils.py: -------------------------------------------------------------------------------- 1 | import h5py 2 | import numpy as np 3 | import tensorflow as tf 4 | import math 5 | 6 | def load_dataset(): 7 | train_dataset = h5py.File('datasets/train_signs.h5', "r") 8 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 9 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 10 | 11 | test_dataset = h5py.File('datasets/test_signs.h5', "r") 12 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 13 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 14 | 15 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 16 | 17 | train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 18 | test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 19 | 20 | return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes 21 | 22 | 23 | def random_mini_batches(X, Y, mini_batch_size = 64, seed = 0): 24 | """ 25 | Creates a list of random minibatches from (X, Y) 26 | 27 | Arguments: 28 | X -- input data, of shape (input size, number of examples) 29 | Y -- true "label" vector (containing 0 if cat, 1 if non-cat), of shape (1, number of examples) 30 | mini_batch_size - size of the mini-batches, integer 31 | seed -- this is only for the purpose of grading, so that you're "random minibatches are the same as ours. 32 | 33 | Returns: 34 | mini_batches -- list of synchronous (mini_batch_X, mini_batch_Y) 35 | """ 36 | 37 | m = X.shape[1] # number of training examples 38 | mini_batches = [] 39 | np.random.seed(seed) 40 | 41 | # Step 1: Shuffle (X, Y) 42 | permutation = list(np.random.permutation(m)) 43 | shuffled_X = X[:, permutation] 44 | shuffled_Y = Y[:, permutation].reshape((Y.shape[0],m)) 45 | 46 | # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case. 47 | num_complete_minibatches = math.floor(m/mini_batch_size) # number of mini batches of size mini_batch_size in your partitionning 48 | for k in range(0, num_complete_minibatches): 49 | mini_batch_X = shuffled_X[:, k * mini_batch_size : k * mini_batch_size + mini_batch_size] 50 | mini_batch_Y = shuffled_Y[:, k * mini_batch_size : k * mini_batch_size + mini_batch_size] 51 | mini_batch = (mini_batch_X, mini_batch_Y) 52 | mini_batches.append(mini_batch) 53 | 54 | # Handling the end case (last mini-batch < mini_batch_size) 55 | if m % mini_batch_size != 0: 56 | mini_batch_X = shuffled_X[:, num_complete_minibatches * mini_batch_size : m] 57 | mini_batch_Y = shuffled_Y[:, num_complete_minibatches * mini_batch_size : m] 58 | mini_batch = (mini_batch_X, mini_batch_Y) 59 | mini_batches.append(mini_batch) 60 | 61 | return mini_batches 62 | 63 | def convert_to_one_hot(Y, C): 64 | Y = np.eye(C)[Y.reshape(-1)].T 65 | return Y 66 | 67 | 68 | def predict(X, parameters): 69 | 70 | W1 = tf.convert_to_tensor(parameters["W1"]) 71 | b1 = tf.convert_to_tensor(parameters["b1"]) 72 | W2 = tf.convert_to_tensor(parameters["W2"]) 73 | b2 = tf.convert_to_tensor(parameters["b2"]) 74 | W3 = tf.convert_to_tensor(parameters["W3"]) 75 | b3 = tf.convert_to_tensor(parameters["b3"]) 76 | 77 | params = {"W1": W1, 78 | "b1": b1, 79 | "W2": W2, 80 | "b2": b2, 81 | "W3": W3, 82 | "b3": b3} 83 | 84 | x = tf.placeholder("float", [12288, 1]) 85 | 86 | z3 = forward_propagation_for_predict(x, params) 87 | p = tf.argmax(z3) 88 | 89 | sess = tf.Session() 90 | prediction = sess.run(p, feed_dict = {x: X}) 91 | 92 | return prediction 93 | 94 | def forward_propagation_for_predict(X, parameters): 95 | """ 96 | Implements the forward propagation for the model: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SOFTMAX 97 | 98 | Arguments: 99 | X -- input dataset placeholder, of shape (input size, number of examples) 100 | parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3" 101 | the shapes are given in initialize_parameters 102 | 103 | Returns: 104 | Z3 -- the output of the last LINEAR unit 105 | """ 106 | 107 | # Retrieve the parameters from the dictionary "parameters" 108 | W1 = parameters['W1'] 109 | b1 = parameters['b1'] 110 | W2 = parameters['W2'] 111 | b2 = parameters['b2'] 112 | W3 = parameters['W3'] 113 | b3 = parameters['b3'] 114 | # Numpy Equivalents: 115 | Z1 = tf.add(tf.matmul(W1, X), b1) # Z1 = np.dot(W1, X) + b1 116 | A1 = tf.nn.relu(Z1) # A1 = relu(Z1) 117 | Z2 = tf.add(tf.matmul(W2, A1), b2) # Z2 = np.dot(W2, a1) + b2 118 | A2 = tf.nn.relu(Z2) # A2 = relu(Z2) 119 | Z3 = tf.add(tf.matmul(W3, A2), b3) # Z3 = np.dot(W3,Z2) + b3 120 | 121 | return Z3 122 | -------------------------------------------------------------------------------- /2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/thumbs_up.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/2.Improving Deep Neural Networks/3.Hyperparameter Tuning, Batch Normalization and Programming Frameworks/thumbs_up.jpg -------------------------------------------------------------------------------- /3.Structuring Machine Learning Projects/Structuring Machine Learning Project.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Structuring Machine Learning Projects" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Introduction to ML Strategy" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "The challenge with deep learning is that there are many ways to improve a model:\n", 22 | "\n", 23 | "* Gather more data\n", 24 | "* Train the algorithm for a longer time\n", 25 | "* Change the architecture of the neural network\n", 26 | "* Get more diverse training set\n", 27 | "\n", 28 | "However, pursuing the wrong strategy can result in an importat loss of time and resources. You could be spending 6 months gathering more data only to realize that it barely improves the model. \n", 29 | "\n", 30 | "Therefore, it is important to have a good strategy when it comes to managing and improving machine learning projects." 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "### Orthogonalization" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "The most effictive ML practitioners have a clear view of what parameters to tune in order to achieve a better result.\n", 45 | "\n", 46 | "Orthogonalization refers to having a control with a very specific function. A single control should impact only one variable or parameter. That way, it is much easier to tune to achieve an expected result. If multiple controls can affect a single variable, then it is much harder to achieve an optimal result.\n", 47 | "\n", 48 | "**Show orthogonal graph**" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "How does that translate to machine learning?\n", 56 | "\n", 57 | "We need to consider the chain of assumptions in machine learning. It is assumed that if the model performs well on the training set, then it will perform well on the dev set and on the test set. Similarly, it should therefore perform well in the real world." 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "* Training: bigger network or change optimization algorithm\n", 65 | "* Dev set: use regularization or bigger train set\n", 66 | "* Test set: use a bigger dev set\n", 67 | "* Real world: Change dev distribution set or change cost function" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "## Setting up your goal" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "### Single number evaluation metric" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "Having a single number evaluation metric allows for quicker assessment of an algorithm." 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "For example, for a classifier, precision and recall are common evaluation metrics. However, there is a tradeoff between these two metrics. Instead, we should use a metric that combines both.\n", 96 | "\n", 97 | "In this case, we use the F1 score, which represents the harmonic mean of precision and recall. That way, it is much easier to assess the quality of different models, and it speeds up iteration." 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": {}, 103 | "source": [ 104 | "### Satisficing and optimizing metrics" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "Suppose you are concerned by both the accuracy and running time of a classifier" 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": {}, 117 | "source": [ 118 | "Now, you would like to maximize accuracy while keeping the running time small (say less than 100ms). Therefore, the accuracy is the *optimizing* metric and the running time is the *satisficing* metric." 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": {}, 124 | "source": [ 125 | "In general, if there are many metrics that you wish to consider, 1 should be an optimizing metric, and the rest should be satisficing." 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "### Set up train/dev/test distributions" 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "metadata": {}, 138 | "source": [ 139 | "The way these sets are set up can really make a difference between slowing down a team or increasing its efficiency and speed of iterations towards the right direction." 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "Aa a general guideline, the dev set and test set should reflect the data that you expect to get in the future and consider to do well on. \n", 147 | "\n", 148 | "For example, a credit modelling alogrithm should not be trained on low income instances if it is to be deployed with medium-income individuals.\n", 149 | "\n", 150 | "In other words, they must come from the same distribution." 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "### Size of dev and test sets" 158 | ] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "metadata": {}, 163 | "source": [ 164 | "How large should they be?" 165 | ] 166 | }, 167 | { 168 | "cell_type": "markdown", 169 | "metadata": {}, 170 | "source": [ 171 | "The old way of doing it was a 70/30 train/test split or 60/20/20 for train/dev/test.\n", 172 | "\n", 173 | "This is still valid in the case where data is not abundant." 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "However, in this new era of deep learning, huge amounts of data are available. Therefore, it is better to do a 98/1/1 train/test/dev split." 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "For the test set, it should be big enough to give high confidence in the overall performance of the system. This could be 10 000 or 100 000 examples that would represent less than 10% of the available data." 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "metadata": {}, 193 | "source": [ 194 | "## Comparing to human-level performance" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "### Why human-level performance" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "In the last few years, we have been comparing machine learning systems to human-level performance. This is possible, because the performance of many of these systems are very good." 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "Often, progress is fast as it approaches human-level performance. After though, progress slows down until it reaches the bayes optimal error. This is the error where no possible function can be accurate at a 100%. For example, a picture is too blurry or a sound sample is too noisy." 216 | ] 217 | }, 218 | { 219 | "cell_type": "markdown", 220 | "metadata": {}, 221 | "source": [ 222 | "Now, humans are very good at a lot of tasks involving natural data. While your algorithm is not as good as humans you can:\n", 223 | "* get labeled data from humans\n", 224 | "* perform manual error analysis" 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "### Avoidable bias" 232 | ] 233 | }, 234 | { 235 | "cell_type": "markdown", 236 | "metadata": {}, 237 | "source": [ 238 | "Human performance is a good estimate of the Bayes error. Now, if the training error is far for the human-level performance, then you should focus on reducing bias and increasing the performance on the training set.\n", 239 | "\n", 240 | "However, if on the training set, the system performs closely to human-level, then you should focus on reducing variance and improving performance on the dev set error." 241 | ] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "metadata": {}, 246 | "source": [ 247 | "The avoidable bias is the difference between human-level performance and the algorithm's bias" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "metadata": {}, 253 | "source": [ 254 | "### How to improve a model?" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "Reduce bias:\n", 262 | "* train bigger model\n", 263 | "* train longer/better optimization algorithm(Adam, momentum)\n", 264 | "* Change NN architecture/hyperparameters (use RNN or CNN)\n", 265 | "\n", 266 | "Reduce bias:\n", 267 | "* more data\n", 268 | "* regularization (L2, dropout, data augmentation)\n", 269 | "* Change NN architecture/hyperparameters (use RNN or CNN)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "markdown", 274 | "metadata": {}, 275 | "source": [ 276 | "## Error Analysis" 277 | ] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "metadata": {}, 282 | "source": [ 283 | "### Carrying out error analysis" 284 | ] 285 | }, 286 | { 287 | "cell_type": "markdown", 288 | "metadata": {}, 289 | "source": [ 290 | "Analyze the errors and determine what would be the best outcome.\n", 291 | "\n", 292 | "For example, 50 images were misclassified out of a 100, then your error rate will improve by 50%!" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "### Cleaning up incorrectly labeled examples" 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "metadata": {}, 305 | "source": [ 306 | "DL algorithms are quite robust to random errors in random set. Not worth correcting if the amount of errors is small and random. However, they will suffer with systematic errors." 307 | ] 308 | }, 309 | { 310 | "cell_type": "markdown", 311 | "metadata": {}, 312 | "source": [ 313 | "* Apply same process to dev/test set to make sure they continue to come from the same distribution\n", 314 | "* Consider examining examples the algo got right and wrong\n", 315 | "* Train and dev/test data may now come from slightly different distributions" 316 | ] 317 | }, 318 | { 319 | "cell_type": "markdown", 320 | "metadata": {}, 321 | "source": [ 322 | "### Build quickly and iterate" 323 | ] 324 | }, 325 | { 326 | "cell_type": "markdown", 327 | "metadata": {}, 328 | "source": [ 329 | "Don't build something too simple or too complex. Buid fast and use error anaylsis to iterate afterwards." 330 | ] 331 | }, 332 | { 333 | "cell_type": "markdown", 334 | "metadata": {}, 335 | "source": [ 336 | "## Mismatched training and dev/test set" 337 | ] 338 | }, 339 | { 340 | "cell_type": "markdown", 341 | "metadata": {}, 342 | "source": [ 343 | "### Training and testing on different distributions" 344 | ] 345 | }, 346 | { 347 | "cell_type": "code", 348 | "execution_count": null, 349 | "metadata": {}, 350 | "outputs": [], 351 | "source": [] 352 | } 353 | ], 354 | "metadata": { 355 | "kernelspec": { 356 | "display_name": "Python 3", 357 | "language": "python", 358 | "name": "python3" 359 | }, 360 | "language_info": { 361 | "codemirror_mode": { 362 | "name": "ipython", 363 | "version": 3 364 | }, 365 | "file_extension": ".py", 366 | "mimetype": "text/x-python", 367 | "name": "python", 368 | "nbconvert_exporter": "python", 369 | "pygments_lexer": "ipython3", 370 | "version": "3.6.6" 371 | } 372 | }, 373 | "nbformat": 4, 374 | "nbformat_minor": 2 375 | } 376 | -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/1.Foundations of Convolutional Neural Networks/__pycache__/cnn_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/1.Foundations of Convolutional Neural Networks/__pycache__/cnn_utils.cpython-36.pyc -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/1.Foundations of Convolutional Neural Networks/cnn_utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import h5py 4 | import matplotlib.pyplot as plt 5 | import tensorflow as tf 6 | from tensorflow.python.framework import ops 7 | 8 | def load_dataset(): 9 | train_dataset = h5py.File('datasets/train_signs.h5', "r") 10 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 11 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 12 | 13 | test_dataset = h5py.File('datasets/test_signs.h5', "r") 14 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 15 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 16 | 17 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 18 | 19 | train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 20 | test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 21 | 22 | return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes 23 | 24 | 25 | def random_mini_batches(X, Y, mini_batch_size = 64, seed = 0): 26 | """ 27 | Creates a list of random minibatches from (X, Y) 28 | 29 | Arguments: 30 | X -- input data, of shape (input size, number of examples) (m, Hi, Wi, Ci) 31 | Y -- true "label" vector (containing 0 if cat, 1 if non-cat), of shape (1, number of examples) (m, n_y) 32 | mini_batch_size - size of the mini-batches, integer 33 | seed -- this is only for the purpose of grading, so that you're "random minibatches are the same as ours. 34 | 35 | Returns: 36 | mini_batches -- list of synchronous (mini_batch_X, mini_batch_Y) 37 | """ 38 | 39 | m = X.shape[0] # number of training examples 40 | mini_batches = [] 41 | np.random.seed(seed) 42 | 43 | # Step 1: Shuffle (X, Y) 44 | permutation = list(np.random.permutation(m)) 45 | shuffled_X = X[permutation,:,:,:] 46 | shuffled_Y = Y[permutation,:] 47 | 48 | # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case. 49 | num_complete_minibatches = math.floor(m/mini_batch_size) # number of mini batches of size mini_batch_size in your partitionning 50 | for k in range(0, num_complete_minibatches): 51 | mini_batch_X = shuffled_X[k * mini_batch_size : k * mini_batch_size + mini_batch_size,:,:,:] 52 | mini_batch_Y = shuffled_Y[k * mini_batch_size : k * mini_batch_size + mini_batch_size,:] 53 | mini_batch = (mini_batch_X, mini_batch_Y) 54 | mini_batches.append(mini_batch) 55 | 56 | # Handling the end case (last mini-batch < mini_batch_size) 57 | if m % mini_batch_size != 0: 58 | mini_batch_X = shuffled_X[num_complete_minibatches * mini_batch_size : m,:,:,:] 59 | mini_batch_Y = shuffled_Y[num_complete_minibatches * mini_batch_size : m,:] 60 | mini_batch = (mini_batch_X, mini_batch_Y) 61 | mini_batches.append(mini_batch) 62 | 63 | return mini_batches 64 | 65 | 66 | def convert_to_one_hot(Y, C): 67 | Y = np.eye(C)[Y.reshape(-1)].T 68 | return Y 69 | 70 | 71 | def forward_propagation_for_predict(X, parameters): 72 | """ 73 | Implements the forward propagation for the model: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SOFTMAX 74 | 75 | Arguments: 76 | X -- input dataset placeholder, of shape (input size, number of examples) 77 | parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3" 78 | the shapes are given in initialize_parameters 79 | 80 | Returns: 81 | Z3 -- the output of the last LINEAR unit 82 | """ 83 | 84 | # Retrieve the parameters from the dictionary "parameters" 85 | W1 = parameters['W1'] 86 | b1 = parameters['b1'] 87 | W2 = parameters['W2'] 88 | b2 = parameters['b2'] 89 | W3 = parameters['W3'] 90 | b3 = parameters['b3'] 91 | # Numpy Equivalents: 92 | Z1 = tf.add(tf.matmul(W1, X), b1) # Z1 = np.dot(W1, X) + b1 93 | A1 = tf.nn.relu(Z1) # A1 = relu(Z1) 94 | Z2 = tf.add(tf.matmul(W2, A1), b2) # Z2 = np.dot(W2, a1) + b2 95 | A2 = tf.nn.relu(Z2) # A2 = relu(Z2) 96 | Z3 = tf.add(tf.matmul(W3, A2), b3) # Z3 = np.dot(W3,Z2) + b3 97 | 98 | return Z3 99 | 100 | def predict(X, parameters): 101 | 102 | W1 = tf.convert_to_tensor(parameters["W1"]) 103 | b1 = tf.convert_to_tensor(parameters["b1"]) 104 | W2 = tf.convert_to_tensor(parameters["W2"]) 105 | b2 = tf.convert_to_tensor(parameters["b2"]) 106 | W3 = tf.convert_to_tensor(parameters["W3"]) 107 | b3 = tf.convert_to_tensor(parameters["b3"]) 108 | 109 | params = {"W1": W1, 110 | "b1": b1, 111 | "W2": W2, 112 | "b2": b2, 113 | "W3": W3, 114 | "b3": b3} 115 | 116 | x = tf.placeholder("float", [12288, 1]) 117 | 118 | z3 = forward_propagation_for_predict(x, params) 119 | p = tf.argmax(z3) 120 | 121 | sess = tf.Session() 122 | prediction = sess.run(p, feed_dict = {x: X}) 123 | 124 | return prediction 125 | 126 | #def predict(X, parameters): 127 | # 128 | # W1 = tf.convert_to_tensor(parameters["W1"]) 129 | # b1 = tf.convert_to_tensor(parameters["b1"]) 130 | # W2 = tf.convert_to_tensor(parameters["W2"]) 131 | # b2 = tf.convert_to_tensor(parameters["b2"]) 132 | ## W3 = tf.convert_to_tensor(parameters["W3"]) 133 | ## b3 = tf.convert_to_tensor(parameters["b3"]) 134 | # 135 | ## params = {"W1": W1, 136 | ## "b1": b1, 137 | ## "W2": W2, 138 | ## "b2": b2, 139 | ## "W3": W3, 140 | ## "b3": b3} 141 | # 142 | # params = {"W1": W1, 143 | # "b1": b1, 144 | # "W2": W2, 145 | # "b2": b2} 146 | # 147 | # x = tf.placeholder("float", [12288, 1]) 148 | # 149 | # z3 = forward_propagation(x, params) 150 | # p = tf.argmax(z3) 151 | # 152 | # with tf.Session() as sess: 153 | # prediction = sess.run(p, feed_dict = {x: X}) 154 | # 155 | # return prediction -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/1.Foundations of Convolutional Neural Networks/datasets/test_signs.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/1.Foundations of Convolutional Neural Networks/datasets/test_signs.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/1.Foundations of Convolutional Neural Networks/datasets/train_signs.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/1.Foundations of Convolutional Neural Networks/datasets/train_signs.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/HappyModel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/2.Deep Convolutional Models/HappyModel.png -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/ResNet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/2.Deep Convolutional Models/ResNet.png -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/__pycache__/kt_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/2.Deep Convolutional Models/__pycache__/kt_utils.cpython-36.pyc -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/__pycache__/resnets_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/2.Deep Convolutional Models/__pycache__/resnets_utils.cpython-36.pyc -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/datasets/test_happy.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/2.Deep Convolutional Models/datasets/test_happy.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/datasets/test_signs.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/2.Deep Convolutional Models/datasets/test_signs.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/datasets/train_happy.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/2.Deep Convolutional Models/datasets/train_happy.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/datasets/train_signs.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/2.Deep Convolutional Models/datasets/train_signs.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/kt_utils.py: -------------------------------------------------------------------------------- 1 | import keras.backend as K 2 | import math 3 | import numpy as np 4 | import h5py 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | def mean_pred(y_true, y_pred): 9 | return K.mean(y_pred) 10 | 11 | def load_dataset(): 12 | train_dataset = h5py.File('datasets/train_happy.h5', "r") 13 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 14 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 15 | 16 | test_dataset = h5py.File('datasets/test_happy.h5', "r") 17 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 18 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 19 | 20 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 21 | 22 | train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 23 | test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 24 | 25 | return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes 26 | 27 | -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/2.Deep Convolutional Models/resnets_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | import h5py 5 | import math 6 | 7 | def load_dataset(): 8 | train_dataset = h5py.File('datasets/train_signs.h5', "r") 9 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 10 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 11 | 12 | test_dataset = h5py.File('datasets/test_signs.h5', "r") 13 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 14 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 15 | 16 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 17 | 18 | train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 19 | test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 20 | 21 | return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes 22 | 23 | 24 | def random_mini_batches(X, Y, mini_batch_size = 64, seed = 0): 25 | """ 26 | Creates a list of random minibatches from (X, Y) 27 | 28 | Arguments: 29 | X -- input data, of shape (input size, number of examples) (m, Hi, Wi, Ci) 30 | Y -- true "label" vector (containing 0 if cat, 1 if non-cat), of shape (1, number of examples) (m, n_y) 31 | mini_batch_size - size of the mini-batches, integer 32 | seed -- this is only for the purpose of grading, so that you're "random minibatches are the same as ours. 33 | 34 | Returns: 35 | mini_batches -- list of synchronous (mini_batch_X, mini_batch_Y) 36 | """ 37 | 38 | m = X.shape[0] # number of training examples 39 | mini_batches = [] 40 | np.random.seed(seed) 41 | 42 | # Step 1: Shuffle (X, Y) 43 | permutation = list(np.random.permutation(m)) 44 | shuffled_X = X[permutation,:,:,:] 45 | shuffled_Y = Y[permutation,:] 46 | 47 | # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case. 48 | num_complete_minibatches = math.floor(m/mini_batch_size) # number of mini batches of size mini_batch_size in your partitionning 49 | for k in range(0, num_complete_minibatches): 50 | mini_batch_X = shuffled_X[k * mini_batch_size : k * mini_batch_size + mini_batch_size,:,:,:] 51 | mini_batch_Y = shuffled_Y[k * mini_batch_size : k * mini_batch_size + mini_batch_size,:] 52 | mini_batch = (mini_batch_X, mini_batch_Y) 53 | mini_batches.append(mini_batch) 54 | 55 | # Handling the end case (last mini-batch < mini_batch_size) 56 | if m % mini_batch_size != 0: 57 | mini_batch_X = shuffled_X[num_complete_minibatches * mini_batch_size : m,:,:,:] 58 | mini_batch_Y = shuffled_Y[num_complete_minibatches * mini_batch_size : m,:] 59 | mini_batch = (mini_batch_X, mini_batch_Y) 60 | mini_batches.append(mini_batch) 61 | 62 | return mini_batches 63 | 64 | 65 | def convert_to_one_hot(Y, C): 66 | Y = np.eye(C)[Y.reshape(-1)].T 67 | return Y 68 | 69 | 70 | def forward_propagation_for_predict(X, parameters): 71 | """ 72 | Implements the forward propagation for the model: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SOFTMAX 73 | 74 | Arguments: 75 | X -- input dataset placeholder, of shape (input size, number of examples) 76 | parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3" 77 | the shapes are given in initialize_parameters 78 | 79 | Returns: 80 | Z3 -- the output of the last LINEAR unit 81 | """ 82 | 83 | # Retrieve the parameters from the dictionary "parameters" 84 | W1 = parameters['W1'] 85 | b1 = parameters['b1'] 86 | W2 = parameters['W2'] 87 | b2 = parameters['b2'] 88 | W3 = parameters['W3'] 89 | b3 = parameters['b3'] 90 | # Numpy Equivalents: 91 | Z1 = tf.add(tf.matmul(W1, X), b1) # Z1 = np.dot(W1, X) + b1 92 | A1 = tf.nn.relu(Z1) # A1 = relu(Z1) 93 | Z2 = tf.add(tf.matmul(W2, A1), b2) # Z2 = np.dot(W2, a1) + b2 94 | A2 = tf.nn.relu(Z2) # A2 = relu(Z2) 95 | Z3 = tf.add(tf.matmul(W3, A2), b3) # Z3 = np.dot(W3,Z2) + b3 96 | 97 | return Z3 98 | 99 | def predict(X, parameters): 100 | 101 | W1 = tf.convert_to_tensor(parameters["W1"]) 102 | b1 = tf.convert_to_tensor(parameters["b1"]) 103 | W2 = tf.convert_to_tensor(parameters["W2"]) 104 | b2 = tf.convert_to_tensor(parameters["b2"]) 105 | W3 = tf.convert_to_tensor(parameters["W3"]) 106 | b3 = tf.convert_to_tensor(parameters["b3"]) 107 | 108 | params = {"W1": W1, 109 | "b1": b1, 110 | "W2": W2, 111 | "b2": b2, 112 | "W3": W3, 113 | "b3": b3} 114 | 115 | x = tf.placeholder("float", [12288, 1]) 116 | 117 | z3 = forward_propagation_for_predict(x, params) 118 | p = tf.argmax(z3) 119 | 120 | sess = tf.Session() 121 | prediction = sess.run(p, feed_dict = {x: X}) 122 | 123 | return prediction -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/datasets/test_happy.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/datasets/test_happy.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/datasets/train_face.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/datasets/train_face.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/datasets/train_happy.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/datasets/train_happy.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/fr_utils.py: -------------------------------------------------------------------------------- 1 | #### PART OF THIS CODE IS USING CODE FROM VICTOR SY WANG: https://github.com/iwantooxxoox/Keras-OpenFace/blob/master/utils.py #### 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import os 6 | import cv2 7 | from numpy import genfromtxt 8 | from keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate 9 | from keras.models import Model 10 | from keras.layers.normalization import BatchNormalization 11 | from keras.layers.pooling import MaxPooling2D, AveragePooling2D 12 | import h5py 13 | import matplotlib.pyplot as plt 14 | 15 | 16 | _FLOATX = 'float32' 17 | 18 | def variable(value, dtype=_FLOATX, name=None): 19 | v = tf.Variable(np.asarray(value, dtype=dtype), name=name) 20 | _get_session().run(v.initializer) 21 | return v 22 | 23 | def shape(x): 24 | return x.get_shape() 25 | 26 | def square(x): 27 | return tf.square(x) 28 | 29 | def zeros(shape, dtype=_FLOATX, name=None): 30 | return variable(np.zeros(shape), dtype, name) 31 | 32 | def concatenate(tensors, axis=-1): 33 | if axis < 0: 34 | axis = axis % len(tensors[0].get_shape()) 35 | return tf.concat(axis, tensors) 36 | 37 | def LRN2D(x): 38 | return tf.nn.lrn(x, alpha=1e-4, beta=0.75) 39 | 40 | def conv2d_bn(x, 41 | layer=None, 42 | cv1_out=None, 43 | cv1_filter=(1, 1), 44 | cv1_strides=(1, 1), 45 | cv2_out=None, 46 | cv2_filter=(3, 3), 47 | cv2_strides=(1, 1), 48 | padding=None): 49 | num = '' if cv2_out == None else '1' 50 | tensor = Conv2D(cv1_out, cv1_filter, strides=cv1_strides, data_format='channels_first', name=layer+'_conv'+num)(x) 51 | tensor = BatchNormalization(axis=1, epsilon=0.00001, name=layer+'_bn'+num)(tensor) 52 | tensor = Activation('relu')(tensor) 53 | if padding == None: 54 | return tensor 55 | tensor = ZeroPadding2D(padding=padding, data_format='channels_first')(tensor) 56 | if cv2_out == None: 57 | return tensor 58 | tensor = Conv2D(cv2_out, cv2_filter, strides=cv2_strides, data_format='channels_first', name=layer+'_conv'+'2')(tensor) 59 | tensor = BatchNormalization(axis=1, epsilon=0.00001, name=layer+'_bn'+'2')(tensor) 60 | tensor = Activation('relu')(tensor) 61 | return tensor 62 | 63 | WEIGHTS = [ 64 | 'conv1', 'bn1', 'conv2', 'bn2', 'conv3', 'bn3', 65 | 'inception_3a_1x1_conv', 'inception_3a_1x1_bn', 66 | 'inception_3a_pool_conv', 'inception_3a_pool_bn', 67 | 'inception_3a_5x5_conv1', 'inception_3a_5x5_conv2', 'inception_3a_5x5_bn1', 'inception_3a_5x5_bn2', 68 | 'inception_3a_3x3_conv1', 'inception_3a_3x3_conv2', 'inception_3a_3x3_bn1', 'inception_3a_3x3_bn2', 69 | 'inception_3b_3x3_conv1', 'inception_3b_3x3_conv2', 'inception_3b_3x3_bn1', 'inception_3b_3x3_bn2', 70 | 'inception_3b_5x5_conv1', 'inception_3b_5x5_conv2', 'inception_3b_5x5_bn1', 'inception_3b_5x5_bn2', 71 | 'inception_3b_pool_conv', 'inception_3b_pool_bn', 72 | 'inception_3b_1x1_conv', 'inception_3b_1x1_bn', 73 | 'inception_3c_3x3_conv1', 'inception_3c_3x3_conv2', 'inception_3c_3x3_bn1', 'inception_3c_3x3_bn2', 74 | 'inception_3c_5x5_conv1', 'inception_3c_5x5_conv2', 'inception_3c_5x5_bn1', 'inception_3c_5x5_bn2', 75 | 'inception_4a_3x3_conv1', 'inception_4a_3x3_conv2', 'inception_4a_3x3_bn1', 'inception_4a_3x3_bn2', 76 | 'inception_4a_5x5_conv1', 'inception_4a_5x5_conv2', 'inception_4a_5x5_bn1', 'inception_4a_5x5_bn2', 77 | 'inception_4a_pool_conv', 'inception_4a_pool_bn', 78 | 'inception_4a_1x1_conv', 'inception_4a_1x1_bn', 79 | 'inception_4e_3x3_conv1', 'inception_4e_3x3_conv2', 'inception_4e_3x3_bn1', 'inception_4e_3x3_bn2', 80 | 'inception_4e_5x5_conv1', 'inception_4e_5x5_conv2', 'inception_4e_5x5_bn1', 'inception_4e_5x5_bn2', 81 | 'inception_5a_3x3_conv1', 'inception_5a_3x3_conv2', 'inception_5a_3x3_bn1', 'inception_5a_3x3_bn2', 82 | 'inception_5a_pool_conv', 'inception_5a_pool_bn', 83 | 'inception_5a_1x1_conv', 'inception_5a_1x1_bn', 84 | 'inception_5b_3x3_conv1', 'inception_5b_3x3_conv2', 'inception_5b_3x3_bn1', 'inception_5b_3x3_bn2', 85 | 'inception_5b_pool_conv', 'inception_5b_pool_bn', 86 | 'inception_5b_1x1_conv', 'inception_5b_1x1_bn', 87 | 'dense_layer' 88 | ] 89 | 90 | conv_shape = { 91 | 'conv1': [64, 3, 7, 7], 92 | 'conv2': [64, 64, 1, 1], 93 | 'conv3': [192, 64, 3, 3], 94 | 'inception_3a_1x1_conv': [64, 192, 1, 1], 95 | 'inception_3a_pool_conv': [32, 192, 1, 1], 96 | 'inception_3a_5x5_conv1': [16, 192, 1, 1], 97 | 'inception_3a_5x5_conv2': [32, 16, 5, 5], 98 | 'inception_3a_3x3_conv1': [96, 192, 1, 1], 99 | 'inception_3a_3x3_conv2': [128, 96, 3, 3], 100 | 'inception_3b_3x3_conv1': [96, 256, 1, 1], 101 | 'inception_3b_3x3_conv2': [128, 96, 3, 3], 102 | 'inception_3b_5x5_conv1': [32, 256, 1, 1], 103 | 'inception_3b_5x5_conv2': [64, 32, 5, 5], 104 | 'inception_3b_pool_conv': [64, 256, 1, 1], 105 | 'inception_3b_1x1_conv': [64, 256, 1, 1], 106 | 'inception_3c_3x3_conv1': [128, 320, 1, 1], 107 | 'inception_3c_3x3_conv2': [256, 128, 3, 3], 108 | 'inception_3c_5x5_conv1': [32, 320, 1, 1], 109 | 'inception_3c_5x5_conv2': [64, 32, 5, 5], 110 | 'inception_4a_3x3_conv1': [96, 640, 1, 1], 111 | 'inception_4a_3x3_conv2': [192, 96, 3, 3], 112 | 'inception_4a_5x5_conv1': [32, 640, 1, 1,], 113 | 'inception_4a_5x5_conv2': [64, 32, 5, 5], 114 | 'inception_4a_pool_conv': [128, 640, 1, 1], 115 | 'inception_4a_1x1_conv': [256, 640, 1, 1], 116 | 'inception_4e_3x3_conv1': [160, 640, 1, 1], 117 | 'inception_4e_3x3_conv2': [256, 160, 3, 3], 118 | 'inception_4e_5x5_conv1': [64, 640, 1, 1], 119 | 'inception_4e_5x5_conv2': [128, 64, 5, 5], 120 | 'inception_5a_3x3_conv1': [96, 1024, 1, 1], 121 | 'inception_5a_3x3_conv2': [384, 96, 3, 3], 122 | 'inception_5a_pool_conv': [96, 1024, 1, 1], 123 | 'inception_5a_1x1_conv': [256, 1024, 1, 1], 124 | 'inception_5b_3x3_conv1': [96, 736, 1, 1], 125 | 'inception_5b_3x3_conv2': [384, 96, 3, 3], 126 | 'inception_5b_pool_conv': [96, 736, 1, 1], 127 | 'inception_5b_1x1_conv': [256, 736, 1, 1], 128 | } 129 | 130 | def load_weights_from_FaceNet(FRmodel): 131 | # Load weights from csv files (which was exported from Openface torch model) 132 | weights = WEIGHTS 133 | weights_dict = load_weights() 134 | 135 | # Set layer weights of the model 136 | for name in weights: 137 | if FRmodel.get_layer(name) != None: 138 | FRmodel.get_layer(name).set_weights(weights_dict[name]) 139 | elif model.get_layer(name) != None: 140 | model.get_layer(name).set_weights(weights_dict[name]) 141 | 142 | def load_weights(): 143 | # Set weights path 144 | dirPath = './weights' 145 | fileNames = filter(lambda f: not f.startswith('.'), os.listdir(dirPath)) 146 | paths = {} 147 | weights_dict = {} 148 | 149 | for n in fileNames: 150 | paths[n.replace('.csv', '')] = dirPath + '/' + n 151 | 152 | for name in WEIGHTS: 153 | if 'conv' in name: 154 | conv_w = genfromtxt(paths[name + '_w'], delimiter=',', dtype=None) 155 | conv_w = np.reshape(conv_w, conv_shape[name]) 156 | conv_w = np.transpose(conv_w, (2, 3, 1, 0)) 157 | conv_b = genfromtxt(paths[name + '_b'], delimiter=',', dtype=None) 158 | weights_dict[name] = [conv_w, conv_b] 159 | elif 'bn' in name: 160 | bn_w = genfromtxt(paths[name + '_w'], delimiter=',', dtype=None) 161 | bn_b = genfromtxt(paths[name + '_b'], delimiter=',', dtype=None) 162 | bn_m = genfromtxt(paths[name + '_m'], delimiter=',', dtype=None) 163 | bn_v = genfromtxt(paths[name + '_v'], delimiter=',', dtype=None) 164 | weights_dict[name] = [bn_w, bn_b, bn_m, bn_v] 165 | elif 'dense' in name: 166 | dense_w = genfromtxt(dirPath+'/dense_w.csv', delimiter=',', dtype=None) 167 | dense_w = np.reshape(dense_w, (128, 736)) 168 | dense_w = np.transpose(dense_w, (1, 0)) 169 | dense_b = genfromtxt(dirPath+'/dense_b.csv', delimiter=',', dtype=None) 170 | weights_dict[name] = [dense_w, dense_b] 171 | 172 | return weights_dict 173 | 174 | 175 | def load_dataset(): 176 | train_dataset = h5py.File('datasets/train_happy.h5', "r") 177 | train_set_x_orig = np.array(train_dataset["train_set_x"][:]) # your train set features 178 | train_set_y_orig = np.array(train_dataset["train_set_y"][:]) # your train set labels 179 | 180 | test_dataset = h5py.File('datasets/test_happy.h5', "r") 181 | test_set_x_orig = np.array(test_dataset["test_set_x"][:]) # your test set features 182 | test_set_y_orig = np.array(test_dataset["test_set_y"][:]) # your test set labels 183 | 184 | classes = np.array(test_dataset["list_classes"][:]) # the list of classes 185 | 186 | train_set_y_orig = train_set_y_orig.reshape((1, train_set_y_orig.shape[0])) 187 | test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0])) 188 | 189 | return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes 190 | 191 | def img_to_encoding(image_path, model): 192 | img1 = cv2.imread(image_path, 1) 193 | img = img1[...,::-1] 194 | img = np.around(np.transpose(img, (2,0,1))/255.0, decimals=12) 195 | x_train = np.array([img]) 196 | embedding = model.predict_on_batch(x_train) 197 | return embedding -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/inception_blocks.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import os 4 | from numpy import genfromtxt 5 | from keras import backend as K 6 | from keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate 7 | from keras.models import Model 8 | from keras.layers.normalization import BatchNormalization 9 | from keras.layers.pooling import MaxPooling2D, AveragePooling2D 10 | import fr_utils 11 | from keras.layers.core import Lambda, Flatten, Dense 12 | 13 | def inception_block_1a(X): 14 | """ 15 | Implementation of an inception block 16 | """ 17 | 18 | X_3x3 = Conv2D(96, (1, 1), data_format='channels_first', name ='inception_3a_3x3_conv1')(X) 19 | X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name = 'inception_3a_3x3_bn1')(X_3x3) 20 | X_3x3 = Activation('relu')(X_3x3) 21 | X_3x3 = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_3x3) 22 | X_3x3 = Conv2D(128, (3, 3), data_format='channels_first', name='inception_3a_3x3_conv2')(X_3x3) 23 | X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_3x3_bn2')(X_3x3) 24 | X_3x3 = Activation('relu')(X_3x3) 25 | 26 | X_5x5 = Conv2D(16, (1, 1), data_format='channels_first', name='inception_3a_5x5_conv1')(X) 27 | X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_5x5_bn1')(X_5x5) 28 | X_5x5 = Activation('relu')(X_5x5) 29 | X_5x5 = ZeroPadding2D(padding=(2, 2), data_format='channels_first')(X_5x5) 30 | X_5x5 = Conv2D(32, (5, 5), data_format='channels_first', name='inception_3a_5x5_conv2')(X_5x5) 31 | X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_5x5_bn2')(X_5x5) 32 | X_5x5 = Activation('relu')(X_5x5) 33 | 34 | X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X) 35 | X_pool = Conv2D(32, (1, 1), data_format='channels_first', name='inception_3a_pool_conv')(X_pool) 36 | X_pool = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_pool_bn')(X_pool) 37 | X_pool = Activation('relu')(X_pool) 38 | X_pool = ZeroPadding2D(padding=((3, 4), (3, 4)), data_format='channels_first')(X_pool) 39 | 40 | X_1x1 = Conv2D(64, (1, 1), data_format='channels_first', name='inception_3a_1x1_conv')(X) 41 | X_1x1 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_1x1_bn')(X_1x1) 42 | X_1x1 = Activation('relu')(X_1x1) 43 | 44 | # CONCAT 45 | inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1) 46 | 47 | return inception 48 | 49 | def inception_block_1b(X): 50 | X_3x3 = Conv2D(96, (1, 1), data_format='channels_first', name='inception_3b_3x3_conv1')(X) 51 | X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_3x3_bn1')(X_3x3) 52 | X_3x3 = Activation('relu')(X_3x3) 53 | X_3x3 = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_3x3) 54 | X_3x3 = Conv2D(128, (3, 3), data_format='channels_first', name='inception_3b_3x3_conv2')(X_3x3) 55 | X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_3x3_bn2')(X_3x3) 56 | X_3x3 = Activation('relu')(X_3x3) 57 | 58 | X_5x5 = Conv2D(32, (1, 1), data_format='channels_first', name='inception_3b_5x5_conv1')(X) 59 | X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_5x5_bn1')(X_5x5) 60 | X_5x5 = Activation('relu')(X_5x5) 61 | X_5x5 = ZeroPadding2D(padding=(2, 2), data_format='channels_first')(X_5x5) 62 | X_5x5 = Conv2D(64, (5, 5), data_format='channels_first', name='inception_3b_5x5_conv2')(X_5x5) 63 | X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_5x5_bn2')(X_5x5) 64 | X_5x5 = Activation('relu')(X_5x5) 65 | 66 | X_pool = AveragePooling2D(pool_size=(3, 3), strides=(3, 3), data_format='channels_first')(X) 67 | X_pool = Conv2D(64, (1, 1), data_format='channels_first', name='inception_3b_pool_conv')(X_pool) 68 | X_pool = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_pool_bn')(X_pool) 69 | X_pool = Activation('relu')(X_pool) 70 | X_pool = ZeroPadding2D(padding=(4, 4), data_format='channels_first')(X_pool) 71 | 72 | X_1x1 = Conv2D(64, (1, 1), data_format='channels_first', name='inception_3b_1x1_conv')(X) 73 | X_1x1 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_1x1_bn')(X_1x1) 74 | X_1x1 = Activation('relu')(X_1x1) 75 | 76 | inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1) 77 | 78 | return inception 79 | 80 | def inception_block_1c(X): 81 | X_3x3 = fr_utils.conv2d_bn(X, 82 | layer='inception_3c_3x3', 83 | cv1_out=128, 84 | cv1_filter=(1, 1), 85 | cv2_out=256, 86 | cv2_filter=(3, 3), 87 | cv2_strides=(2, 2), 88 | padding=(1, 1)) 89 | 90 | X_5x5 = fr_utils.conv2d_bn(X, 91 | layer='inception_3c_5x5', 92 | cv1_out=32, 93 | cv1_filter=(1, 1), 94 | cv2_out=64, 95 | cv2_filter=(5, 5), 96 | cv2_strides=(2, 2), 97 | padding=(2, 2)) 98 | 99 | X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X) 100 | X_pool = ZeroPadding2D(padding=((0, 1), (0, 1)), data_format='channels_first')(X_pool) 101 | 102 | inception = concatenate([X_3x3, X_5x5, X_pool], axis=1) 103 | 104 | return inception 105 | 106 | def inception_block_2a(X): 107 | X_3x3 = fr_utils.conv2d_bn(X, 108 | layer='inception_4a_3x3', 109 | cv1_out=96, 110 | cv1_filter=(1, 1), 111 | cv2_out=192, 112 | cv2_filter=(3, 3), 113 | cv2_strides=(1, 1), 114 | padding=(1, 1)) 115 | X_5x5 = fr_utils.conv2d_bn(X, 116 | layer='inception_4a_5x5', 117 | cv1_out=32, 118 | cv1_filter=(1, 1), 119 | cv2_out=64, 120 | cv2_filter=(5, 5), 121 | cv2_strides=(1, 1), 122 | padding=(2, 2)) 123 | 124 | X_pool = AveragePooling2D(pool_size=(3, 3), strides=(3, 3), data_format='channels_first')(X) 125 | X_pool = fr_utils.conv2d_bn(X_pool, 126 | layer='inception_4a_pool', 127 | cv1_out=128, 128 | cv1_filter=(1, 1), 129 | padding=(2, 2)) 130 | X_1x1 = fr_utils.conv2d_bn(X, 131 | layer='inception_4a_1x1', 132 | cv1_out=256, 133 | cv1_filter=(1, 1)) 134 | inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1) 135 | 136 | return inception 137 | 138 | def inception_block_2b(X): 139 | #inception4e 140 | X_3x3 = fr_utils.conv2d_bn(X, 141 | layer='inception_4e_3x3', 142 | cv1_out=160, 143 | cv1_filter=(1, 1), 144 | cv2_out=256, 145 | cv2_filter=(3, 3), 146 | cv2_strides=(2, 2), 147 | padding=(1, 1)) 148 | X_5x5 = fr_utils.conv2d_bn(X, 149 | layer='inception_4e_5x5', 150 | cv1_out=64, 151 | cv1_filter=(1, 1), 152 | cv2_out=128, 153 | cv2_filter=(5, 5), 154 | cv2_strides=(2, 2), 155 | padding=(2, 2)) 156 | 157 | X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X) 158 | X_pool = ZeroPadding2D(padding=((0, 1), (0, 1)), data_format='channels_first')(X_pool) 159 | 160 | inception = concatenate([X_3x3, X_5x5, X_pool], axis=1) 161 | 162 | return inception 163 | 164 | def inception_block_3a(X): 165 | X_3x3 = fr_utils.conv2d_bn(X, 166 | layer='inception_5a_3x3', 167 | cv1_out=96, 168 | cv1_filter=(1, 1), 169 | cv2_out=384, 170 | cv2_filter=(3, 3), 171 | cv2_strides=(1, 1), 172 | padding=(1, 1)) 173 | X_pool = AveragePooling2D(pool_size=(3, 3), strides=(3, 3), data_format='channels_first')(X) 174 | X_pool = fr_utils.conv2d_bn(X_pool, 175 | layer='inception_5a_pool', 176 | cv1_out=96, 177 | cv1_filter=(1, 1), 178 | padding=(1, 1)) 179 | X_1x1 = fr_utils.conv2d_bn(X, 180 | layer='inception_5a_1x1', 181 | cv1_out=256, 182 | cv1_filter=(1, 1)) 183 | 184 | inception = concatenate([X_3x3, X_pool, X_1x1], axis=1) 185 | 186 | return inception 187 | 188 | def inception_block_3b(X): 189 | X_3x3 = fr_utils.conv2d_bn(X, 190 | layer='inception_5b_3x3', 191 | cv1_out=96, 192 | cv1_filter=(1, 1), 193 | cv2_out=384, 194 | cv2_filter=(3, 3), 195 | cv2_strides=(1, 1), 196 | padding=(1, 1)) 197 | X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X) 198 | X_pool = fr_utils.conv2d_bn(X_pool, 199 | layer='inception_5b_pool', 200 | cv1_out=96, 201 | cv1_filter=(1, 1)) 202 | X_pool = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_pool) 203 | 204 | X_1x1 = fr_utils.conv2d_bn(X, 205 | layer='inception_5b_1x1', 206 | cv1_out=256, 207 | cv1_filter=(1, 1)) 208 | inception = concatenate([X_3x3, X_pool, X_1x1], axis=1) 209 | 210 | return inception 211 | 212 | def faceRecoModel(input_shape): 213 | """ 214 | Implementation of the Inception model used for FaceNet 215 | 216 | Arguments: 217 | input_shape -- shape of the images of the dataset 218 | 219 | Returns: 220 | model -- a Model() instance in Keras 221 | """ 222 | 223 | # Define the input as a tensor with shape input_shape 224 | X_input = Input(input_shape) 225 | 226 | # Zero-Padding 227 | X = ZeroPadding2D((3, 3))(X_input) 228 | 229 | # First Block 230 | X = Conv2D(64, (7, 7), strides = (2, 2), name = 'conv1')(X) 231 | X = BatchNormalization(axis = 1, name = 'bn1')(X) 232 | X = Activation('relu')(X) 233 | 234 | # Zero-Padding + MAXPOOL 235 | X = ZeroPadding2D((1, 1))(X) 236 | X = MaxPooling2D((3, 3), strides = 2)(X) 237 | 238 | # Second Block 239 | X = Conv2D(64, (1, 1), strides = (1, 1), name = 'conv2')(X) 240 | X = BatchNormalization(axis = 1, epsilon=0.00001, name = 'bn2')(X) 241 | X = Activation('relu')(X) 242 | 243 | # Zero-Padding + MAXPOOL 244 | X = ZeroPadding2D((1, 1))(X) 245 | 246 | # Second Block 247 | X = Conv2D(192, (3, 3), strides = (1, 1), name = 'conv3')(X) 248 | X = BatchNormalization(axis = 1, epsilon=0.00001, name = 'bn3')(X) 249 | X = Activation('relu')(X) 250 | 251 | # Zero-Padding + MAXPOOL 252 | X = ZeroPadding2D((1, 1))(X) 253 | X = MaxPooling2D(pool_size = 3, strides = 2)(X) 254 | 255 | # Inception 1: a/b/c 256 | X = inception_block_1a(X) 257 | X = inception_block_1b(X) 258 | X = inception_block_1c(X) 259 | 260 | # Inception 2: a/b 261 | X = inception_block_2a(X) 262 | X = inception_block_2b(X) 263 | 264 | # Inception 3: a/b 265 | X = inception_block_3a(X) 266 | X = inception_block_3b(X) 267 | 268 | # Top layer 269 | X = AveragePooling2D(pool_size=(3, 3), strides=(1, 1), data_format='channels_first')(X) 270 | X = Flatten()(X) 271 | X = Dense(128, name='dense_layer')(X) 272 | 273 | # L2 normalization 274 | X = Lambda(lambda x: K.l2_normalize(x,axis=1))(X) 275 | 276 | # Create model instance 277 | model = Model(inputs = X_input, outputs = X, name='FaceRecoModel') 278 | 279 | return model -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/nn4.small2.v7.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/nn4.small2.v7.h5 -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/4.Special Applications - Facial Recognition & Neural Style Transfer/nst_utils.py: -------------------------------------------------------------------------------- 1 | ### Part of this code is due to the MatConvNet team and is used to load the parameters of the pretrained VGG19 model in the notebook ### 2 | 3 | import os 4 | import sys 5 | import scipy.io 6 | import scipy.misc 7 | import matplotlib.pyplot as plt 8 | from matplotlib.pyplot import imshow 9 | from PIL import Image 10 | from nst_utils import * 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | 15 | class CONFIG: 16 | IMAGE_WIDTH = 400 17 | IMAGE_HEIGHT = 300 18 | COLOR_CHANNELS = 3 19 | NOISE_RATIO = 0.6 20 | MEANS = np.array([123.68, 116.779, 103.939]).reshape((1,1,1,3)) 21 | VGG_MODEL = 'pretrained-model/imagenet-vgg-verydeep-19.mat' # Pick the VGG 19-layer model by from the paper "Very Deep Convolutional Networks for Large-Scale Image Recognition". 22 | STYLE_IMAGE = 'images/stone_style.jpg' # Style image to use. 23 | CONTENT_IMAGE = 'images/content300.jpg' # Content image to use. 24 | OUTPUT_DIR = 'output/' 25 | 26 | def load_vgg_model(path): 27 | """ 28 | Returns a model for the purpose of 'painting' the picture. 29 | Takes only the convolution layer weights and wrap using the TensorFlow 30 | Conv2d, Relu and AveragePooling layer. VGG actually uses maxpool but 31 | the paper indicates that using AveragePooling yields better results. 32 | The last few fully connected layers are not used. 33 | Here is the detailed configuration of the VGG model: 34 | 0 is conv1_1 (3, 3, 3, 64) 35 | 1 is relu 36 | 2 is conv1_2 (3, 3, 64, 64) 37 | 3 is relu 38 | 4 is maxpool 39 | 5 is conv2_1 (3, 3, 64, 128) 40 | 6 is relu 41 | 7 is conv2_2 (3, 3, 128, 128) 42 | 8 is relu 43 | 9 is maxpool 44 | 10 is conv3_1 (3, 3, 128, 256) 45 | 11 is relu 46 | 12 is conv3_2 (3, 3, 256, 256) 47 | 13 is relu 48 | 14 is conv3_3 (3, 3, 256, 256) 49 | 15 is relu 50 | 16 is conv3_4 (3, 3, 256, 256) 51 | 17 is relu 52 | 18 is maxpool 53 | 19 is conv4_1 (3, 3, 256, 512) 54 | 20 is relu 55 | 21 is conv4_2 (3, 3, 512, 512) 56 | 22 is relu 57 | 23 is conv4_3 (3, 3, 512, 512) 58 | 24 is relu 59 | 25 is conv4_4 (3, 3, 512, 512) 60 | 26 is relu 61 | 27 is maxpool 62 | 28 is conv5_1 (3, 3, 512, 512) 63 | 29 is relu 64 | 30 is conv5_2 (3, 3, 512, 512) 65 | 31 is relu 66 | 32 is conv5_3 (3, 3, 512, 512) 67 | 33 is relu 68 | 34 is conv5_4 (3, 3, 512, 512) 69 | 35 is relu 70 | 36 is maxpool 71 | 37 is fullyconnected (7, 7, 512, 4096) 72 | 38 is relu 73 | 39 is fullyconnected (1, 1, 4096, 4096) 74 | 40 is relu 75 | 41 is fullyconnected (1, 1, 4096, 1000) 76 | 42 is softmax 77 | """ 78 | 79 | vgg = scipy.io.loadmat(path) 80 | 81 | vgg_layers = vgg['layers'] 82 | 83 | def _weights(layer, expected_layer_name): 84 | """ 85 | Return the weights and bias from the VGG model for a given layer. 86 | """ 87 | wb = vgg_layers[0][layer][0][0][2] 88 | W = wb[0][0] 89 | b = wb[0][1] 90 | layer_name = vgg_layers[0][layer][0][0][0][0] 91 | assert layer_name == expected_layer_name 92 | return W, b 93 | 94 | return W, b 95 | 96 | def _relu(conv2d_layer): 97 | """ 98 | Return the RELU function wrapped over a TensorFlow layer. Expects a 99 | Conv2d layer input. 100 | """ 101 | return tf.nn.relu(conv2d_layer) 102 | 103 | def _conv2d(prev_layer, layer, layer_name): 104 | """ 105 | Return the Conv2D layer using the weights, biases from the VGG 106 | model at 'layer'. 107 | """ 108 | W, b = _weights(layer, layer_name) 109 | W = tf.constant(W) 110 | b = tf.constant(np.reshape(b, (b.size))) 111 | return tf.nn.conv2d(prev_layer, filter=W, strides=[1, 1, 1, 1], padding='SAME') + b 112 | 113 | def _conv2d_relu(prev_layer, layer, layer_name): 114 | """ 115 | Return the Conv2D + RELU layer using the weights, biases from the VGG 116 | model at 'layer'. 117 | """ 118 | return _relu(_conv2d(prev_layer, layer, layer_name)) 119 | 120 | def _avgpool(prev_layer): 121 | """ 122 | Return the AveragePooling layer. 123 | """ 124 | return tf.nn.avg_pool(prev_layer, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') 125 | 126 | # Constructs the graph model. 127 | graph = {} 128 | graph['input'] = tf.Variable(np.zeros((1, CONFIG.IMAGE_HEIGHT, CONFIG.IMAGE_WIDTH, CONFIG.COLOR_CHANNELS)), dtype = 'float32') 129 | graph['conv1_1'] = _conv2d_relu(graph['input'], 0, 'conv1_1') 130 | graph['conv1_2'] = _conv2d_relu(graph['conv1_1'], 2, 'conv1_2') 131 | graph['avgpool1'] = _avgpool(graph['conv1_2']) 132 | graph['conv2_1'] = _conv2d_relu(graph['avgpool1'], 5, 'conv2_1') 133 | graph['conv2_2'] = _conv2d_relu(graph['conv2_1'], 7, 'conv2_2') 134 | graph['avgpool2'] = _avgpool(graph['conv2_2']) 135 | graph['conv3_1'] = _conv2d_relu(graph['avgpool2'], 10, 'conv3_1') 136 | graph['conv3_2'] = _conv2d_relu(graph['conv3_1'], 12, 'conv3_2') 137 | graph['conv3_3'] = _conv2d_relu(graph['conv3_2'], 14, 'conv3_3') 138 | graph['conv3_4'] = _conv2d_relu(graph['conv3_3'], 16, 'conv3_4') 139 | graph['avgpool3'] = _avgpool(graph['conv3_4']) 140 | graph['conv4_1'] = _conv2d_relu(graph['avgpool3'], 19, 'conv4_1') 141 | graph['conv4_2'] = _conv2d_relu(graph['conv4_1'], 21, 'conv4_2') 142 | graph['conv4_3'] = _conv2d_relu(graph['conv4_2'], 23, 'conv4_3') 143 | graph['conv4_4'] = _conv2d_relu(graph['conv4_3'], 25, 'conv4_4') 144 | graph['avgpool4'] = _avgpool(graph['conv4_4']) 145 | graph['conv5_1'] = _conv2d_relu(graph['avgpool4'], 28, 'conv5_1') 146 | graph['conv5_2'] = _conv2d_relu(graph['conv5_1'], 30, 'conv5_2') 147 | graph['conv5_3'] = _conv2d_relu(graph['conv5_2'], 32, 'conv5_3') 148 | graph['conv5_4'] = _conv2d_relu(graph['conv5_3'], 34, 'conv5_4') 149 | graph['avgpool5'] = _avgpool(graph['conv5_4']) 150 | 151 | return graph 152 | 153 | def generate_noise_image(content_image, noise_ratio = CONFIG.NOISE_RATIO): 154 | """ 155 | Generates a noisy image by adding random noise to the content_image 156 | """ 157 | 158 | # Generate a random noise_image 159 | noise_image = np.random.uniform(-20, 20, (1, CONFIG.IMAGE_HEIGHT, CONFIG.IMAGE_WIDTH, CONFIG.COLOR_CHANNELS)).astype('float32') 160 | 161 | # Set the input_image to be a weighted average of the content_image and a noise_image 162 | input_image = noise_image * noise_ratio + content_image * (1 - noise_ratio) 163 | 164 | return input_image 165 | 166 | 167 | def reshape_and_normalize_image(image): 168 | """ 169 | Reshape and normalize the input image (content or style) 170 | """ 171 | 172 | # Reshape image to mach expected input of VGG16 173 | image = np.reshape(image, ((1,) + image.shape)) 174 | 175 | # Substract the mean to match the expected input of VGG16 176 | image = image - CONFIG.MEANS 177 | 178 | return image 179 | 180 | 181 | def save_image(path, image): 182 | 183 | # Un-normalize the image so that it looks good 184 | image = image + CONFIG.MEANS 185 | 186 | # Clip and Save the image 187 | image = np.clip(image[0], 0, 255).astype('uint8') 188 | scipy.misc.imsave(path, image) -------------------------------------------------------------------------------- /4.Convolutional Neural Networks/Object Detection/sample.txt: -------------------------------------------------------------------------------- 1 | sample 2 | -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/data_utils.py: -------------------------------------------------------------------------------- 1 | from music_utils import * 2 | from preprocess import * 3 | from keras.utils import to_categorical 4 | 5 | chords, abstract_grammars = get_musical_data('data/original_metheny.mid') 6 | corpus, tones, tones_indices, indices_tones = get_corpus_data(abstract_grammars) 7 | N_tones = len(set(corpus)) 8 | n_a = 64 9 | x_initializer = np.zeros((1, 1, 78)) 10 | a_initializer = np.zeros((1, n_a)) 11 | c_initializer = np.zeros((1, n_a)) 12 | 13 | def load_music_utils(): 14 | chords, abstract_grammars = get_musical_data('data/original_metheny.mid') 15 | corpus, tones, tones_indices, indices_tones = get_corpus_data(abstract_grammars) 16 | N_tones = len(set(corpus)) 17 | X, Y, N_tones = data_processing(corpus, tones_indices, 60, 30) 18 | return (X, Y, N_tones, indices_tones) 19 | 20 | 21 | def generate_music(inference_model, corpus = corpus, abstract_grammars = abstract_grammars, tones = tones, tones_indices = tones_indices, indices_tones = indices_tones, T_y = 10, max_tries = 1000, diversity = 0.5): 22 | """ 23 | Generates music using a model trained to learn musical patterns of a jazz soloist. Creates an audio stream 24 | to save the music and play it. 25 | 26 | Arguments: 27 | model -- Keras model Instance, output of djmodel() 28 | corpus -- musical corpus, list of 193 tones as strings (ex: 'C,0.333,') 29 | abstract_grammars -- list of grammars, on element can be: 'S,0.250, C,0.250, A,0.250,' 30 | tones -- set of unique tones, ex: 'A,0.250,' is one element of the set. 31 | tones_indices -- a python dictionary mapping unique tone (ex: A,0.250,< m2,P-4 >) into their corresponding indices (0-77) 32 | indices_tones -- a python dictionary mapping indices (0-77) into their corresponding unique tone (ex: A,0.250,< m2,P-4 >) 33 | Tx -- integer, number of time-steps used at training time 34 | temperature -- scalar value, defines how conservative/creative the model is when generating music 35 | 36 | Returns: 37 | predicted_tones -- python list containing predicted tones 38 | """ 39 | 40 | # set up audio stream 41 | out_stream = stream.Stream() 42 | 43 | # Initialize chord variables 44 | curr_offset = 0.0 # variable used to write sounds to the Stream. 45 | num_chords = int(len(chords) / 3) # number of different set of chords 46 | 47 | print("Predicting new values for different set of chords.") 48 | # Loop over all 18 set of chords. At each iteration generate a sequence of tones 49 | # and use the current chords to convert it into actual sounds 50 | for i in range(1, num_chords): 51 | 52 | # Retrieve current chord from stream 53 | curr_chords = stream.Voice() 54 | 55 | # Loop over the chords of the current set of chords 56 | for j in chords[i]: 57 | # Add chord to the current chords with the adequate offset, no need to understand this 58 | curr_chords.insert((j.offset % 4), j) 59 | 60 | # Generate a sequence of tones using the model 61 | _, indices = predict_and_sample(inference_model) 62 | indices = list(indices.squeeze()) 63 | pred = [indices_tones[p] for p in indices] 64 | 65 | predicted_tones = 'C,0.25 ' 66 | for k in range(len(pred) - 1): 67 | predicted_tones += pred[k] + ' ' 68 | 69 | predicted_tones += pred[-1] 70 | 71 | #### POST PROCESSING OF THE PREDICTED TONES #### 72 | # We will consider "A" and "X" as "C" tones. It is a common choice. 73 | predicted_tones = predicted_tones.replace(' A',' C').replace(' X',' C') 74 | 75 | # Pruning #1: smoothing measure 76 | predicted_tones = prune_grammar(predicted_tones) 77 | 78 | # Use predicted tones and current chords to generate sounds 79 | sounds = unparse_grammar(predicted_tones, curr_chords) 80 | 81 | # Pruning #2: removing repeated and too close together sounds 82 | sounds = prune_notes(sounds) 83 | 84 | # Quality assurance: clean up sounds 85 | sounds = clean_up_notes(sounds) 86 | 87 | # Print number of tones/notes in sounds 88 | print('Generated %s sounds using the predicted values for the set of chords ("%s") and after pruning' % (len([k for k in sounds if isinstance(k, note.Note)]), i)) 89 | 90 | # Insert sounds into the output stream 91 | for m in sounds: 92 | out_stream.insert(curr_offset + m.offset, m) 93 | for mc in curr_chords: 94 | out_stream.insert(curr_offset + mc.offset, mc) 95 | 96 | curr_offset += 4.0 97 | 98 | # Initialize tempo of the output stream with 130 bit per minute 99 | out_stream.insert(0.0, tempo.MetronomeMark(number=130)) 100 | 101 | # Save audio stream to fine 102 | mf = midi.translate.streamToMidiFile(out_stream) 103 | mf.open("output/my_music.midi", 'wb') 104 | mf.write() 105 | print("Your generated music is saved in output/my_music.midi") 106 | mf.close() 107 | 108 | # Play the final stream through output (see 'play' lambda function above) 109 | # play = lambda x: midi.realtime.StreamPlayer(x).play() 110 | # play(out_stream) 111 | 112 | return out_stream 113 | 114 | 115 | def predict_and_sample(inference_model, x_initializer = x_initializer, a_initializer = a_initializer, 116 | c_initializer = c_initializer): 117 | """ 118 | Predicts the next value of values using the inference model. 119 | 120 | Arguments: 121 | inference_model -- Keras model instance for inference time 122 | x_initializer -- numpy array of shape (1, 1, 78), one-hot vector initializing the values generation 123 | a_initializer -- numpy array of shape (1, n_a), initializing the hidden state of the LSTM_cell 124 | c_initializer -- numpy array of shape (1, n_a), initializing the cell state of the LSTM_cel 125 | Ty -- length of the sequence you'd like to generate. 126 | 127 | Returns: 128 | results -- numpy-array of shape (Ty, 78), matrix of one-hot vectors representing the values generated 129 | indices -- numpy-array of shape (Ty, 1), matrix of indices representing the values generated 130 | """ 131 | 132 | ### START CODE HERE ### 133 | pred = inference_model.predict([x_initializer, a_initializer, c_initializer]) 134 | indices = np.argmax(pred, axis = -1) 135 | results = to_categorical(indices, num_classes=78) 136 | ### END CODE HERE ### 137 | 138 | return results, indices -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/music_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import tensorflow as tf 3 | import keras.backend as K 4 | from keras.layers import RepeatVector 5 | import sys 6 | from music21 import * 7 | import numpy as np 8 | from grammar import * 9 | from preprocess import * 10 | from qa import * 11 | 12 | 13 | def data_processing(corpus, values_indices, m = 60, Tx = 30): 14 | # cut the corpus into semi-redundant sequences of Tx values 15 | Tx = Tx 16 | N_values = len(set(corpus)) 17 | np.random.seed(0) 18 | X = np.zeros((m, Tx, N_values), dtype=np.bool) 19 | Y = np.zeros((m, Tx, N_values), dtype=np.bool) 20 | for i in range(m): 21 | # for t in range(1, Tx): 22 | random_idx = np.random.choice(len(corpus) - Tx) 23 | corp_data = corpus[random_idx:(random_idx + Tx)] 24 | for j in range(Tx): 25 | idx = values_indices[corp_data[j]] 26 | if j != 0: 27 | X[i, j, idx] = 1 28 | Y[i, j-1, idx] = 1 29 | 30 | Y = np.swapaxes(Y,0,1) 31 | Y = Y.tolist() 32 | return np.asarray(X), np.asarray(Y), N_values 33 | 34 | def next_value_processing(model, next_value, x, predict_and_sample, indices_values, abstract_grammars, duration, max_tries = 1000, temperature = 0.5): 35 | """ 36 | Helper function to fix the first value. 37 | 38 | Arguments: 39 | next_value -- predicted and sampled value, index between 0 and 77 40 | x -- numpy-array, one-hot encoding of next_value 41 | predict_and_sample -- predict function 42 | indices_values -- a python dictionary mapping indices (0-77) into their corresponding unique value (ex: A,0.250,< m2,P-4 >) 43 | abstract_grammars -- list of grammars, on element can be: 'S,0.250, C,0.250, A,0.250,' 44 | duration -- scalar, index of the loop in the parent function 45 | max_tries -- Maximum numbers of time trying to fix the value 46 | 47 | Returns: 48 | next_value -- process predicted value 49 | """ 50 | 51 | # fix first note: must not have < > and not be a rest 52 | if (duration < 0.00001): 53 | tries = 0 54 | while (next_value.split(',')[0] == 'R' or 55 | len(next_value.split(',')) != 2): 56 | # give up after 1000 tries; random from input's first notes 57 | if tries >= max_tries: 58 | #print('Gave up on first note generation after', max_tries, 'tries') 59 | # np.random is exclusive to high 60 | rand = np.random.randint(0, len(abstract_grammars)) 61 | next_value = abstract_grammars[rand].split(' ')[0] 62 | else: 63 | next_value = predict_and_sample(model, x, indices_values, temperature) 64 | 65 | tries += 1 66 | 67 | return next_value 68 | 69 | 70 | def sequence_to_matrix(sequence, values_indices): 71 | """ 72 | Convert a sequence (slice of the corpus) into a matrix (numpy) of one-hot vectors corresponding 73 | to indices in values_indices 74 | 75 | Arguments: 76 | sequence -- python list 77 | 78 | Returns: 79 | x -- numpy-array of one-hot vectors 80 | """ 81 | sequence_len = len(sequence) 82 | x = np.zeros((1, sequence_len, len(values_indices))) 83 | for t, value in enumerate(sequence): 84 | if (not value in values_indices): print(value) 85 | x[0, t, values_indices[value]] = 1. 86 | return x 87 | 88 | def one_hot(x): 89 | x = K.argmax(x) 90 | x = tf.one_hot(x, 78) 91 | x = RepeatVector(1)(x) 92 | return x -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/my_music.midi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcopeix/Deep_Learning_AI/b9a6ebc5d688223fbadcfdbfb7dfcce4319dccd9/5.Sequence Models/1.Recurrent Neural Networks/my_music.midi -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/preprocess.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Author: Ji-Sung Kim 3 | Project: deepjazz 4 | Purpose: Parse, cleanup and process data. 5 | 6 | Code adapted from Evan Chow's jazzml, https://github.com/evancchow/jazzml with 7 | express permission. 8 | ''' 9 | 10 | from __future__ import print_function 11 | 12 | from music21 import * 13 | from collections import defaultdict, OrderedDict 14 | from itertools import groupby, zip_longest 15 | 16 | from grammar import * 17 | 18 | from grammar import parse_melody 19 | from music_utils import * 20 | 21 | #----------------------------HELPER FUNCTIONS----------------------------------# 22 | 23 | ''' Helper function to parse a MIDI file into its measures and chords ''' 24 | def __parse_midi(data_fn): 25 | # Parse the MIDI data for separate melody and accompaniment parts. 26 | midi_data = converter.parse(data_fn) 27 | # Get melody part, compress into single voice. 28 | melody_stream = midi_data[5] # For Metheny piece, Melody is Part #5. 29 | melody1, melody2 = melody_stream.getElementsByClass(stream.Voice) 30 | for j in melody2: 31 | melody1.insert(j.offset, j) 32 | melody_voice = melody1 33 | 34 | for i in melody_voice: 35 | if i.quarterLength == 0.0: 36 | i.quarterLength = 0.25 37 | 38 | # Change key signature to adhere to comp_stream (1 sharp, mode = major). 39 | # Also add Electric Guitar. 40 | melody_voice.insert(0, instrument.ElectricGuitar()) 41 | melody_voice.insert(0, key.KeySignature(sharps=1)) 42 | 43 | # The accompaniment parts. Take only the best subset of parts from 44 | # the original data. Maybe add more parts, hand-add valid instruments. 45 | # Should add least add a string part (for sparse solos). 46 | # Verified are good parts: 0, 1, 6, 7 ''' 47 | partIndices = [0, 1, 6, 7] 48 | comp_stream = stream.Voice() 49 | comp_stream.append([j.flat for i, j in enumerate(midi_data) 50 | if i in partIndices]) 51 | 52 | # Full stream containing both the melody and the accompaniment. 53 | # All parts are flattened. 54 | full_stream = stream.Voice() 55 | for i in range(len(comp_stream)): 56 | full_stream.append(comp_stream[i]) 57 | full_stream.append(melody_voice) 58 | 59 | # Extract solo stream, assuming you know the positions ..ByOffset(i, j). 60 | # Note that for different instruments (with stream.flat), you NEED to use 61 | # stream.Part(), not stream.Voice(). 62 | # Accompanied solo is in range [478, 548) 63 | solo_stream = stream.Voice() 64 | for part in full_stream: 65 | curr_part = stream.Part() 66 | curr_part.append(part.getElementsByClass(instrument.Instrument)) 67 | curr_part.append(part.getElementsByClass(tempo.MetronomeMark)) 68 | curr_part.append(part.getElementsByClass(key.KeySignature)) 69 | curr_part.append(part.getElementsByClass(meter.TimeSignature)) 70 | curr_part.append(part.getElementsByOffset(476, 548, 71 | includeEndBoundary=True)) 72 | cp = curr_part.flat 73 | solo_stream.insert(cp) 74 | 75 | # Group by measure so you can classify. 76 | # Note that measure 0 is for the time signature, metronome, etc. which have 77 | # an offset of 0.0. 78 | melody_stream = solo_stream[-1] 79 | measures = OrderedDict() 80 | offsetTuples = [(int(n.offset / 4), n) for n in melody_stream] 81 | measureNum = 0 # for now, don't use real m. nums (119, 120) 82 | for key_x, group in groupby(offsetTuples, lambda x: x[0]): 83 | measures[measureNum] = [n[1] for n in group] 84 | measureNum += 1 85 | 86 | # Get the stream of chords. 87 | # offsetTuples_chords: group chords by measure number. 88 | chordStream = solo_stream[0] 89 | chordStream.removeByClass(note.Rest) 90 | chordStream.removeByClass(note.Note) 91 | offsetTuples_chords = [(int(n.offset / 4), n) for n in chordStream] 92 | 93 | # Generate the chord structure. Use just track 1 (piano) since it is 94 | # the only instrument that has chords. 95 | # Group into 4s, just like before. 96 | chords = OrderedDict() 97 | measureNum = 0 98 | for key_x, group in groupby(offsetTuples_chords, lambda x: x[0]): 99 | chords[measureNum] = [n[1] for n in group] 100 | measureNum += 1 101 | 102 | # Fix for the below problem. 103 | # 1) Find out why len(measures) != len(chords). 104 | # ANSWER: resolves at end but melody ends 1/16 before last measure so doesn't 105 | # actually show up, while the accompaniment's beat 1 right after does. 106 | # Actually on second thought: melody/comp start on Ab, and resolve to 107 | # the same key (Ab) so could actually just cut out last measure to loop. 108 | # Decided: just cut out the last measure. 109 | del chords[len(chords) - 1] 110 | assert len(chords) == len(measures) 111 | 112 | return measures, chords 113 | 114 | ''' Helper function to get the grammatical data from given musical data. ''' 115 | def __get_abstract_grammars(measures, chords): 116 | # extract grammars 117 | abstract_grammars = [] 118 | for ix in range(1, len(measures)): 119 | m = stream.Voice() 120 | for i in measures[ix]: 121 | m.insert(i.offset, i) 122 | c = stream.Voice() 123 | for j in chords[ix]: 124 | c.insert(j.offset, j) 125 | parsed = parse_melody(m, c) 126 | abstract_grammars.append(parsed) 127 | 128 | return abstract_grammars 129 | 130 | #----------------------------PUBLIC FUNCTIONS----------------------------------# 131 | 132 | ''' Get musical data from a MIDI file ''' 133 | def get_musical_data(data_fn): 134 | 135 | measures, chords = __parse_midi(data_fn) 136 | abstract_grammars = __get_abstract_grammars(measures, chords) 137 | 138 | return chords, abstract_grammars 139 | 140 | ''' Get corpus data from grammatical data ''' 141 | def get_corpus_data(abstract_grammars): 142 | corpus = [x for sublist in abstract_grammars for x in sublist.split(' ')] 143 | values = set(corpus) 144 | val_indices = dict((v, i) for i, v in enumerate(values)) 145 | indices_val = dict((i, v) for i, v in enumerate(values)) 146 | 147 | return corpus, values, val_indices, indices_val 148 | 149 | ''' 150 | def load_music_utils(): 151 | chord_data, raw_music_data = get_musical_data('data/original_metheny.mid') 152 | music_data, values, values_indices, indices_values = get_corpus_data(raw_music_data) 153 | 154 | X, Y = data_processing(music_data, values_indices, Tx = 20, step = 3) 155 | return (X, Y) 156 | ''' 157 | -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/qa.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Author: Ji-Sung Kim, Evan Chow 3 | Project: deepjazz 4 | Purpose: Provide pruning and cleanup functions. 5 | 6 | Code adapted from Evan Chow's jazzml, https://github.com/evancchow/jazzml 7 | with express permission. 8 | ''' 9 | from itertools import zip_longest 10 | import random 11 | 12 | from music21 import * 13 | 14 | #----------------------------HELPER FUNCTIONS----------------------------------# 15 | 16 | ''' Helper function to down num to the nearest multiple of mult. ''' 17 | def __roundDown(num, mult): 18 | return (float(num) - (float(num) % mult)) 19 | 20 | ''' Helper function to round up num to nearest multiple of mult. ''' 21 | def __roundUp(num, mult): 22 | return __roundDown(num, mult) + mult 23 | 24 | ''' Helper function that, based on if upDown < 0 or upDown >= 0, rounds number 25 | down or up respectively to nearest multiple of mult. ''' 26 | def __roundUpDown(num, mult, upDown): 27 | if upDown < 0: 28 | return __roundDown(num, mult) 29 | else: 30 | return __roundUp(num, mult) 31 | 32 | ''' Helper function, from recipes, to iterate over list in chunks of n 33 | length. ''' 34 | def __grouper(iterable, n, fillvalue=None): 35 | args = [iter(iterable)] * n 36 | return zip_longest(*args, fillvalue=fillvalue) 37 | 38 | #----------------------------PUBLIC FUNCTIONS----------------------------------# 39 | 40 | ''' Smooth the measure, ensuring that everything is in standard note lengths 41 | (e.g., 0.125, 0.250, 0.333 ... ). ''' 42 | def prune_grammar(curr_grammar): 43 | pruned_grammar = curr_grammar.split(' ') 44 | 45 | for ix, gram in enumerate(pruned_grammar): 46 | terms = gram.split(',') 47 | terms[1] = str(__roundUpDown(float(terms[1]), 0.250, 48 | random.choice([-1, 1]))) 49 | pruned_grammar[ix] = ','.join(terms) 50 | pruned_grammar = ' '.join(pruned_grammar) 51 | 52 | return pruned_grammar 53 | 54 | ''' Remove repeated notes, and notes that are too close together. ''' 55 | def prune_notes(curr_notes): 56 | for n1, n2 in __grouper(curr_notes, n=2): 57 | if n2 == None: # corner case: odd-length list 58 | continue 59 | if isinstance(n1, note.Note) and isinstance(n2, note.Note): 60 | if n1.nameWithOctave == n2.nameWithOctave: 61 | curr_notes.remove(n2) 62 | 63 | return curr_notes 64 | 65 | ''' Perform quality assurance on notes ''' 66 | def clean_up_notes(curr_notes): 67 | removeIxs = [] 68 | for ix, m in enumerate(curr_notes): 69 | # QA1: ensure nothing is of 0 quarter note len, if so changes its len 70 | if (m.quarterLength == 0.0): 71 | m.quarterLength = 0.250 72 | # QA2: ensure no two melody notes have same offset, i.e. form a chord. 73 | # Sorted, so same offset would be consecutive notes. 74 | if (ix < (len(curr_notes) - 1)): 75 | if (m.offset == curr_notes[ix + 1].offset and 76 | isinstance(curr_notes[ix + 1], note.Note)): 77 | removeIxs.append((ix + 1)) 78 | curr_notes = [i for ix, i in enumerate(curr_notes) if ix not in removeIxs] 79 | 80 | return curr_notes -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/rnn_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def softmax(x): 4 | e_x = np.exp(x - np.max(x)) 5 | return e_x / e_x.sum(axis=0) 6 | 7 | 8 | def sigmoid(x): 9 | return 1 / (1 + np.exp(-x)) 10 | 11 | 12 | def initialize_adam(parameters) : 13 | """ 14 | Initializes v and s as two python dictionaries with: 15 | - keys: "dW1", "db1", ..., "dWL", "dbL" 16 | - values: numpy arrays of zeros of the same shape as the corresponding gradients/parameters. 17 | 18 | Arguments: 19 | parameters -- python dictionary containing your parameters. 20 | parameters["W" + str(l)] = Wl 21 | parameters["b" + str(l)] = bl 22 | 23 | Returns: 24 | v -- python dictionary that will contain the exponentially weighted average of the gradient. 25 | v["dW" + str(l)] = ... 26 | v["db" + str(l)] = ... 27 | s -- python dictionary that will contain the exponentially weighted average of the squared gradient. 28 | s["dW" + str(l)] = ... 29 | s["db" + str(l)] = ... 30 | 31 | """ 32 | 33 | L = len(parameters) // 2 # number of layers in the neural networks 34 | v = {} 35 | s = {} 36 | 37 | # Initialize v, s. Input: "parameters". Outputs: "v, s". 38 | for l in range(L): 39 | ### START CODE HERE ### (approx. 4 lines) 40 | v["dW" + str(l+1)] = np.zeros(parameters["W" + str(l+1)].shape) 41 | v["db" + str(l+1)] = np.zeros(parameters["b" + str(l+1)].shape) 42 | s["dW" + str(l+1)] = np.zeros(parameters["W" + str(l+1)].shape) 43 | s["db" + str(l+1)] = np.zeros(parameters["b" + str(l+1)].shape) 44 | ### END CODE HERE ### 45 | 46 | return v, s 47 | 48 | 49 | def update_parameters_with_adam(parameters, grads, v, s, t, learning_rate = 0.01, 50 | beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8): 51 | """ 52 | Update parameters using Adam 53 | 54 | Arguments: 55 | parameters -- python dictionary containing your parameters: 56 | parameters['W' + str(l)] = Wl 57 | parameters['b' + str(l)] = bl 58 | grads -- python dictionary containing your gradients for each parameters: 59 | grads['dW' + str(l)] = dWl 60 | grads['db' + str(l)] = dbl 61 | v -- Adam variable, moving average of the first gradient, python dictionary 62 | s -- Adam variable, moving average of the squared gradient, python dictionary 63 | learning_rate -- the learning rate, scalar. 64 | beta1 -- Exponential decay hyperparameter for the first moment estimates 65 | beta2 -- Exponential decay hyperparameter for the second moment estimates 66 | epsilon -- hyperparameter preventing division by zero in Adam updates 67 | 68 | Returns: 69 | parameters -- python dictionary containing your updated parameters 70 | v -- Adam variable, moving average of the first gradient, python dictionary 71 | s -- Adam variable, moving average of the squared gradient, python dictionary 72 | """ 73 | 74 | L = len(parameters) // 2 # number of layers in the neural networks 75 | v_corrected = {} # Initializing first moment estimate, python dictionary 76 | s_corrected = {} # Initializing second moment estimate, python dictionary 77 | 78 | # Perform Adam update on all parameters 79 | for l in range(L): 80 | # Moving average of the gradients. Inputs: "v, grads, beta1". Output: "v". 81 | ### START CODE HERE ### (approx. 2 lines) 82 | v["dW" + str(l+1)] = beta1 * v["dW" + str(l+1)] + (1 - beta1) * grads["dW" + str(l+1)] 83 | v["db" + str(l+1)] = beta1 * v["db" + str(l+1)] + (1 - beta1) * grads["db" + str(l+1)] 84 | ### END CODE HERE ### 85 | 86 | # Compute bias-corrected first moment estimate. Inputs: "v, beta1, t". Output: "v_corrected". 87 | ### START CODE HERE ### (approx. 2 lines) 88 | v_corrected["dW" + str(l+1)] = v["dW" + str(l+1)] / (1 - beta1**t) 89 | v_corrected["db" + str(l+1)] = v["db" + str(l+1)] / (1 - beta1**t) 90 | ### END CODE HERE ### 91 | 92 | # Moving average of the squared gradients. Inputs: "s, grads, beta2". Output: "s". 93 | ### START CODE HERE ### (approx. 2 lines) 94 | s["dW" + str(l+1)] = beta2 * s["dW" + str(l+1)] + (1 - beta2) * (grads["dW" + str(l+1)] ** 2) 95 | s["db" + str(l+1)] = beta2 * s["db" + str(l+1)] + (1 - beta2) * (grads["db" + str(l+1)] ** 2) 96 | ### END CODE HERE ### 97 | 98 | # Compute bias-corrected second raw moment estimate. Inputs: "s, beta2, t". Output: "s_corrected". 99 | ### START CODE HERE ### (approx. 2 lines) 100 | s_corrected["dW" + str(l+1)] = s["dW" + str(l+1)] / (1 - beta2 ** t) 101 | s_corrected["db" + str(l+1)] = s["db" + str(l+1)] / (1 - beta2 ** t) 102 | ### END CODE HERE ### 103 | 104 | # Update parameters. Inputs: "parameters, learning_rate, v_corrected, s_corrected, epsilon". Output: "parameters". 105 | ### START CODE HERE ### (approx. 2 lines) 106 | parameters["W" + str(l+1)] = parameters["W" + str(l+1)] - learning_rate * v_corrected["dW" + str(l+1)] / np.sqrt(s_corrected["dW" + str(l+1)] + epsilon) 107 | parameters["b" + str(l+1)] = parameters["b" + str(l+1)] - learning_rate * v_corrected["db" + str(l+1)] / np.sqrt(s_corrected["db" + str(l+1)] + epsilon) 108 | ### END CODE HERE ### 109 | 110 | return parameters, v, s -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/sample.txt: -------------------------------------------------------------------------------- 1 | sample 2 | -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/shakespeare_utils.py: -------------------------------------------------------------------------------- 1 | # Load Packages 2 | from __future__ import print_function 3 | from keras.callbacks import LambdaCallback 4 | from keras.models import Model, load_model, Sequential 5 | from keras.layers import Dense, Activation, Dropout, Input, Masking 6 | from keras.layers import LSTM 7 | from keras.utils.data_utils import get_file 8 | from keras.preprocessing.sequence import pad_sequences 9 | import numpy as np 10 | import random 11 | import sys 12 | import io 13 | 14 | def build_data(text, Tx = 40, stride = 3): 15 | """ 16 | Create a training set by scanning a window of size Tx over the text corpus, with stride 3. 17 | 18 | Arguments: 19 | text -- string, corpus of Shakespearian poem 20 | Tx -- sequence length, number of time-steps (or characters) in one training example 21 | stride -- how much the window shifts itself while scanning 22 | 23 | Returns: 24 | X -- list of training examples 25 | Y -- list of training labels 26 | """ 27 | 28 | X = [] 29 | Y = [] 30 | 31 | ### START CODE HERE ### (≈ 3 lines) 32 | for i in range(0, len(text) - Tx, stride): 33 | X.append(text[i: i + Tx]) 34 | Y.append(text[i + Tx]) 35 | ### END CODE HERE ### 36 | 37 | print('number of training examples:', len(X)) 38 | 39 | return X, Y 40 | 41 | 42 | def vectorization(X, Y, n_x, char_indices, Tx = 40): 43 | """ 44 | Convert X and Y (lists) into arrays to be given to a recurrent neural network. 45 | 46 | Arguments: 47 | X -- 48 | Y -- 49 | Tx -- integer, sequence length 50 | 51 | Returns: 52 | x -- array of shape (m, Tx, len(chars)) 53 | y -- array of shape (m, len(chars)) 54 | """ 55 | 56 | m = len(X) 57 | x = np.zeros((m, Tx, n_x), dtype=np.bool) 58 | y = np.zeros((m, n_x), dtype=np.bool) 59 | for i, sentence in enumerate(X): 60 | for t, char in enumerate(sentence): 61 | x[i, t, char_indices[char]] = 1 62 | y[i, char_indices[Y[i]]] = 1 63 | 64 | return x, y 65 | 66 | 67 | def sample(preds, temperature=1.0): 68 | # helper function to sample an index from a probability array 69 | preds = np.asarray(preds).astype('float64') 70 | preds = np.log(preds) / temperature 71 | exp_preds = np.exp(preds) 72 | preds = exp_preds / np.sum(exp_preds) 73 | probas = np.random.multinomial(1, preds, 1) 74 | out = np.random.choice(range(len(chars)), p = probas.ravel()) 75 | return out 76 | #return np.argmax(probas) 77 | 78 | def on_epoch_end(epoch, logs): 79 | # Function invoked at end of each epoch. Prints generated text. 80 | None 81 | #start_index = random.randint(0, len(text) - Tx - 1) 82 | 83 | #generated = '' 84 | #sentence = text[start_index: start_index + Tx] 85 | #sentence = '0'*Tx 86 | #usr_input = input("Write the beginning of your poem, the Shakespearian machine will complete it.") 87 | # zero pad the sentence to Tx characters. 88 | #sentence = ('{0:0>' + str(Tx) + '}').format(usr_input).lower() 89 | #generated += sentence 90 | # 91 | #sys.stdout.write(usr_input) 92 | 93 | #for i in range(400): 94 | """ 95 | #x_pred = np.zeros((1, Tx, len(chars))) 96 | 97 | for t, char in enumerate(sentence): 98 | if char != '0': 99 | x_pred[0, t, char_indices[char]] = 1. 100 | 101 | preds = model.predict(x_pred, verbose=0)[0] 102 | next_index = sample(preds, temperature = 1.0) 103 | next_char = indices_char[next_index] 104 | 105 | generated += next_char 106 | sentence = sentence[1:] + next_char 107 | 108 | sys.stdout.write(next_char) 109 | sys.stdout.flush() 110 | 111 | if next_char == '\n': 112 | continue 113 | 114 | # Stop at the end of a line (4 lines) 115 | print() 116 | """ 117 | print("Loading text data...") 118 | text = io.open('shakespeare.txt', encoding='utf-8').read().lower() 119 | #print('corpus length:', len(text)) 120 | 121 | Tx = 40 122 | chars = sorted(list(set(text))) 123 | char_indices = dict((c, i) for i, c in enumerate(chars)) 124 | indices_char = dict((i, c) for i, c in enumerate(chars)) 125 | #print('number of unique characters in the corpus:', len(chars)) 126 | 127 | print("Creating training set...") 128 | X, Y = build_data(text, Tx, stride = 3) 129 | print("Vectorizing training set...") 130 | x, y = vectorization(X, Y, n_x = len(chars), char_indices = char_indices) 131 | print("Loading model...") 132 | model = load_model('models/model_shakespeare_kiank_350_epoch.h5') 133 | 134 | 135 | def generate_output(): 136 | generated = '' 137 | #sentence = text[start_index: start_index + Tx] 138 | #sentence = '0'*Tx 139 | usr_input = input("Write the beginning of your poem, the Shakespeare machine will complete it. Your input is: ") 140 | # zero pad the sentence to Tx characters. 141 | sentence = ('{0:0>' + str(Tx) + '}').format(usr_input).lower() 142 | generated += usr_input 143 | 144 | sys.stdout.write("\n\nHere is your poem: \n\n") 145 | sys.stdout.write(usr_input) 146 | for i in range(400): 147 | 148 | x_pred = np.zeros((1, Tx, len(chars))) 149 | 150 | for t, char in enumerate(sentence): 151 | if char != '0': 152 | x_pred[0, t, char_indices[char]] = 1. 153 | 154 | preds = model.predict(x_pred, verbose=0)[0] 155 | next_index = sample(preds, temperature = 1.0) 156 | next_char = indices_char[next_index] 157 | 158 | generated += next_char 159 | sentence = sentence[1:] + next_char 160 | 161 | sys.stdout.write(next_char) 162 | sys.stdout.flush() 163 | 164 | if next_char == '\n': 165 | continue -------------------------------------------------------------------------------- /5.Sequence Models/1.Recurrent Neural Networks/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def softmax(x): 4 | e_x = np.exp(x - np.max(x)) 5 | return e_x / e_x.sum(axis=0) 6 | 7 | def smooth(loss, cur_loss): 8 | return loss * 0.999 + cur_loss * 0.001 9 | 10 | def print_sample(sample_ix, ix_to_char): 11 | txt = ''.join(ix_to_char[ix] for ix in sample_ix) 12 | txt = txt[0].upper() + txt[1:] # capitalize first character 13 | print ('%s' % (txt, ), end='') 14 | 15 | def get_initial_loss(vocab_size, seq_length): 16 | return -np.log(1.0/vocab_size)*seq_length 17 | 18 | def softmax(x): 19 | e_x = np.exp(x - np.max(x)) 20 | return e_x / e_x.sum(axis=0) 21 | 22 | def initialize_parameters(n_a, n_x, n_y): 23 | """ 24 | Initialize parameters with small random values 25 | 26 | Returns: 27 | parameters -- python dictionary containing: 28 | Wax -- Weight matrix multiplying the input, numpy array of shape (n_a, n_x) 29 | Waa -- Weight matrix multiplying the hidden state, numpy array of shape (n_a, n_a) 30 | Wya -- Weight matrix relating the hidden-state to the output, numpy array of shape (n_y, n_a) 31 | b -- Bias, numpy array of shape (n_a, 1) 32 | by -- Bias relating the hidden-state to the output, numpy array of shape (n_y, 1) 33 | """ 34 | np.random.seed(1) 35 | Wax = np.random.randn(n_a, n_x)*0.01 # input to hidden 36 | Waa = np.random.randn(n_a, n_a)*0.01 # hidden to hidden 37 | Wya = np.random.randn(n_y, n_a)*0.01 # hidden to output 38 | b = np.zeros((n_a, 1)) # hidden bias 39 | by = np.zeros((n_y, 1)) # output bias 40 | 41 | parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "b": b,"by": by} 42 | 43 | return parameters 44 | 45 | def rnn_step_forward(parameters, a_prev, x): 46 | 47 | Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b'] 48 | a_next = np.tanh(np.dot(Wax, x) + np.dot(Waa, a_prev) + b) # hidden state 49 | p_t = softmax(np.dot(Wya, a_next) + by) # unnormalized log probabilities for next chars # probabilities for next chars 50 | 51 | return a_next, p_t 52 | 53 | def rnn_step_backward(dy, gradients, parameters, x, a, a_prev): 54 | 55 | gradients['dWya'] += np.dot(dy, a.T) 56 | gradients['dby'] += dy 57 | da = np.dot(parameters['Wya'].T, dy) + gradients['da_next'] # backprop into h 58 | daraw = (1 - a * a) * da # backprop through tanh nonlinearity 59 | gradients['db'] += daraw 60 | gradients['dWax'] += np.dot(daraw, x.T) 61 | gradients['dWaa'] += np.dot(daraw, a_prev.T) 62 | gradients['da_next'] = np.dot(parameters['Waa'].T, daraw) 63 | return gradients 64 | 65 | def update_parameters(parameters, gradients, lr): 66 | 67 | parameters['Wax'] += -lr * gradients['dWax'] 68 | parameters['Waa'] += -lr * gradients['dWaa'] 69 | parameters['Wya'] += -lr * gradients['dWya'] 70 | parameters['b'] += -lr * gradients['db'] 71 | parameters['by'] += -lr * gradients['dby'] 72 | return parameters 73 | 74 | def rnn_forward(X, Y, a0, parameters, vocab_size = 27): 75 | 76 | # Initialize x, a and y_hat as empty dictionaries 77 | x, a, y_hat = {}, {}, {} 78 | 79 | a[-1] = np.copy(a0) 80 | 81 | # initialize your loss to 0 82 | loss = 0 83 | 84 | for t in range(len(X)): 85 | 86 | # Set x[t] to be the one-hot vector representation of the t'th character in X. 87 | # if X[t] == None, we just have x[t]=0. This is used to set the input for the first timestep to the zero vector. 88 | x[t] = np.zeros((vocab_size,1)) 89 | if (X[t] != None): 90 | x[t][X[t]] = 1 91 | 92 | # Run one step forward of the RNN 93 | a[t], y_hat[t] = rnn_step_forward(parameters, a[t-1], x[t]) 94 | 95 | # Update the loss by substracting the cross-entropy term of this time-step from it. 96 | loss -= np.log(y_hat[t][Y[t],0]) 97 | 98 | cache = (y_hat, a, x) 99 | 100 | return loss, cache 101 | 102 | def rnn_backward(X, Y, parameters, cache): 103 | # Initialize gradients as an empty dictionary 104 | gradients = {} 105 | 106 | # Retrieve from cache and parameters 107 | (y_hat, a, x) = cache 108 | Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b'] 109 | 110 | # each one should be initialized to zeros of the same dimension as its corresponding parameter 111 | gradients['dWax'], gradients['dWaa'], gradients['dWya'] = np.zeros_like(Wax), np.zeros_like(Waa), np.zeros_like(Wya) 112 | gradients['db'], gradients['dby'] = np.zeros_like(b), np.zeros_like(by) 113 | gradients['da_next'] = np.zeros_like(a[0]) 114 | 115 | ### START CODE HERE ### 116 | # Backpropagate through time 117 | for t in reversed(range(len(X))): 118 | dy = np.copy(y_hat[t]) 119 | dy[Y[t]] -= 1 120 | gradients = rnn_step_backward(dy, gradients, parameters, x[t], a[t], a[t-1]) 121 | ### END CODE HERE ### 122 | 123 | return gradients, a 124 | 125 | -------------------------------------------------------------------------------- /5.Sequence Models/2.Natural Language Processing & Word Embeddings/emo_utils.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import numpy as np 3 | import emoji 4 | import pandas as pd 5 | import matplotlib.pyplot as plt 6 | from sklearn.metrics import confusion_matrix 7 | 8 | def read_glove_vecs(glove_file): 9 | with open(glove_file, 'r', encoding='utf8') as f: 10 | words = set() 11 | word_to_vec_map = {} 12 | for line in f: 13 | line = line.strip().split() 14 | curr_word = line[0] 15 | words.add(curr_word) 16 | word_to_vec_map[curr_word] = np.array(line[1:], dtype=np.float64) 17 | 18 | i = 1 19 | words_to_index = {} 20 | index_to_words = {} 21 | for w in sorted(words): 22 | words_to_index[w] = i 23 | index_to_words[i] = w 24 | i = i + 1 25 | return words_to_index, index_to_words, word_to_vec_map 26 | 27 | def softmax(x): 28 | """Compute softmax values for each sets of scores in x.""" 29 | e_x = np.exp(x - np.max(x)) 30 | return e_x / e_x.sum() 31 | 32 | 33 | def read_csv(filename = 'data/emojify_data.csv'): 34 | phrase = [] 35 | emoji = [] 36 | 37 | with open (filename) as csvDataFile: 38 | csvReader = csv.reader(csvDataFile) 39 | 40 | for row in csvReader: 41 | phrase.append(row[0]) 42 | emoji.append(row[1]) 43 | 44 | X = np.asarray(phrase) 45 | Y = np.asarray(emoji, dtype=int) 46 | 47 | return X, Y 48 | 49 | def convert_to_one_hot(Y, C): 50 | Y = np.eye(C)[Y.reshape(-1)] 51 | return Y 52 | 53 | 54 | emoji_dictionary = {"0": "\u2764\uFE0F", # :heart: prints a black instead of red heart depending on the font 55 | "1": ":baseball:", 56 | "2": ":smile:", 57 | "3": ":disappointed:", 58 | "4": ":fork_and_knife:"} 59 | 60 | def label_to_emoji(label): 61 | """ 62 | Converts a label (int or string) into the corresponding emoji code (string) ready to be printed 63 | """ 64 | return emoji.emojize(emoji_dictionary[str(label)], use_aliases=True) 65 | 66 | 67 | def print_predictions(X, pred): 68 | print() 69 | for i in range(X.shape[0]): 70 | print(X[i], label_to_emoji(int(pred[i]))) 71 | 72 | 73 | def plot_confusion_matrix(y_actu, y_pred, title='Confusion matrix', cmap=plt.cm.gray_r): 74 | 75 | df_confusion = pd.crosstab(y_actu, y_pred.reshape(y_pred.shape[0],), rownames=['Actual'], colnames=['Predicted'], margins=True) 76 | 77 | df_conf_norm = df_confusion / df_confusion.sum(axis=1) 78 | 79 | plt.matshow(df_confusion, cmap=cmap) # imshow 80 | #plt.title(title) 81 | plt.colorbar() 82 | tick_marks = np.arange(len(df_confusion.columns)) 83 | plt.xticks(tick_marks, df_confusion.columns, rotation=45) 84 | plt.yticks(tick_marks, df_confusion.index) 85 | #plt.tight_layout() 86 | plt.ylabel(df_confusion.index.name) 87 | plt.xlabel(df_confusion.columns.name) 88 | 89 | 90 | def predict(X, Y, W, b, word_to_vec_map): 91 | """ 92 | Given X (sentences) and Y (emoji indices), predict emojis and compute the accuracy of your model over the given set. 93 | 94 | Arguments: 95 | X -- input data containing sentences, numpy array of shape (m, None) 96 | Y -- labels, containing index of the label emoji, numpy array of shape (m, 1) 97 | 98 | Returns: 99 | pred -- numpy array of shape (m, 1) with your predictions 100 | """ 101 | m = X.shape[0] 102 | pred = np.zeros((m, 1)) 103 | 104 | for j in range(m): # Loop over training examples 105 | 106 | # Split jth test example (sentence) into list of lower case words 107 | words = X[j].lower().split() 108 | 109 | # Average words' vectors 110 | avg = np.zeros((50,)) 111 | for w in words: 112 | avg += word_to_vec_map[w] 113 | avg = avg/len(words) 114 | 115 | # Forward propagation 116 | Z = np.dot(W, avg) + b 117 | A = softmax(Z) 118 | pred[j] = np.argmax(A) 119 | 120 | print("Accuracy: " + str(np.mean((pred[:] == Y.reshape(Y.shape[0],1)[:])))) 121 | 122 | return pred -------------------------------------------------------------------------------- /5.Sequence Models/2.Natural Language Processing & Word Embeddings/sample.txt: -------------------------------------------------------------------------------- 1 | sample 2 | -------------------------------------------------------------------------------- /5.Sequence Models/2.Natural Language Processing & Word Embeddings/w2v_utils.py: -------------------------------------------------------------------------------- 1 | from keras.models import Model 2 | from keras.layers import Input, Dense, Reshape, merge 3 | from keras.layers.embeddings import Embedding 4 | from keras.preprocessing.sequence import skipgrams 5 | from keras.preprocessing import sequence 6 | 7 | import urllib.request 8 | import collections 9 | import os 10 | import zipfile 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | 15 | window_size = 3 16 | vector_dim = 300 17 | epochs = 1000 18 | 19 | valid_size = 16 # Random set of words to evaluate similarity on. 20 | valid_window = 100 # Only pick dev samples in the head of the distribution. 21 | valid_examples = np.random.choice(valid_window, valid_size, replace=False) 22 | 23 | def maybe_download(filename, url, expected_bytes): 24 | """Download a file if not present, and make sure it's the right size.""" 25 | if not os.path.exists(filename): 26 | filename, _ = urllib.request.urlretrieve(url + filename, filename) 27 | statinfo = os.stat(filename) 28 | if statinfo.st_size == expected_bytes: 29 | print('Found and verified', filename) 30 | else: 31 | print(statinfo.st_size) 32 | raise Exception( 33 | 'Failed to verify ' + filename + '. Can you get to it with a browser?') 34 | return filename 35 | 36 | 37 | # Read the data into a list of strings. 38 | def read_data(filename): 39 | """Extract the first file enclosed in a zip file as a list of words.""" 40 | with zipfile.ZipFile(filename) as f: 41 | data = tf.compat.as_str(f.read(f.namelist()[0])).split() 42 | return data 43 | 44 | 45 | def build_dataset(words, n_words): 46 | """Process raw inputs into a dataset.""" 47 | count = [['UNK', -1]] 48 | count.extend(collections.Counter(words).most_common(n_words - 1)) 49 | dictionary = dict() 50 | for word, _ in count: 51 | dictionary[word] = len(dictionary) 52 | data = list() 53 | unk_count = 0 54 | for word in words: 55 | if word in dictionary: 56 | index = dictionary[word] 57 | else: 58 | index = 0 # dictionary['UNK'] 59 | unk_count += 1 60 | data.append(index) 61 | count[0][1] = unk_count 62 | reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys())) 63 | return data, count, dictionary, reversed_dictionary 64 | 65 | def collect_data(vocabulary_size=10000): 66 | url = 'http://mattmahoney.net/dc/' 67 | filename = maybe_download('text8.zip', url, 31344016) 68 | vocabulary = read_data(filename) 69 | print(vocabulary[:7]) 70 | data, count, dictionary, reverse_dictionary = build_dataset(vocabulary, 71 | vocabulary_size) 72 | del vocabulary # Hint to reduce memory. 73 | return data, count, dictionary, reverse_dictionary 74 | 75 | class SimilarityCallback: 76 | def run_sim(self): 77 | for i in range(valid_size): 78 | valid_word = reverse_dictionary[valid_examples[i]] 79 | top_k = 8 # number of nearest neighbors 80 | sim = self._get_sim(valid_examples[i]) 81 | nearest = (-sim).argsort()[1:top_k + 1] 82 | log_str = 'Nearest to %s:' % valid_word 83 | for k in range(top_k): 84 | close_word = reverse_dictionary[nearest[k]] 85 | log_str = '%s %s,' % (log_str, close_word) 86 | print(log_str) 87 | 88 | @staticmethod 89 | def _get_sim(valid_word_idx): 90 | sim = np.zeros((vocab_size,)) 91 | in_arr1 = np.zeros((1,)) 92 | in_arr2 = np.zeros((1,)) 93 | in_arr1[0,] = valid_word_idx 94 | for i in range(vocab_size): 95 | in_arr2[0,] = i 96 | out = validation_model.predict_on_batch([in_arr1, in_arr2]) 97 | sim[i] = out 98 | return sim 99 | 100 | 101 | def read_glove_vecs(glove_file): 102 | with open(glove_file, 'r', encoding='utf8') as f: 103 | words = set() 104 | word_to_vec_map = {} 105 | 106 | for line in f: 107 | line = line.strip().split() 108 | curr_word = line[0] 109 | words.add(curr_word) 110 | word_to_vec_map[curr_word] = np.array(line[1:], dtype=np.float64) 111 | 112 | return words, word_to_vec_map 113 | 114 | def relu(x): 115 | """ 116 | Compute the relu of x 117 | 118 | Arguments: 119 | x -- A scalar or numpy array of any size. 120 | 121 | Return: 122 | s -- relu(x) 123 | """ 124 | s = np.maximum(0,x) 125 | 126 | return s 127 | 128 | 129 | def initialize_parameters(vocab_size, n_h): 130 | """ 131 | Arguments: 132 | layer_dims -- python array (list) containing the dimensions of each layer in our network 133 | 134 | Returns: 135 | parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2": 136 | W1 -- weight matrix of shape (n_h, vocab_size) 137 | b1 -- bias vector of shape (n_h, 1) 138 | W2 -- weight matrix of shape (vocab_size, n_h) 139 | b2 -- bias vector of shape (vocab_size, 1) 140 | """ 141 | 142 | np.random.seed(3) 143 | parameters = {} 144 | 145 | parameters['W1'] = np.random.randn(n_h, vocab_size) / np.sqrt(vocab_size) 146 | parameters['b1'] = np.zeros((n_h, 1)) 147 | parameters['W2'] = np.random.randn(vocab_size, n_h) / np.sqrt(n_h) 148 | parameters['b2'] = np.zeros((vocab_size, 1)) 149 | 150 | return parameters 151 | 152 | def softmax(x): 153 | """Compute softmax values for each sets of scores in x.""" 154 | e_x = np.exp(x - np.max(x)) 155 | return e_x / e_x.sum() 156 | -------------------------------------------------------------------------------- /5.Sequence Models/3.Sequence Models & Attention Mechanism/nmt_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from faker import Faker 3 | import random 4 | from tqdm import tqdm 5 | from babel.dates import format_date 6 | from keras.utils import to_categorical 7 | import keras.backend as K 8 | import matplotlib.pyplot as plt 9 | 10 | fake = Faker() 11 | fake.seed(12345) 12 | random.seed(12345) 13 | 14 | # Define format of the data we would like to generate 15 | FORMATS = ['short', 16 | 'medium', 17 | 'long', 18 | 'full', 19 | 'full', 20 | 'full', 21 | 'full', 22 | 'full', 23 | 'full', 24 | 'full', 25 | 'full', 26 | 'full', 27 | 'full', 28 | 'd MMM YYY', 29 | 'd MMMM YYY', 30 | 'dd MMM YYY', 31 | 'd MMM, YYY', 32 | 'd MMMM, YYY', 33 | 'dd, MMM YYY', 34 | 'd MM YY', 35 | 'd MMMM YYY', 36 | 'MMMM d YYY', 37 | 'MMMM d, YYY', 38 | 'dd.MM.YY'] 39 | 40 | # change this if you want it to work with another language 41 | LOCALES = ['en_US'] 42 | 43 | def load_date(): 44 | """ 45 | Loads some fake dates 46 | :returns: tuple containing human readable string, machine readable string, and date object 47 | """ 48 | dt = fake.date_object() 49 | 50 | try: 51 | human_readable = format_date(dt, format=random.choice(FORMATS), locale='en_US') # locale=random.choice(LOCALES)) 52 | human_readable = human_readable.lower() 53 | human_readable = human_readable.replace(',','') 54 | machine_readable = dt.isoformat() 55 | 56 | except AttributeError as e: 57 | return None, None, None 58 | 59 | return human_readable, machine_readable, dt 60 | 61 | def load_dataset(m): 62 | """ 63 | Loads a dataset with m examples and vocabularies 64 | :m: the number of examples to generate 65 | """ 66 | 67 | human_vocab = set() 68 | machine_vocab = set() 69 | dataset = [] 70 | Tx = 30 71 | 72 | 73 | for i in tqdm(range(m)): 74 | h, m, _ = load_date() 75 | if h is not None: 76 | dataset.append((h, m)) 77 | human_vocab.update(tuple(h)) 78 | machine_vocab.update(tuple(m)) 79 | 80 | human = dict(zip(sorted(human_vocab) + ['', ''], 81 | list(range(len(human_vocab) + 2)))) 82 | inv_machine = dict(enumerate(sorted(machine_vocab))) 83 | machine = {v:k for k,v in inv_machine.items()} 84 | 85 | return dataset, human, machine, inv_machine 86 | 87 | def preprocess_data(dataset, human_vocab, machine_vocab, Tx, Ty): 88 | 89 | X, Y = zip(*dataset) 90 | 91 | X = np.array([string_to_int(i, Tx, human_vocab) for i in X]) 92 | Y = [string_to_int(t, Ty, machine_vocab) for t in Y] 93 | 94 | Xoh = np.array(list(map(lambda x: to_categorical(x, num_classes=len(human_vocab)), X))) 95 | Yoh = np.array(list(map(lambda x: to_categorical(x, num_classes=len(machine_vocab)), Y))) 96 | 97 | return X, np.array(Y), Xoh, Yoh 98 | 99 | def string_to_int(string, length, vocab): 100 | """ 101 | Converts all strings in the vocabulary into a list of integers representing the positions of the 102 | input string's characters in the "vocab" 103 | 104 | Arguments: 105 | string -- input string, e.g. 'Wed 10 Jul 2007' 106 | length -- the number of time steps you'd like, determines if the output will be padded or cut 107 | vocab -- vocabulary, dictionary used to index every character of your "string" 108 | 109 | Returns: 110 | rep -- list of integers (or '') (size = length) representing the position of the string's character in the vocabulary 111 | """ 112 | 113 | #make lower to standardize 114 | string = string.lower() 115 | string = string.replace(',','') 116 | 117 | if len(string) > length: 118 | string = string[:length] 119 | 120 | rep = list(map(lambda x: vocab.get(x, ''), string)) 121 | 122 | if len(string) < length: 123 | rep += [vocab['']] * (length - len(string)) 124 | 125 | #print (rep) 126 | return rep 127 | 128 | 129 | def int_to_string(ints, inv_vocab): 130 | """ 131 | Output a machine readable list of characters based on a list of indexes in the machine's vocabulary 132 | 133 | Arguments: 134 | ints -- list of integers representing indexes in the machine's vocabulary 135 | inv_vocab -- dictionary mapping machine readable indexes to machine readable characters 136 | 137 | Returns: 138 | l -- list of characters corresponding to the indexes of ints thanks to the inv_vocab mapping 139 | """ 140 | 141 | l = [inv_vocab[i] for i in ints] 142 | return l 143 | 144 | 145 | EXAMPLES = ['3 May 1979', '5 Apr 09', '20th February 2016', 'Wed 10 Jul 2007'] 146 | 147 | def run_example(model, input_vocabulary, inv_output_vocabulary, text): 148 | encoded = string_to_int(text, TIME_STEPS, input_vocabulary) 149 | prediction = model.predict(np.array([encoded])) 150 | prediction = np.argmax(prediction[0], axis=-1) 151 | return int_to_string(prediction, inv_output_vocabulary) 152 | 153 | def run_examples(model, input_vocabulary, inv_output_vocabulary, examples=EXAMPLES): 154 | predicted = [] 155 | for example in examples: 156 | predicted.append(''.join(run_example(model, input_vocabulary, inv_output_vocabulary, example))) 157 | print('input:', example) 158 | print('output:', predicted[-1]) 159 | return predicted 160 | 161 | 162 | def softmax(x, axis=1): 163 | """Softmax activation function. 164 | # Arguments 165 | x : Tensor. 166 | axis: Integer, axis along which the softmax normalization is applied. 167 | # Returns 168 | Tensor, output of softmax transformation. 169 | # Raises 170 | ValueError: In case `dim(x) == 1`. 171 | """ 172 | ndim = K.ndim(x) 173 | if ndim == 2: 174 | return K.softmax(x) 175 | elif ndim > 2: 176 | e = K.exp(x - K.max(x, axis=axis, keepdims=True)) 177 | s = K.sum(e, axis=axis, keepdims=True) 178 | return e / s 179 | else: 180 | raise ValueError('Cannot apply softmax to a tensor that is 1D') 181 | 182 | 183 | def plot_attention_map(model, input_vocabulary, inv_output_vocabulary, text, n_s = 128, num = 6, Tx = 30, Ty = 10): 184 | """ 185 | Plot the attention map. 186 | 187 | """ 188 | attention_map = np.zeros((10, 30)) 189 | Ty, Tx = attention_map.shape 190 | 191 | s0 = np.zeros((1, n_s)) 192 | c0 = np.zeros((1, n_s)) 193 | layer = model.layers[num] 194 | 195 | encoded = np.array(string_to_int(text, Tx, input_vocabulary)).reshape((1, 30)) 196 | encoded = np.array(list(map(lambda x: to_categorical(x, num_classes=len(input_vocabulary)), encoded))) 197 | 198 | f = K.function(model.inputs, [layer.get_output_at(t) for t in range(Ty)]) 199 | r = f([encoded, s0, c0]) 200 | 201 | for t in range(Ty): 202 | for t_prime in range(Tx): 203 | attention_map[t][t_prime] = r[t][0,t_prime,0] 204 | 205 | # Normalize attention map 206 | # row_max = attention_map.max(axis=1) 207 | # attention_map = attention_map / row_max[:, None] 208 | 209 | prediction = model.predict([encoded, s0, c0]) 210 | 211 | predicted_text = [] 212 | for i in range(len(prediction)): 213 | predicted_text.append(int(np.argmax(prediction[i], axis=1))) 214 | 215 | predicted_text = list(predicted_text) 216 | predicted_text = int_to_string(predicted_text, inv_output_vocabulary) 217 | text_ = list(text) 218 | 219 | # get the lengths of the string 220 | input_length = len(text) 221 | output_length = Ty 222 | 223 | # Plot the attention_map 224 | plt.clf() 225 | f = plt.figure(figsize=(8, 8.5)) 226 | ax = f.add_subplot(1, 1, 1) 227 | 228 | # add image 229 | i = ax.imshow(attention_map, interpolation='nearest', cmap='Blues') 230 | 231 | # add colorbar 232 | cbaxes = f.add_axes([0.2, 0, 0.6, 0.03]) 233 | cbar = f.colorbar(i, cax=cbaxes, orientation='horizontal') 234 | cbar.ax.set_xlabel('Alpha value (Probability output of the "softmax")', labelpad=2) 235 | 236 | # add labels 237 | ax.set_yticks(range(output_length)) 238 | ax.set_yticklabels(predicted_text[:output_length]) 239 | 240 | ax.set_xticks(range(input_length)) 241 | ax.set_xticklabels(text_[:input_length], rotation=45) 242 | 243 | ax.set_xlabel('Input Sequence') 244 | ax.set_ylabel('Output Sequence') 245 | 246 | # add grid and legend 247 | ax.grid() 248 | 249 | #f.show() 250 | 251 | return attention_map -------------------------------------------------------------------------------- /5.Sequence Models/3.Sequence Models & Attention Mechanism/sample.txt: -------------------------------------------------------------------------------- 1 | sample 2 | --------------------------------------------------------------------------------